pygpt-net 2.6.30__py3-none-any.whl → 2.6.32__py3-none-any.whl

pygpt_net/provider/api/openai/realtime/realtime.py

@@ -0,0 +1,193 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.09.01 23:00:00                  #
+# ================================================== #
+
+import json
+from typing import Optional, Dict, Any
+
+from pygpt_net.core.bridge import BridgeContext
+from pygpt_net.core.events import RealtimeEvent
+from pygpt_net.core.realtime.options import RealtimeOptions
+from pygpt_net.core.realtime.shared.session import extract_last_session_id
+from pygpt_net.item.model import ModelItem
+from pygpt_net.utils import trans
+
+from .client import OpenAIRealtimeClient
+
+class Realtime:
+
+    PROVIDER = "openai"
+
+    def __init__(self, window=None):
+        """
+        OpenAI API realtime controller
+
+        :param window: Window instance
+        """
+        self.window = window
+        self.handler = OpenAIRealtimeClient(window)
+        self.prev_auto_turn = False
+        self.prev_vad_silence = 2000
+        self.prev_vad_prefix = 300
+
+    def begin(
+            self,
+            context: BridgeContext,
+            model: Optional[ModelItem] = None,
+            extra: Optional[Dict[str, Any]] = None,
+            rt_signals=None
+    ) -> bool:
+        """
+        Begin realtime session if applicable
+
+        :param context: BridgeContext
+        :param model: Optional[ModelItem]
+        :param extra: Optional dict with extra parameters
+        :param rt_signals: RealtimeSignals
+        :return: True if realtime session started, False otherwise
+        """
+        mm = context.multimodal_ctx
+        audio_bytes = getattr(mm, "audio_data", None) if mm and getattr(mm, "is_audio_input", False) else None
+        audio_format = getattr(mm, "audio_format", None) if mm else None
+        audio_rate = getattr(mm, "audio_rate", None) if mm else None
+        is_debug = self.window.core.config.get("log.realtime", False)
+        auto_turn = self.window.core.config.get("audio.input.auto_turn", True)
+        opt_vad_silence = self.window.core.config.get("audio.input.vad.silence", 2000)
+        opt_vad_prefix = self.window.core.config.get("audio.input.vad.prefix", 300)
+
+        # setup manager
+        self.window.controller.realtime.set_current_active(self.PROVIDER)
+        self.window.controller.realtime.set_busy()
+        self.handler.set_debug(is_debug)
+
+        # tools
+        tools = self.window.core.api.openai.tools.prepare(model, context.external_functions)
+
+        # remote tools
+        remote_tools = []
+        remote_tools = self.window.core.api.openai.remote_tools.append_to_tools(
+            mode=context.mode,
+            model=model,
+            stream=context.stream,
+            is_expert_call=context.is_expert_call,
+            tools=remote_tools,
+            preset=context.preset,
+        )
+
+        # handle sub-reply (tool results from tool calls)
+        if context.ctx.internal:
+            if context.ctx.prev_ctx and context.ctx.prev_ctx.extra.get("prev_tool_calls"):
+                tool_calls = context.ctx.prev_ctx.extra.get("prev_tool_calls", [])
+                tool_call_id = None
+                if isinstance(tool_calls, list) and len(tool_calls) > 0:
+                    tool_call_id = tool_calls[0].get("call_id", "")  # get first call_id
+                    if not tool_call_id:
+                        tool_call_id = tool_calls[0].get("id", "")  # fallback to id
+                if tool_call_id:
+                    tool_results = context.ctx.input
+                    try:
+                        tool_results = json.loads(tool_results)
+                    except Exception:
+                        pass
+                    self.handler.send_tool_results_sync({
+                        tool_call_id: tool_results
+                    })
+                    self.handler.update_ctx(context.ctx)
+                    return True  # do not start new session, just send tool results
+
+        # update auto-turn in active session
+        if (self.handler.is_session_active()
+                and (auto_turn != self.prev_auto_turn
+                     or opt_vad_silence != self.prev_vad_silence
+                     or opt_vad_prefix != self.prev_vad_prefix)):
+            self.handler.update_session_autoturn_sync(auto_turn, opt_vad_silence, opt_vad_prefix)
+
+        # if auto-turn is enabled and prompt is empty, update session and context only
+        if auto_turn and self.handler.is_session_active() and (context.prompt.strip() == "" or context.prompt == "..."):
+            self.handler.update_session_tools_sync(tools, remote_tools)
+            self.handler.update_ctx(context.ctx)
+            self.window.update_status(trans("speech.listening"))
+            return True # do not send new request if session is active
+
+        # Last session ID
+        last_session_id = extract_last_session_id(context.history)
+        if is_debug:
+            print("[realtime session] Last ID", last_session_id)
+
+        # Voice
+        voice = "alloy"
+        try:
+            v = self.window.core.plugins.get_option("audio_output", "openai_voice")
+            if v:
+                voice = str(v)
+        except Exception:
+            pass
+
+        # Options
+        opts = RealtimeOptions(
+            provider=self.PROVIDER,
+            model=context.model.id,
+            system_prompt=context.system_prompt,
+            prompt=context.prompt,
+            voice=voice,
+            audio_data=audio_bytes,
+            audio_format=audio_format,
+            audio_rate=audio_rate,
+            vad="server_vad",
+            extra=extra or {},
+            tools=tools,
+            remote_tools=remote_tools,
+            rt_signals=rt_signals,
+            rt_session_id=last_session_id,
+            auto_turn=auto_turn,
+            vad_end_silence_ms=opt_vad_silence,
+            vad_prefix_padding_ms=opt_vad_prefix,
+        )
+
+        # Start or append to realtime session via manager
+        try:
+            if is_debug:
+                print("[realtime] Starting session with options:", opts.to_dict())
+            rt = self.window.controller.realtime.manager
+            rt.start(context.ctx, opts)
+
+            self.prev_auto_turn = auto_turn
+            self.prev_vad_silence = opt_vad_silence
+            self.prev_vad_prefix = opt_vad_prefix
+            return True
+        except Exception as e:
+            self.window.core.debug.log(e)
+            return False  # fallback to non-live path
+
+    def handle_audio_input(self, event: RealtimeEvent):
+        """
+        Handle Realtime audio input event
+
+        :param event: RealtimeEvent
+        """
+        self.handler.rt_handle_audio_input_sync(event)
+
+    def manual_commit(self):
+        """Manually commit audio input to realtime session"""
+        self.handler.force_response_now_sync()
+
+    def shutdown(self):
+        """Shutdown realtime loops"""
+        if self.handler.is_session_active():
+            self.handler.close_session_sync()
+        try:
+            self.handler.stop_loop_sync()
+        except Exception:
+            pass
+
+    def reset(self):
+        """Close realtime session"""
+        if self.handler.is_session_active():
+            self.handler.close_session_sync()

pygpt_net/provider/audio_input/google_genai.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.29 18:00:00                  #
+# ================================================== #
+
+from .base import BaseProvider
+
+
+class GoogleGenAIAudioInput(BaseProvider):
+
+    PROMPT_TRANSCRIBE = (
+        "You are a speech-to-text transcriber. "
+        "Return only the verbatim transcript as plain text. "
+        "Do not add any explanations, timestamps, labels or formatting."
+    )
+
+    def __init__(self, *args, **kwargs):
+        """
+        Google GenAI (Gemini) audio provider for transcription (via API).
+
+        :param args: args
+        :param kwargs: kwargs
+        """
+        super(GoogleGenAIAudioInput, self).__init__(*args, **kwargs)
+        self.plugin = kwargs.get("plugin")
+        self.id = "google_genai"
+        self.name = "Google GenAI"
+
+    def init_options(self):
+        """Initialize options"""
+        # Keep option shape consistent with Whisper provider
+        self.plugin.add_option(
+            "google_genai_audio_model",
+            type="text",
+            value="gemini-2.5-flash",
+            label="Model",
+            tab="google_genai",
+            description="Specify Gemini model supporting audio, e.g., gemini-2.5-flash",
+        )
+        self.plugin.add_option(
+            "google_genai_audio_prompt",
+            type="textarea",
+            value=self.PROMPT_TRANSCRIBE,
+            label="System Prompt",
+            tab="google_genai",
+            description="System prompt to guide the transcription output",
+            tooltip="System prompt for transcription",
+            persist=True,
+        )
+
+    def transcribe(self, path: str) -> str:
+        """
+        Audio to text transcription using Google GenAI (Gemini).
+
+        :param path: path to audio file to transcribe
+        :return: transcribed text
+        """
+        # Get pre-configured GenAI client from the app core
+        client = self.plugin.window.core.api.google.get_client()
+
+        # Upload the audio file via the Files API
+        uploaded_file = client.files.upload(file=path)
+
+        # Ask the model to produce a plain text transcript only
+        # Using system_instruction keeps the public API surface simple (no extra options needed)
+        config = {
+            "system_instruction": self.plugin.get_option_value("google_genai_audio_prompt") or self.PROMPT_TRANSCRIBE,
+            "temperature": 0.0,
+        }
+
+        # Generate content (transcription) with the selected model
+        model_name = self.plugin.get_option_value("google_genai_audio_model")
+        response = client.models.generate_content(
+            model=model_name,
+            contents=[uploaded_file],
+            config=config,
+        )
+
+        # The SDK exposes the unified .text property for convenience
+        return response.text or ""
+
+    def is_configured(self) -> bool:
+        """
+        Check if provider is configured
+
+        :return: True if configured, False otherwise
+        """
+        api_key = self.plugin.window.core.config.get("api_key_google")
+        return api_key is not None and api_key != ""
+
+    def get_config_message(self) -> str:
+        """
+        Return message to display when provider is not configured
+
+        :return: message
+        """
+        return "Google GenAI API key is not set yet. Please configure it in settings."

pygpt_net/provider/audio_output/google_genai_tts.py

@@ -0,0 +1,229 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.29 18:00:00                  #
+# ================================================== #
+
+import os
+import wave
+import base64
+
+from .base import BaseProvider
+
+
+class GoogleGenAITextToSpeech(BaseProvider):
+    def __init__(self, *args, **kwargs):
+        """
+        Google GenAI Text-to-Speech provider (Gemini TTS via API).
+
+        :param args: args
+        :param kwargs: kwargs
+        """
+        super(GoogleGenAITextToSpeech, self).__init__(*args, **kwargs)
+        self.plugin = kwargs.get("plugin")
+        self.id = "google_genai_tts"
+        self.name = "Google GenAI TTS"
+
+        # Supported preview TTS models (fallback to flash if invalid)
+        self.allowed_models = [
+            "gemini-2.5-flash-preview-tts",
+            "gemini-2.5-pro-preview-tts",
+        ]
+
+        # Prebuilt voice names exposed by Gemini TTS
+        # Keep list in sync with official docs; fallback to "Puck" if invalid.
+        self.allowed_voices = [
+            "Zephyr", "Puck", "Charon", "Kore", "Fenrir", "Leda", "Orus",
+            "Aoede", "Callirrhoe", "Autonoe", "Enceladus", "Iapetus",
+            "Umbriel", "Algieba", "Despina", "Erinome", "Algenib",
+            "Rasalgethi", "Laomedeia", "Achernar", "Alnilam", "Schedar",
+            "Gacrux", "Pulcherrima", "Achird", "Zubenelgenubi",
+            "Vindemiatrix", "Sadachbia", "Sadaltager", "Sulafat",
+        ]
+
+    def init_options(self):
+        """Initialize options"""
+        # Keep option names consistent with the app style; simple text fields are enough.
+        self.plugin.add_option(
+            "google_genai_tts_model",
+            type="text",
+            value="gemini-2.5-flash-preview-tts",
+            label="Model",
+            tab="google_genai_tts",
+            description="Specify Gemini TTS model, e.g.: gemini-2.5-flash-preview-tts or gemini-2.5-pro-preview-tts",
+        )
+        self.plugin.add_option(
+            "google_genai_tts_voice",
+            type="text",
+            value="Kore",
+            label="Voice",
+            tab="google_genai_tts",
+            description="Specify voice, e.g.: Puck, Kore, Charon, Leda, Zephyr... (case-sensitive)",
+            urls={
+                "Voices": "https://ai.google.dev/gemini-api/docs/speech-generation"
+            },
+        )
+
+    def speech(self, text: str) -> str:
+        """
+        Text to speech synthesis using Google GenAI (Gemini TTS).
+
+        :param text: text to synthesize
+        :return: path to generated audio file
+        """
+        # Get pre-configured GenAI client
+        client = self.plugin.window.core.api.google.get_client()
+
+        # Resolve path where audio should be written
+        output_file = self.plugin.output_file
+        path = os.path.join(self.plugin.window.core.config.path, output_file)
+
+        # Validate/select model
+        model = self.plugin.get_option_value("google_genai_tts_model") or "gemini-2.5-flash-preview-tts"
+        model = self._normalize_model_name(model)
+        if model not in self.allowed_models:
+            model = "gemini-2.5-flash-preview-tts"
+
+        # Validate/select voice
+        voice = self.plugin.get_option_value("google_genai_tts_voice") or "Kore"
+        # if voice not in self.allowed_voices:
+            # voice = "Kore"
+
+        # Build generation config for audio modality + voice
+        # Using explicit types for clarity and forward-compatibility
+        try:
+            from google.genai import types
+        except Exception as ex:
+            # Fail fast if SDK is missing or incompatible
+            raise RuntimeError("google.genai SDK is not available. Please install/update Google GenAI SDK.") from ex
+
+        gen_config = types.GenerateContentConfig(
+            response_modalities=["AUDIO"],
+            speech_config=types.SpeechConfig(
+                voice_config=types.VoiceConfig(
+                    prebuilt_voice_config=types.PrebuiltVoiceConfig(
+                        voice_name=voice
+                    )
+                )
+            ),
+            temperature=0.8,  # balanced default; keep configurable later if needed
+        )
+
+        # Perform TTS request
+        response = client.models.generate_content(
+            model=model,
+            contents=text,
+            config=gen_config,
+        )
+
+        # Extract PCM bytes from the first candidate/part
+        pcm = self._extract_pcm_bytes(response)
+
+        # Persist as standard WAV (PCM 16-bit, mono, 24 kHz)
+        self._save_wav(path, pcm, channels=1, rate=24000, sample_width=2)
+
+        return str(path)
+
+    def _extract_pcm_bytes(self, response) -> bytes:
+        """
+        Extract PCM bytes from generate_content response.
+
+        :param response: Google GenAI response object
+        :return: raw PCM byte data
+        """
+        # Defensive extraction to support minor SDK variations
+        data = None
+        try:
+            cand = response.candidates[0]
+            part = cand.content.parts[0]
+            if getattr(part, "inline_data", None) and getattr(part.inline_data, "data", None):
+                data = part.inline_data.data
+        except Exception:
+            pass
+
+        if data is None:
+            # Some SDK builds may return base64 str; try resolving alternative layout
+            try:
+                parts = getattr(response, "candidates", [])[0].content.parts
+                for p in parts:
+                    if getattr(p, "inline_data", None) and getattr(p.inline_data, "data", None):
+                        data = p.inline_data.data
+                        break
+            except Exception:
+                pass
+
+        if data is None:
+            raise RuntimeError("No audio data returned by Gemini TTS response.")
+
+        # Normalize to raw bytes
+        if isinstance(data, (bytes, bytearray)):
+            return bytes(data)
+        if isinstance(data, str):
+            # Fallback: treat as base64-encoded PCM
+            return base64.b64decode(data)
+
+        # Last resort: try bytes() cast
+        try:
+            return bytes(data)
+        except Exception as ex:
+            raise RuntimeError("Unsupported audio payload type returned by Gemini TTS.") from ex
+
+    def _save_wav(
+            self,
+            filename: str,
+            pcm_bytes: bytes,
+            channels: int = 1,
+            rate: int = 24000,
+            sample_width: int = 2
+    ):
+        """
+        Save raw PCM bytes to a WAV file.
+
+        :param filename: output WAV file path
+        :param pcm_bytes: raw PCM byte data
+        :param channels: number of audio channels (1=mono, 2=stereo)
+        :param rate: sample rate in Hz (e.g., 24000)
+        :param sample_width: sample width in bytes (e.g., 2 for 16-bit)
+        """
+        # Ensure parent directory exists
+        os.makedirs(os.path.dirname(filename), exist_ok=True)
+
+        # Write PCM payload as WAV
+        with wave.open(filename, "wb") as wf:
+            wf.setnchannels(channels)
+            wf.setsampwidth(sample_width)  # bytes per sample (2 -> 16-bit)
+            wf.setframerate(rate)
+            wf.writeframes(pcm_bytes)
+
+    def _normalize_model_name(self, model: str) -> str:
+        """
+        Normalize model id (strip optional 'models/' prefix).
+
+        :param model: model id
+        """
+        try:
+            return model.split("/")[-1]
+        except Exception:
+            return model
+
+    def is_configured(self) -> bool:
+        """
+        Check if provider is configured
+
+        :return: True if configured, False otherwise
+        """
+        api_key = self.plugin.window.core.config.get("api_key_google")
+        return api_key is not None and api_key != ""
+
+    def get_config_message(self) -> str:
+        """
+        Return message to display when provider is not configured
+
+        :return: message
+        """
+        return "Google GenAI API key is not set yet. Please configure it in settings."

pygpt_net/provider/audio_output/google_tts.py

@@ -58,18 +58,6 @@ class GoogleTextToSpeech(BaseProvider):
                 "Voices": "https://cloud.google.com/text-to-speech/docs/voices"
             },
         )
-        self.plugin.add_option(
-            "google_voice_native",
-            type="text",
-            value="Kore",
-            label="Voice (Gemini API)",
-            tab="google",
-            description="Specify voice for Gemini API (supported voices may differ)",
-            tooltip="Voice name",
-            urls={
-                "Voices": "https://ai.google.dev/gemini-api/docs/speech-generation"
-            },
-        )
         self.plugin.add_option(
             "google_lang",
             type="text",

pygpt_net/provider/audio_output/openai_tts.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2025.08.07 22:00:00                  #
+# Updated Date: 2025.08.29 18:00:00                  #
 # ================================================== #
 
 import os
@@ -51,6 +51,9 @@ class OpenAITextToSpeech(BaseProvider):
             use="audio_tts_whisper_voices",
             description="Specify voice, available voices: "
                         "alloy, echo, fable, onyx, nova, shimmer",
+            urls={
+                "Voices": "https://platform.openai.com/docs/guides/text-to-speech/voice-options"
+            },
         )
 
     def speech(self, text: str) -> str:
@@ -65,10 +68,10 @@ class OpenAITextToSpeech(BaseProvider):
         voice = self.plugin.get_option_value('openai_voice')
         model = self.plugin.get_option_value('openai_model')
         allowed_voices = self.plugin.window.core.audio.whisper.get_voices()
-        if model not in self.allowed_models:
-            model = 'tts-1'
-        if voice not in allowed_voices:
-            voice = 'alloy'
+        # if model not in self.allowed_models:
+            # model = 'tts-1'
+        # if voice not in allowed_voices:
+            # voice = 'alloy'
         path = os.path.join(
             self.plugin.window.core.config.path,
             output_file,