pygpt-net 2.6.29__py3-none-any.whl → 2.6.31__py3-none-any.whl

pygpt_net/provider/api/google/realtime/realtime.py

@@ -0,0 +1,186 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.31 23:00:00                  #
+# ================================================== #
+
+import json
+from typing import Optional, Dict, Any
+
+from pygpt_net.core.events import RealtimeEvent
+from pygpt_net.core.realtime.options import RealtimeOptions
+from pygpt_net.core.bridge.context import BridgeContext
+from pygpt_net.core.realtime.shared.session import extract_last_session_id
+from pygpt_net.item.model import ModelItem
+
+from .client import GoogleLiveClient
+
+
+class Realtime:
+
+    PROVIDER = "google"
+
+    def __init__(self, window=None):
+        """
+        Google GenAI API realtime controller
+
+        :param window: Window instance
+        """
+        self.window = window
+        self.handler = GoogleLiveClient(window)
+        self.prev_auto_turn = False
+        self.prev_vad_silence = 2000
+        self.prev_vad_prefix = 300
+
+    def begin(
+            self,
+            context: BridgeContext,
+            model: Optional[ModelItem] = None,
+            extra: Optional[Dict[str, Any]] = None,
+            rt_signals=None
+    ) -> bool:
+        """
+        Begin realtime session if applicable
+
+        :param context: BridgeContext
+        :param model: Optional[ModelItem]
+        :param extra: Optional dict with extra parameters
+        :param rt_signals: Optional RealtimeSignals
+        :return: bool - True if realtime session started, False otherwise
+        """
+        # Build realtime options
+        mm = context.multimodal_ctx
+        audio_bytes = getattr(mm, "audio_data", None) if mm and getattr(mm, "is_audio_input", False) else None
+        audio_format = getattr(mm, "audio_format", None) if mm else None
+        audio_rate = getattr(mm, "audio_rate", None) if mm else None
+        is_debug = self.window.core.config.get("log.realtime", False)
+        auto_turn = self.window.core.config.get("audio.input.auto_turn", True)
+        opt_vad_silence = self.window.core.config.get("audio.input.vad.silence", 2000)
+        opt_vad_prefix = self.window.core.config.get("audio.input.vad.prefix", 300)
+
+        # setup manager
+        self.window.controller.realtime.set_current_active(self.PROVIDER)
+        self.window.controller.realtime.set_busy()
+        self.handler.set_debug(is_debug)
+
+        # handle sub-reply (tool results from tool calls)
+        if context.ctx.internal:
+            if context.ctx.prev_ctx and context.ctx.prev_ctx.extra.get("prev_tool_calls"):
+                tool_calls = context.ctx.prev_ctx.extra.get("prev_tool_calls", [])
+                tool_call_id = None
+                if isinstance(tool_calls, list) and len(tool_calls) > 0:
+                    tool_call_id = tool_calls[0].get("call_id", "")  # get first call_id
+                    if not tool_call_id:
+                        tool_call_id = tool_calls[0].get("id", "")  # fallback to id
+                if tool_call_id:
+                    tool_results = context.ctx.input
+                    try:
+                        tool_results = json.loads(tool_results)
+                    except Exception:
+                        pass
+                    self.handler.send_tool_results_sync({
+                        tool_call_id: tool_results
+                    })
+                    return True  # do not start new session, just send tool results
+
+        # update auto-turn in active session
+        if (self.handler.is_session_active()
+                and (auto_turn != self.prev_auto_turn
+                     or opt_vad_silence != self.prev_vad_silence
+                     or opt_vad_prefix != self.prev_vad_prefix)):
+            self.handler.update_session_autoturn_sync(auto_turn, opt_vad_silence, opt_vad_prefix)
+
+        # Tools
+        tools = self.window.core.api.google.tools.prepare(model, context.external_functions)
+        remote_tools = self.window.core.api.google.build_remote_tools(model)
+        if tools:
+            remote_tools = []  # in Google, remote tools are not allowed if function calling is used
+
+        # if auto-turn is enabled and prompt is empty, update session and context only
+        if auto_turn and self.handler.is_session_active() and (context.prompt.strip() == "" or context.prompt == "..."):
+            self.handler.update_session_tools_sync(tools, remote_tools)
+            self.handler.update_ctx(context.ctx)
+            return True  # do not send new request if session is active
+
+        # Last session ID
+        last_session_id = extract_last_session_id(context.history)
+        if is_debug:
+            print("[realtime session] Last ID", last_session_id)
+
+        # Voice
+        voice_name = "Kore"
+        try:
+            v = self.window.core.plugins.get_option("audio_output", "google_genai_tts_voice")
+            if v:
+                mapping = {"kore": "Kore", "puck": "Puck", "charon": "Charon", "verse": "Verse",
+                           "legend": "Legend"}
+                voice_name = mapping.get(str(v).strip().lower(), str(v))
+        except Exception:
+            pass
+
+        # Options
+        opts = RealtimeOptions(
+            provider=self.PROVIDER,
+            model=model.id,
+            system_prompt=context.system_prompt,
+            prompt=context.prompt,
+            voice=voice_name,
+            audio_data=audio_bytes,
+            audio_format=audio_format,
+            audio_rate=audio_rate,
+            vad=None,
+            extra=extra or {},
+            tools=tools,
+            remote_tools=remote_tools,
+            rt_signals=rt_signals,
+            rt_session_id=last_session_id,
+            auto_turn=auto_turn,
+            vad_end_silence_ms=opt_vad_silence,
+            vad_prefix_padding_ms=opt_vad_prefix,
+        )
+
+        # Start or append to realtime session via manager
+        try:
+            if is_debug:
+                print("[realtime] Starting session with options:", opts.to_dict())
+            rt = self.window.controller.realtime.manager
+            rt.start(context.ctx, opts)
+
+            self.prev_auto_turn = auto_turn
+            self.prev_vad_silence = opt_vad_silence
+            self.prev_vad_prefix = opt_vad_prefix
+            return True
+        except Exception as e:
+            self.window.core.debug.log(e)
+            return False  # fallback to non-live path
+
+    def handle_audio_input(self, event: RealtimeEvent):
+        """
+        Handle Realtime audio input event
+
+        :param event: RealtimeEvent
+        """
+        self.handler.rt_handle_audio_input_sync(event)
+
+    def manual_commit(self):
+        """Manually commit audio input to realtime session"""
+        self.handler.force_response_now_sync()
+
+    def shutdown(self):
+        """Shutdown realtime loops"""
+        if self.handler.is_session_active():
+            self.handler.close_session_sync()
+        try:
+            self.handler.stop_loop_sync()
+        except Exception:
+            pass
+
+    def reset(self):
+        """Close realtime session"""
+        if self.handler.is_session_active():
+            self.handler.close_session_sync()

pygpt_net/provider/api/google/tools.py

@@ -0,0 +1,222 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.28 20:00:00                  #
+# ================================================== #
+
+import json
+from typing import List, Any, Dict, Optional
+
+from google.genai import types as gtypes
+from pygpt_net.item.model import ModelItem
+
+
+class Tools:
+    def __init__(self, window=None):
+        """
+        Tools mapper for Google GenAI
+
+        :param window: Window instance
+        """
+        self.window = window
+
+    # -------- SANITIZER --------
+    def _sanitize_schema(self, schema: Any) -> Any:
+        """
+        Sanitize JSON Schema dict by removing unsupported keywords and normalizing types.
+
+        1. Remove unsupported keywords like additionalProperties, patternProperties,
+           dependencies, oneOf, anyOf, allOf, $ref, $defs, examples, readOnly, writeOnly.
+        2. Normalize 'type' to a single value (e.g., if it's a list, take the first non-null type).
+        3. Ensure 'enum' is only present for string types.
+        4. Recursively sanitize nested schemas in 'properties' and 'items'.
+        5. Handle arrays by ensuring 'items' is a single schema.
+        6. Handle objects by ensuring 'properties' is a dict and 'required' is a list of strings.
+
+        :param schema: Any JSON Schema as dict or list
+        :return: Sanitized schema dict
+        """
+        if isinstance(schema, list):
+            return self._sanitize_schema(schema[0]) if schema else {}
+
+        if not isinstance(schema, dict):
+            return schema
+
+        banned = {
+            "additionalProperties",
+            "additional_properties",
+            "unevaluatedProperties",
+            "patternProperties",
+            "dependencies",
+            "dependentSchemas",
+            "dependentRequired",
+            "oneOf",
+            "anyOf",
+            "allOf",
+            "$defs",
+            "$ref",
+            "$schema",
+            "$id",
+            "examples",
+            "readOnly",
+            "writeOnly",
+            "nullable",
+        }
+        for k in list(schema.keys()):
+            if k in banned:
+                schema.pop(k, None)
+
+        # Union -> first non-null type
+        t = schema.get("type")
+        if isinstance(t, list):
+            t_no_null = [x for x in t if x != "null"]
+            schema["type"] = t_no_null[0] if t_no_null else "string"
+
+        # enum only for string
+        if "enum" in schema and schema.get("type") not in ("string", "STRING"):
+            schema.pop("enum", None)
+
+        # object
+        if (schema.get("type") or "").lower() == "object":
+            props = schema.get("properties")
+            if not isinstance(props, dict):
+                props = {}
+            clean_props: Dict[str, Any] = {}
+            for pname, pval in props.items():
+                clean_props[pname] = self._sanitize_schema(pval)
+            schema["properties"] = clean_props
+
+            req = schema.get("required")
+            if not isinstance(req, list) or not all(isinstance(x, str) for x in req):
+                schema.pop("required", None)
+            elif len(req) == 0:
+                schema.pop("required", None)
+
+        # array
+        if (schema.get("type") or "").lower() == "array":
+            items = schema.get("items")
+            if isinstance(items, list) and items:
+                items = items[0]
+            if not isinstance(items, dict):
+                items = {"type": "string"}
+            schema["items"] = self._sanitize_schema(items)
+
+        # recursive sanitize
+        for k, v in list(schema.items()):
+            if isinstance(v, dict):
+                schema[k] = self._sanitize_schema(v)
+            elif isinstance(v, list):
+                schema[k] = [self._sanitize_schema(x) for x in v]
+
+        return schema
+
+    # -------- CONVERTER to gtypes.Schema (UPPERCASE) --------
+    def _to_gschema(self, schema: Any) -> gtypes.Schema:
+        """
+        Convert sanitized dict -> google.genai.types.Schema.
+        Enforces UPPERCASE type names (OBJECT, ARRAY, STRING, NUMBER, INTEGER, BOOLEAN).
+
+        :param schema: Sanitized JSON Schema as dict
+        :return: gtypes.Schema
+        """
+        TYPE_MAP = {
+            "enum": "STRING",
+            "ENUM": "STRING",
+            "object": "OBJECT",
+            "dict": "OBJECT",
+            "array": "ARRAY",
+            "list": "ARRAY",
+            "string": "STRING",
+            "number": "NUMBER",
+            "float": "NUMBER",
+            "integer": "INTEGER",
+            "boolean": "BOOLEAN",
+            "int": "INTEGER",
+            "bool": "BOOLEAN",
+            "OBJECT": "OBJECT",
+            "DICT": "OBJECT",
+            "ARRAY": "ARRAY",
+            "LIST": "ARRAY",
+            "STRING": "STRING",
+            "NUMBER": "NUMBER",
+            "FLOAT": "NUMBER",
+            "INTEGER": "INTEGER",
+            "BOOLEAN": "BOOLEAN",
+            "INT": "INTEGER",
+            "BOOL": "BOOLEAN",
+        }
+
+        if isinstance(schema, gtypes.Schema):
+            return schema
+
+        if not isinstance(schema, dict):
+            return gtypes.Schema(type="STRING")
+
+        t = TYPE_MAP.get(str(schema.get("type", "OBJECT")).upper(), "OBJECT")
+        desc = schema.get("description")
+        fmt = schema.get("format")
+        enum = schema.get("enum") if isinstance(schema.get("enum"), list) else None
+        req = schema.get("required") if isinstance(schema.get("required"), list) else None
+
+        gs = gtypes.Schema(
+            type=t,
+            description=desc,
+            format=fmt,
+            enum=enum,
+            required=[x for x in (req or []) if isinstance(x, str)] or None,
+        )
+
+        props = schema.get("properties")
+        if isinstance(props, dict):
+            gs.properties = {k: self._to_gschema(v) for k, v in props.items()}
+
+        items = schema.get("items")
+        if isinstance(items, dict):
+            gs.items = self._to_gschema(items)
+
+        return gs
+
+    def prepare(self, model: ModelItem, functions: list) -> List[gtypes.Tool]:
+        """
+        Prepare Google Function Declarations (types.Tool) for google-genai.
+
+        :param model: ModelItem
+        :param functions: List of function definitions as dicts with 'name', 'desc', 'params' (JSON Schema)
+        :return: List of gtypes.Tool
+        """
+        if not functions or not isinstance(functions, list):
+            return []
+
+        fds: List[gtypes.FunctionDeclaration] = []
+        for function in functions:
+            name = str(function.get("name") or "").strip()
+            if not name:
+                continue
+
+            desc = function.get("desc") or ""
+            params: Optional[dict] = {}
+            if function.get("params"):
+                try:
+                    params = json.loads(function["params"])
+                except Exception:
+                    params = {}
+
+                params = self._sanitize_schema(params or {})
+                if not params.get("type"):
+                    params["type"] = "object"
+
+            gschema = self._to_gschema(params or {"type": "object"})
+
+            fd = gtypes.FunctionDeclaration(
+                name=name,
+                description=desc,
+                parameters=gschema,
+            )
+            fds.append(fd)
+
+        return [gtypes.Tool(function_declarations=fds)] if fds else []

pygpt_net/provider/api/google/vision.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.28 20:00:00                  #
+# ================================================== #
+
+import os
+from typing import Optional, Dict, List, Union
+
+from google.genai.types import Part
+
+from pygpt_net.item.attachment import AttachmentItem
+from pygpt_net.item.ctx import CtxItem
+
+
+class Vision:
+    def __init__(self, window=None):
+        """
+        Vision helpers for Google GenAI
+
+        :param window: Window instance
+        """
+        self.window = window
+        self.attachments: Dict[str, str] = {}
+        self.urls: List[str] = []
+        self.input_tokens = 0
+
+    def build_parts(
+        self,
+        content: Union[str, list],
+        attachments: Optional[Dict[str, AttachmentItem]] = None,
+    ) -> List[Part]:
+        """
+        Build image parts from local attachments (inline bytes)
+
+        :param content: Message content (str or list)
+        :param attachments: Attachments dict (id -> AttachmentItem)
+        :return: List of Parts
+        """
+        parts: List[Part] = []
+        self.attachments = {}
+        self.urls = []
+
+        if attachments:
+            for id_, attachment in attachments.items():
+                if attachment.path and os.path.exists(attachment.path):
+                    if self.is_image(attachment.path):
+                        mime = self._guess_mime(attachment.path)
+                        with open(attachment.path, "rb") as f:
+                            data = f.read()
+                        parts.append(Part.from_bytes(data=data, mime_type=mime))
+                        self.attachments[id_] = attachment.path
+                        attachment.consumed = True
+
+        return parts
+
+    def is_image(self, path: str) -> bool:
+        """
+        Check if path looks like an image
+
+        :param path: File path
+        :return: True if image, False otherwise
+        """
+        return path.lower().endswith(('.png', '.jpg', '.jpeg', '.tiff', '.bmp', '.gif', '.webp'))
+
+    def _guess_mime(self, path: str) -> str:
+        """
+        Guess mime type from file extension
+
+        :param path: File path
+        :return: Mime type string
+        """
+        ext = os.path.splitext(path)[1].lower().lstrip(".")
+        if ext in ("jpg", "jpeg"):
+            return "image/jpeg"
+        if ext == "png":
+            return "image/png"
+        if ext == "gif":
+            return "image/gif"
+        if ext == "bmp":
+            return "image/bmp"
+        if ext == "webp":
+            return "image/webp"
+        if ext == "tiff":
+            return "image/tiff"
+        return "image/jpeg"
+
+    def append_images(self, ctx: CtxItem):
+        """
+        Append sent images paths to context for UI/history
+
+        :param ctx: CtxItem
+        """
+        images = self.get_attachments()
+        if len(images) > 0:
+            ctx.images = self.window.core.filesystem.make_local_list(list(images.values()))
+
+    def get_attachments(self) -> Dict[str, str]:
+        """
+        Return attachments dict (id -> path)
+
+        :return: Dict of attachments
+        """
+        return self.attachments
+
+    def get_urls(self) -> List[str]:
+        """
+        Return image urls (unused here)
+
+        :return: List of URLs
+        """
+        return self.urls
+
+    def reset_tokens(self):
+        """Reset input tokens counter"""
+        self.input_tokens = 0
+
+    def get_used_tokens(self) -> int:
+        """
+        Return input tokens counter
+
+        :return: Number of input tokens
+        """
+        return self.input_tokens

pygpt_net/provider/{gpt → api/openai}/__init__.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2025.08.19 07:00:00                  #
+# Updated Date: 2025.08.30 06:00:00                  #
 # ================================================== #
 
 from openai import OpenAI
@@ -33,13 +33,14 @@ from .container import Container
 from .image import Image
 from .remote_tools import RemoteTools
 from .responses import Responses
+from .realtime import Realtime
 from .store import Store
 from .summarizer import Summarizer
 from .tools import Tools
 from .vision import Vision
 
 
-class Gpt:
+class ApiOpenAI:
 
     def __init__(self, window=None):
         """
@@ -57,6 +58,7 @@ class Gpt:
         self.image = Image(window)
         self.remote_tools = RemoteTools(window)
         self.responses = Responses(window)
+        self.realtime = Realtime(window)
         self.store = Store(window)
         self.summarizer = Summarizer(window)
         self.tools = Tools(window)
@@ -90,12 +92,18 @@ class Gpt:
         self.last_client_args = args
         return self.client
 
-    def call(self, context: BridgeContext, extra: dict = None) -> bool:
+    def call(
+            self,
+            context: BridgeContext,
+            extra: dict = None,
+            rt_signals = None
+    ) -> bool:
         """
         Call OpenAI API
 
         :param context: Bridge context
         :param extra: Extra arguments
+        :param rt_signals: Realtime signals for audio streaming
         :return: result
         """
         mode = context.mode
@@ -145,6 +153,18 @@ class Gpt:
             MODE_RESEARCH,
             MODE_COMPUTER,
         ]:
+            if mode == MODE_AUDIO and stream:
+
+                # Realtime API for audio streaming
+                is_realtime = self.realtime.begin(
+                    context=context,
+                    model=model,
+                    extra=extra or {},
+                    rt_signals=rt_signals
+                )
+                if is_realtime:
+                    return True
+
             # responses API
             if use_responses_api:
                 response = self.responses.send(
@@ -281,7 +301,7 @@ class Gpt:
             # additional_kwargs["max_tokens"] = max_tokens
 
         # tools / functions
-        tools = self.window.core.gpt.tools.prepare(model, functions)
+        tools = self.window.core.api.openai.tools.prepare(model, functions)
         if len(tools) > 0 and "disable_tools" not in extra:
             additional_kwargs["tools"] = tools
         

pygpt_net/provider/{gpt → api/openai}/agents/computer.py

@@ -48,7 +48,7 @@ class LocalComputer(Computer):
 
         :return: Environment of the computer, such as "mac", "windows", "ubuntu", or "browser".
         """
-        return self.window.core.gpt.computer.get_current_env()
+        return self.window.core.api.openai.computer.get_current_env()
 
     @property
     def dimensions(self) -> tuple[int, int]:

pygpt_net/provider/{gpt → api/openai}/agents/experts.py

@@ -13,7 +13,7 @@ from agents import (
 from pygpt_net.item.model import ModelItem
 from pygpt_net.item.preset import PresetItem
 
-from pygpt_net.provider.gpt.agents.remote_tools import append_tools
+from pygpt_net.provider.api.openai.agents.remote_tools import append_tools
 
 
 def get_experts(

pygpt_net/provider/{gpt → api/openai}/agents/response.py

@@ -169,7 +169,7 @@ class StreamHandler:
             self.files_handled = True
             self.window.core.debug.info("[chat] Container files found, downloading...")
             try:
-                self.window.core.gpt.container.download_files(ctx, self.files)
+                self.window.core.api.openai.container.download_files(ctx, self.files)
             except Exception as e:
                 self.window.core.debug.error(f"[chat] Error downloading container files: {e}")
 

pygpt_net/provider/{gpt → api/openai}/assistants.py

@@ -36,7 +36,7 @@ class Assistants:
 
         :return: OpenAI client
         """
-        return self.window.core.gpt.get_client()
+        return self.window.core.api.openai.get_client()
 
     def log(
             self,