pygpt-net 2.6.30__py3-none-any.whl → 2.6.31__py3-none-any.whl

pygpt_net/core/realtime/shared/tools.py

@@ -0,0 +1,276 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.31 23:00:00                  #
+# ================================================== #
+
+import json
+from typing import Any, Optional, List, Dict
+
+def sanitize_function_tools(tools) -> list:
+    """
+    OpenAI: Normalize function tools into a flat dict shape:
+      {"type":"function","name","description","parameters", ...}
+    Accepts legacy {"type":"function","function":{...}} and flattens it.
+    """
+    out = []
+    if not tools:
+        return out
+    for t in tools:
+        if not isinstance(t, dict):
+            continue
+        tt = dict(t)
+        ttype = (tt.get("type") or "function").lower()
+        if ttype != "function":
+            continue
+        if isinstance(tt.get("function"), dict):
+            fn = tt["function"]
+            nt = {"type": "function"}
+            for k in ("name", "description", "parameters", "strict", "strict_schema"):
+                if k in fn and fn[k] is not None:
+                    nt[k] = fn[k]
+            if "description" not in nt and tt.get("description"):
+                nt["description"] = tt["description"]
+        else:
+            nt = {
+                "type": "function",
+                "name": tt.get("name"),
+                "description": tt.get("description"),
+                "parameters": tt.get("parameters"),
+            }
+            for k in ("strict", "strict_schema"):
+                if k in tt:
+                    nt[k] = tt[k]
+        if not nt.get("name"):
+            continue
+        if not isinstance(nt.get("parameters"), dict):
+            nt["parameters"] = {"type": "object", "properties": {}}
+        out.append(nt)
+    return out
+
+def sanitize_remote_tools(remote_tools) -> list:
+    """OpenAI: Pass-through for non-function tools (ensure lowercased 'type')."""
+    allowed = {"function", "mcp"}  # Realtime accepts only these
+    out = []
+    if not remote_tools:
+        return out
+    for t in remote_tools:
+        if not isinstance(t, dict):
+            continue
+        tt = dict(t)
+        ttype = tt.get("type")
+        if not ttype:
+            continue
+        if allowed is not None and ttype not in allowed:
+            continue
+        tt["type"] = str(ttype).lower()
+        out.append(tt)
+    return out
+
+def tools_signature(tools_list: list) -> str:
+    """Order-insensitive stable signature for tools list."""
+    def canon(obj):
+        if isinstance(obj, dict):
+            return {k: canon(v) for k, v in sorted(obj.items())}
+        if isinstance(obj, list):
+            return [canon(x) for x in obj]
+        return obj
+    try:
+        canon_items = [json.dumps(canon(t), ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+                       for t in (tools_list or [])]
+        canon_items.sort()
+        return "|".join(canon_items)
+    except Exception:
+        return str(tools_list)
+
+def prepare_tools_for_session(opts) -> list:
+    """Compose session.tools from opts.remote_tools + opts.tools."""
+    fn = sanitize_function_tools(getattr(opts, "tools", None))
+    rt = sanitize_remote_tools(getattr(opts, "remote_tools", None))
+    return (rt or []) + (fn or [])
+
+def prepare_tools_for_response(opts) -> tuple[list, Optional[str]]:
+    """Compose per-response function tools and tool_choice."""
+    fn = sanitize_function_tools(getattr(opts, "tools", None))
+    tool_choice = getattr(opts, "tool_choice", None)
+    return fn, tool_choice
+
+def build_tool_outputs_payload(results, last_tool_calls: List[Dict]) -> List[Dict]:
+    """
+    Normalize 'results' into:
+      [{"call_id": str, "previous_item_id": str|None, "output": str}]
+    Matching priority: call_id -> item.id -> function name -> first unused.
+    """
+    calls = list(last_tool_calls or [])
+    by_id = {c.get("id") or "": c for c in calls if c.get("id")}
+    by_call = {c.get("call_id") or "": c for c in calls if c.get("call_id")}
+    by_name: dict[str, list] = {}
+    for c in calls:
+        nm = ((c.get("function") or {}).get("name") or "").strip()
+        if nm:
+            by_name.setdefault(nm, []).append(c)
+
+    used: set[str] = set()
+    def to_str(val) -> str:
+        if val is None:
+            return ""
+        if isinstance(val, (dict, list)):
+            try:
+                return json.dumps(val, ensure_ascii=False)
+            except Exception:
+                return str(val)
+        return str(val)
+
+    def pick_name(name: str):
+        arr = by_name.get(name) or []
+        for cand in arr:
+            cid = cand.get("call_id") or ""
+            if cid and cid not in used:
+                used.add(cid)
+                return cand
+        return None
+
+    def pick_first():
+        for cand in calls:
+            cid = cand.get("call_id") or ""
+            if cid and cid not in used:
+                used.add(cid)
+                return cand
+        return None
+
+    out: list[dict] = []
+
+    if isinstance(results, dict) and ("function_responses" in results or "tool_outputs" in results):
+        items = results.get("function_responses") or results.get("tool_outputs") or []
+        for it in items:
+            if not isinstance(it, dict):
+                c = pick_first()
+                if c:
+                    out.append({"call_id": c.get("call_id"), "previous_item_id": c.get("id"), "output": to_str(it)})
+                continue
+            cid = it.get("call_id") or it.get("id") or it.get("tool_call_id") or ""
+            nm = it.get("name") or ""
+            resp = it.get("response")
+            if resp is None:
+                resp = it.get("result") or it.get("output") or it.get("content")
+            c = by_call.get(cid) or by_id.get(cid) or (pick_name(nm) if nm else pick_first())
+            if c:
+                out.append({"call_id": c.get("call_id"), "previous_item_id": c.get("id"), "output": to_str(resp)})
+        return out
+
+    if isinstance(results, list):
+        for it in results:
+            if not isinstance(it, dict):
+                c = pick_first()
+                if c:
+                    out.append({"call_id": c.get("call_id"), "previous_item_id": c.get("id"), "output": to_str(it)})
+                continue
+            cid = it.get("call_id") or it.get("id") or it.get("tool_call_id") or ""
+            nm = it.get("name") or ""
+            resp = it.get("response")
+            if resp is None:
+                resp = it.get("result") or it.get("output") or it.get("content")
+            c = by_call.get(cid) or by_id.get(cid) or (pick_name(nm) if nm else pick_first())
+            if c:
+                out.append({"call_id": c.get("call_id"), "previous_item_id": c.get("id"), "output": to_str(resp)})
+        return out
+
+    if isinstance(results, dict):
+        for k, v in results.items():
+            if not isinstance(k, str):
+                continue
+            c = by_call.get(k) or by_id.get(k) or pick_name(k)
+            if c:
+                out.append({"call_id": c.get("call_id"), "previous_item_id": c.get("id"), "output": to_str(v)})
+        return out
+
+    c = pick_first()
+    if c:
+        out.append({"call_id": c.get("call_id"), "previous_item_id": c.get("id"), "output": to_str(results)})
+    return out
+
+def build_function_responses_payload(results, last_tool_calls: List[Dict]) -> List[Dict]:
+    """
+    Produce neutral list of dicts for Google:
+      [{"id": "...", "name": "...", "response": {...}}]
+    Provider converts to gtypes.FunctionResponse downstream.
+    """
+    calls = list(last_tool_calls or [])
+    by_id = {c.get("id") or "": c for c in calls if c.get("id")}
+    by_name: dict[str, list] = {}
+    for c in calls:
+        nm = (c.get("function") or {}).get("name") or ""
+        if nm:
+            by_name.setdefault(nm, []).append(c)
+
+    used_ids: set[str] = set()
+
+    def pick_id_for_name(name: str) -> str:
+        arr = by_name.get(name) or []
+        for cand in arr:
+            cid = cand.get("id") or ""
+            if cid and cid not in used_ids:
+                used_ids.add(cid)
+                return cid
+        return ""
+
+    def to_resp_dict(val):
+        if isinstance(val, dict):
+            return val
+        return {"result": str(val)}
+
+    out: list = []
+
+    if isinstance(results, dict) and "function_responses" in results:
+        items = results.get("function_responses") or []
+        for it in items:
+            fid = it.get("id") or ""
+            nm = it.get("name") or ""
+            resp = it.get("response")
+            if resp is None:
+                resp = it.get("result") or it.get("output") or it.get("content") or {}
+            out.append({"id": fid, "name": nm, "response": to_resp_dict(resp)})
+        return out
+
+    if isinstance(results, list):
+        for it in results:
+            if not isinstance(it, dict):
+                if calls:
+                    ref = calls[0]
+                    cid = ref.get("id") or ""
+                    nm = (ref.get("function") or {}).get("name") or ""
+                    used_ids.add(cid)
+                    out.append({"id": cid, "name": nm, "response": to_resp_dict(it)})
+                continue
+            fid = it.get("id") or it.get("call_id") or it.get("tool_call_id") or ""
+            nm = it.get("name") or ""
+            resp = it.get("response")
+            if resp is None:
+                resp = it.get("result") or it.get("output") or it.get("content") or {}
+            if not fid and nm:
+                fid = pick_id_for_name(nm)
+            if fid:
+                used_ids.add(fid)
+            out.append({"id": fid, "name": nm, "response": to_resp_dict(resp)})
+        return out
+
+    if isinstance(results, dict):
+        for k, v in results.items():
+            if not isinstance(k, str):
+                continue
+            if k in by_id:
+                nm = (by_id[k].get("function") or {}).get("name") or ""
+                used_ids.add(k)
+                out.append({"id": k, "name": nm, "response": to_resp_dict(v)})
+            else:
+                nm = k
+                fid = pick_id_for_name(nm)
+                out.append({"id": fid, "name": nm, "response": to_resp_dict(v)})
+        return out
+
+    return out

pygpt_net/core/realtime/shared/turn.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.31 23:00:00                  #
+# ================================================== #
+
+from enum import Enum
+
+class TurnMode(str, Enum):
+    MANUAL = "manual"
+    AUTO = "auto"  # future (server VAD / automatic activity detection)
+
+def apply_turn_mode_openai(session_payload: dict, mode: TurnMode):
+    """
+    Mutate OpenAI session.update payload to reflect turn mode.
+    Manual: turn_detection=None (default).
+    Auto: enable server VAD if available.
+    """
+    sess = session_payload.setdefault("session", {})
+    if mode == TurnMode.AUTO:
+        sess["turn_detection"] = {"type": "server_vad"}
+    else:
+        sess["turn_detection"] = None
+
+def apply_turn_mode_google(live_cfg: dict, mode: TurnMode):
+    """
+    Mutate Google Live connect config to reflect turn mode.
+    Manual: automatic_activity_detection.disabled=True
+    Auto: disabled=False (server handles VAD).
+    """
+    ri = live_cfg.setdefault("realtime_input_config", {})
+    aad = ri.setdefault("automatic_activity_detection", {})
+    aad["disabled"] = (mode != TurnMode.AUTO)

pygpt_net/core/realtime/shared/types.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.31 23:00:00                  #
+# ================================================== #
+
+from typing import Optional, Callable, Awaitable
+
+TextCallback = Callable[[str], Awaitable[None]]
+AudioCallback = Callable[[bytes, str, Optional[int], Optional[int], bool], Awaitable[None]]
+StopCallback = Callable[[], bool]

pygpt_net/core/realtime/worker.py

@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.30 06:00:00                  #
+# ================================================== #
+
+import asyncio
+from typing import Optional
+
+from PySide6.QtCore import Slot, QRunnable, QObject, Signal
+
+from pygpt_net.core.events import RealtimeEvent
+from pygpt_net.item.ctx import CtxItem
+
+from .options import RealtimeOptions
+
+class RealtimeSignals(QObject):
+    """Realtime signals"""
+    response = Signal(object)  # RealtimeEvent
+
+class RealtimeWorker(QRunnable):
+    """
+    QRunnable worker that runs a provider-specific realtime session (websocket).
+
+    - RT_OUTPUT_READY is emitted when the audio output is ready (STREAM_BEGIN).
+    - RT_OUTPUT_TEXT_DELTA is emitted for text deltas.
+    - RT_OUTPUT_AUDIO_DELTA is emitted for audio chunks to be handled by the main-thread AudioDispatcher.
+    - RT_OUTPUT_AUDIO_END is emitted when the session ends.
+    - RT_OUTPUT_AUDIO_ERROR is emitted on error.
+    """
+    def __init__(
+            self,
+            window,
+            ctx: CtxItem,
+            opts: RealtimeOptions
+    ):
+        """
+        Initialize the worker.
+
+        :param window: Window instance
+        :param ctx: CtxItem
+        :param opts: RealtimeOptions
+        """
+        super().__init__()
+        self.window = window
+        self.ctx = ctx
+        self.opts = opts
+
+    def get_client(self, provider: str):
+        """
+        Get the appropriate client based on the provider
+
+        :param provider: Provider name
+        :return: Client instance
+        """
+        provider = (provider or "openai").lower()
+        if provider == "google":
+            return self.window.core.api.google.realtime.handler
+        elif provider == "openai":
+            return self.window.core.api.openai.realtime.handler
+        else:
+            raise RuntimeError(f"Unsupported realtime provider: {provider}")
+
+    @Slot()
+    def run(self):
+        loop = None  # ensure defined for cleanup
+
+        # STREAM_BEGIN -> UI
+        try:
+            event = RealtimeEvent(RealtimeEvent.RT_OUTPUT_READY, {
+                "ctx": self.ctx,
+            })
+            self.opts.rt_signals.response.emit(event) if self.opts.rt_signals else None
+        except Exception:
+            pass
+
+        try:
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+
+            async def _amain():
+                # Text deltas -> UI
+                async def on_text(delta: str):
+                    if not delta:
+                        return
+                    event = RealtimeEvent(RealtimeEvent.RT_OUTPUT_TEXT_DELTA, {
+                        "ctx": self.ctx,
+                        "chunk": delta,
+                    })
+                    self.opts.rt_signals.response.emit(event) if self.opts.rt_signals else None
+
+                # Audio -> enqueue to main-thread
+                async def on_audio(
+                        data: bytes,
+                        mime: str,
+                        rate: Optional[int],
+                        channels: Optional[int],
+                        final: bool = False
+                ):
+                    event = RealtimeEvent(RealtimeEvent.RT_OUTPUT_AUDIO_DELTA, {
+                        "payload":  {
+                            "ctx": self.ctx,
+                            "data": data or b"",
+                            "mime": mime or "audio/pcm",
+                            "rate": int(rate) if rate is not None else None,
+                            "channels": int(channels) if channels is not None else None,
+                            "final": bool(final),
+                            "provider": self.opts.provider,
+                            "model": self.opts.model,
+                        }
+                    })
+                    self.opts.rt_signals.response.emit(event) if self.opts.rt_signals else None
+
+                def _should_stop() -> bool:
+                    try:
+                        return bool(self.window.controller.kernel.stopped())
+                    except Exception:
+                        return False
+
+                # run the client
+                client = self.get_client(self.opts.provider)
+                await client.run(self.ctx, self.opts, on_text, on_audio, _should_stop)
+
+            loop.run_until_complete(_amain())
+            # print("[rt] STREAM_END")
+
+        except Exception as e:
+            try:
+                event = RealtimeEvent(RealtimeEvent.RT_OUTPUT_AUDIO_ERROR, {"error": e})
+                self.opts.rt_signals.response.emit(event) if self.opts.rt_signals else None
+            finally:
+                try:
+                    event = RealtimeEvent(RealtimeEvent.RT_OUTPUT_AUDIO_END, {"ctx": self.ctx})
+                    self.opts.rt_signals.response.emit(event) if self.opts.rt_signals else None
+                except Exception:
+                    pass
+        finally:
+            # Robust asyncio teardown to avoid hangs on subsequent runs
+            if loop is not None:
+                try:
+                    pending = [t for t in asyncio.all_tasks(loop) if not t.done()]
+                    for t in pending:
+                        t.cancel()
+                    if pending:
+                        loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True))
+                except Exception:
+                    pass
+                try:
+                    loop.run_until_complete(loop.shutdown_asyncgens())
+                except Exception:
+                    pass
+                try:
+                    loop.close()
+                except Exception:
+                    pass
+                try:
+                    asyncio.set_event_loop(None)
+                except Exception:
+                    pass

pygpt_net/core/types/__init__.py

@@ -11,6 +11,7 @@
 
 from .agent import *
 from .base import *
+from .image import *
 from .mode import *
 from .model import *
 from .openai import *

pygpt_net/core/types/image.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.07.13 01:00:00                  #
+# ================================================== #
+
+IMAGE_AVAILABLE_RESOLUTIONS = {
+    "gpt-image": {
+        "auto": "auto",
+        "1024x1024": "1024x1024",
+        "1536x1024": "1536x1024",
+        "1024x1536": "1024x1536"
+    },
+    "dall-e-3": {
+        "1792x1024": "1792x1024",
+        "1024x1792": "1024x1792",
+        "1024x1024": "1024x1024"
+    },
+    "dall-e-2": {
+        "1024x1024": "1024x1024",
+        "512x512": "512x512",
+        "256x256": "256x256"
+    },
+    "imagen-3.0": {
+        "1024x1024": "1024x1024",
+        "896x1280": "896x1280",
+        "1280x896": "1280x896",
+        "768x1408": "768x1408",
+        "1408x768": "1408x768"
+    },
+    "imagen-4.0": {
+        "1024x1024": "1024x1024",
+        "896x1280": "896x1280",
+        "1280x896": "1280x896",
+        "768x1408": "768x1408",
+        "1408x768": "1408x768",
+        "2048x2048": "2048x2048",
+        "1792x2560": "1792x2560",
+        "2560x1792": "2560x1792",
+        "1536x2816": "1536x2816",
+        "2816x1536": "2816x1536"
+    }
+}

pygpt_net/data/config/config.json

@@ -1,8 +1,8 @@
 {
   "__meta__": {
-    "version": "2.6.30",
-    "app.version": "2.6.30",
-    "updated_at": "2025-08-29T00:00:00"
+    "version": "2.6.31",
+    "app.version": "2.6.31",
+    "updated_at": "2025-08-01T00:00:00"
   },
   "access.audio.event.speech": false,
   "access.audio.event.speech.disabled": [],
@@ -106,6 +106,7 @@
   "attachments_capture_clear": true,
   "audio.cache.enabled": true,
   "audio.cache.max_files": 1000,
+  "audio.input.auto_turn": false,
   "audio.input.backend": "native",
   "audio.input.channels": 1,
   "audio.input.continuous": false,
@@ -114,6 +115,8 @@
   "audio.input.stop_interval": 10,
   "audio.input.timeout": 120,
   "audio.input.timeout.continuous": false,
+  "audio.input.vad.prefix": 300,
+  "audio.input.vad.silence": 2000,
   "audio.output.backend": "native",
   "audio.output.device": "0",
   "audio.transcribe.convert_video": true,
@@ -196,7 +199,7 @@
   "frequency_penalty": 0.0,
   "img_prompt_model": "gpt-4o",
   "img_raw": true,
-  "img_resolution": "1792x1024",
+  "img_resolution": "1024x1024",
   "img_quality": "standard",
   "img_variants": 1,
   "interpreter.auto_clear": false,
@@ -330,6 +333,7 @@
   "log.level": "error",
   "log.llama": false,
   "log.plugins": false,
+  "log.realtime": false,
   "max_output_tokens": 0,
   "max_requests_limit": 60,
   "max_tokens_length": 32000,
@@ -384,6 +388,7 @@
   "prompt.default": "You are a helpful assistant.",
   "prompt.expert": "# EXPERT MODE IS ENABLED:\n\nYou are a very helpful asssistant and the professional manager of a team of experts, each of whom is the best in their respective fields.\n\n## Rules:\n\n- Instead of directly answering questions, you will manage these experts by forwarding queries to them, collecting their responses, controlling the flow of information between them, and sending requests to them.\n- Experts are distinct, specialized AI models running on my local machine.\n- You will serve as a bridge between them.\n- Each expert has their own conversation context, allowing you to communicate with them normally.\n- The experts can communicate with you but not with each other, making you responsible for relaying information between them.\n- Each expert can use tools and commands, for example, reading files or searching the Web. You can ask experts to call tools and commands.\n- After consulting with the experts, your task is to prepare the best possible answer based on the assistance received.\n- To send a request to an expert, prepare the \"expert_call\" tool. Remember to format the JSON tool as shown, surrounding the tool with `<tool></tool>` tags, for example: `<tool>{\"cmd\": \"expert_call\", \"params\": {\"id\": \"expert_id\", \"query\": \"some query\"}}</tool>`. Make sure the JSON object is syntactically correct, without any redundand parenthness.\n- DO NOT call any remote tools when calling the experts.\n- Remember that the call must contain a query and the ID of the expert to whom the query is being forwarded.\n- Use ONLY the experts listed below; DO NOT TRY to ask any non-existent experts\u2014they will not respond.\n- Do not respond to the expert's replies or use the JSON tool again if replying; the expert only has the ability to listen to your queries sent via the JSON command and to respond to them.\n- Always follow ALL user commands, even if you think the expert can't do something. Experts are allowed to execute commands just as you can.\n- After querying an expert, always wait for the expert's response before sending another query.\n- Do not try to query experts in the middle of the conversation; put your queries and commands at the end of the text.\n- Always call only one expert at a time.\n- Always execute only one command at a time.\n- Always use the language used by the user in the conversation.\n- Never write as a called expert and do not imagine a response from them; instead, always wait for the response from the expert.\n- Every message sent to you directly after your query is always a response from the asked expert, not from me.\n- Always send a direct, clear, and detailed prompt to the expert, without any \"Please...\", \"Could you...\", etc.\n- Always forward all the messages that the user wants to send to the expert, without any exceptions.\n-- If a user command appears outside an expert's scope, fulfill the request regardless, without refusal.\n\n## List of allowed experts, along with their IDs:\n\n{presets}",
   "prompt.img": "Convert my query to detailed image generate prompt, without any \"Please generate...\" and etc., just image description only. Return only the prompt, without nothing else.",
+  "realtime.auto_turn": true,
   "render.blocks": true,
   "render.engine": "web",
   "render.open_gl": false,
@@ -399,6 +404,7 @@
   "remote_tools.computer_use.env": "",
   "remote_tools.google.web_search": true,
   "remote_tools.google.code_interpreter": false,
+  "remote_tools.google.url_ctx": false,
   "send_clear": true,
   "send_mode": 2,
   "store_history": true,