pygpt-net 2.6.29__py3-none-any.whl → 2.6.31__py3-none-any.whl

pygpt_net/core/realtime/shared/session.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.31 23:00:00                  #
+# ================================================== #
+
+from typing import Optional
+from pygpt_net.item.ctx import CtxItem
+
+def set_ctx_rt_handle(ctx: Optional[CtxItem], handle: Optional[str], window=None):
+    """Persist server session handle into ctx.extra['rt_session_id'] (best effort)."""
+    try:
+        if not ctx:
+            return
+        if not isinstance(ctx.extra, dict):
+            ctx.extra = {}
+        val = (handle or "").strip()
+        if val:
+            ctx.extra["rt_session_id"] = val
+            if window:
+                try:
+                    window.core.ctx.update_item(ctx)
+                except Exception:
+                    pass
+    except Exception:
+        pass
+
+def set_rt_session_expires_at(ctx: Optional[CtxItem], epoch_seconds: Optional[int], window=None):
+    """Persist optional session expiration timestamp into ctx.extra."""
+    if not ctx or epoch_seconds is None:
+        return
+    try:
+        if not isinstance(ctx.extra, dict):
+            ctx.extra = {}
+        ctx.extra["rt_session_expires_at"] = int(epoch_seconds)
+        if window:
+            try:
+                window.core.ctx.update_item(ctx)
+            except Exception:
+                pass
+    except Exception:
+        pass
+
+def extract_last_session_id(items: list[CtxItem]) -> Optional[str]:
+    """Extract last known session ID from a list of CtxItems."""
+    if not items:
+        return None
+    for item in reversed(items):
+        if not item or not isinstance(item.extra, dict):
+            continue
+        val = item.extra.get("rt_session_id")
+        if isinstance(val, str) and val.strip():
+            return val.strip()
+    return None

pygpt_net/core/realtime/shared/text.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.31 23:00:00                  #
+# ================================================== #
+
+import re
+
+def coalesce_text(parts):
+    """Merge text parts, preserving intentional newlines and fixing spaces."""
+    if not parts:
+        return ""
+    out = []
+    for piece in parts:
+        if not piece:
+            continue
+        s = str(piece)
+        s = re.sub(r"[ \t\f\v]+", " ", s)
+        s = re.sub(r"[ \t]*\n[ \t]*", "\n", s)
+        if not out:
+            out.append(s.strip())
+            continue
+        if out[-1].endswith("\n") or s.startswith("\n"):
+            out.append(s.lstrip())
+        else:
+            out.append(" " + s.strip())
+    text = "".join(out)
+    text = re.sub(r"[ \t]+([,.;:!?%])", r"\1", text)
+    text = re.sub(r"[ \t]+([\)\]\}])", r"\1", text)
+    text = re.sub(r"[ \t]+(['\"])", r"\1", text)
+    text = re.sub(r"\n{3,}", "\n\n", text)
+    return text.strip()

pygpt_net/core/realtime/shared/tools.py

@@ -0,0 +1,276 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.31 23:00:00                  #
+# ================================================== #
+
+import json
+from typing import Any, Optional, List, Dict
+
+def sanitize_function_tools(tools) -> list:
+    """
+    OpenAI: Normalize function tools into a flat dict shape:
+      {"type":"function","name","description","parameters", ...}
+    Accepts legacy {"type":"function","function":{...}} and flattens it.
+    """
+    out = []
+    if not tools:
+        return out
+    for t in tools:
+        if not isinstance(t, dict):
+            continue
+        tt = dict(t)
+        ttype = (tt.get("type") or "function").lower()
+        if ttype != "function":
+            continue
+        if isinstance(tt.get("function"), dict):
+            fn = tt["function"]
+            nt = {"type": "function"}
+            for k in ("name", "description", "parameters", "strict", "strict_schema"):
+                if k in fn and fn[k] is not None:
+                    nt[k] = fn[k]
+            if "description" not in nt and tt.get("description"):
+                nt["description"] = tt["description"]
+        else:
+            nt = {
+                "type": "function",
+                "name": tt.get("name"),
+                "description": tt.get("description"),
+                "parameters": tt.get("parameters"),
+            }
+            for k in ("strict", "strict_schema"):
+                if k in tt:
+                    nt[k] = tt[k]
+        if not nt.get("name"):
+            continue
+        if not isinstance(nt.get("parameters"), dict):
+            nt["parameters"] = {"type": "object", "properties": {}}
+        out.append(nt)
+    return out
+
+def sanitize_remote_tools(remote_tools) -> list:
+    """OpenAI: Pass-through for non-function tools (ensure lowercased 'type')."""
+    allowed = {"function", "mcp"}  # Realtime accepts only these
+    out = []
+    if not remote_tools:
+        return out
+    for t in remote_tools:
+        if not isinstance(t, dict):
+            continue
+        tt = dict(t)
+        ttype = tt.get("type")
+        if not ttype:
+            continue
+        if allowed is not None and ttype not in allowed:
+            continue
+        tt["type"] = str(ttype).lower()
+        out.append(tt)
+    return out
+
+def tools_signature(tools_list: list) -> str:
+    """Order-insensitive stable signature for tools list."""
+    def canon(obj):
+        if isinstance(obj, dict):
+            return {k: canon(v) for k, v in sorted(obj.items())}
+        if isinstance(obj, list):
+            return [canon(x) for x in obj]
+        return obj
+    try:
+        canon_items = [json.dumps(canon(t), ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+                       for t in (tools_list or [])]
+        canon_items.sort()
+        return "|".join(canon_items)
+    except Exception:
+        return str(tools_list)
+
+def prepare_tools_for_session(opts) -> list:
+    """Compose session.tools from opts.remote_tools + opts.tools."""
+    fn = sanitize_function_tools(getattr(opts, "tools", None))
+    rt = sanitize_remote_tools(getattr(opts, "remote_tools", None))
+    return (rt or []) + (fn or [])
+
+def prepare_tools_for_response(opts) -> tuple[list, Optional[str]]:
+    """Compose per-response function tools and tool_choice."""
+    fn = sanitize_function_tools(getattr(opts, "tools", None))
+    tool_choice = getattr(opts, "tool_choice", None)
+    return fn, tool_choice
+
+def build_tool_outputs_payload(results, last_tool_calls: List[Dict]) -> List[Dict]:
+    """
+    Normalize 'results' into:
+      [{"call_id": str, "previous_item_id": str|None, "output": str}]
+    Matching priority: call_id -> item.id -> function name -> first unused.
+    """
+    calls = list(last_tool_calls or [])
+    by_id = {c.get("id") or "": c for c in calls if c.get("id")}
+    by_call = {c.get("call_id") or "": c for c in calls if c.get("call_id")}
+    by_name: dict[str, list] = {}
+    for c in calls:
+        nm = ((c.get("function") or {}).get("name") or "").strip()
+        if nm:
+            by_name.setdefault(nm, []).append(c)
+
+    used: set[str] = set()
+    def to_str(val) -> str:
+        if val is None:
+            return ""
+        if isinstance(val, (dict, list)):
+            try:
+                return json.dumps(val, ensure_ascii=False)
+            except Exception:
+                return str(val)
+        return str(val)
+
+    def pick_name(name: str):
+        arr = by_name.get(name) or []
+        for cand in arr:
+            cid = cand.get("call_id") or ""
+            if cid and cid not in used:
+                used.add(cid)
+                return cand
+        return None
+
+    def pick_first():
+        for cand in calls:
+            cid = cand.get("call_id") or ""
+            if cid and cid not in used:
+                used.add(cid)
+                return cand
+        return None
+
+    out: list[dict] = []
+
+    if isinstance(results, dict) and ("function_responses" in results or "tool_outputs" in results):
+        items = results.get("function_responses") or results.get("tool_outputs") or []
+        for it in items:
+            if not isinstance(it, dict):
+                c = pick_first()
+                if c:
+                    out.append({"call_id": c.get("call_id"), "previous_item_id": c.get("id"), "output": to_str(it)})
+                continue
+            cid = it.get("call_id") or it.get("id") or it.get("tool_call_id") or ""
+            nm = it.get("name") or ""
+            resp = it.get("response")
+            if resp is None:
+                resp = it.get("result") or it.get("output") or it.get("content")
+            c = by_call.get(cid) or by_id.get(cid) or (pick_name(nm) if nm else pick_first())
+            if c:
+                out.append({"call_id": c.get("call_id"), "previous_item_id": c.get("id"), "output": to_str(resp)})
+        return out
+
+    if isinstance(results, list):
+        for it in results:
+            if not isinstance(it, dict):
+                c = pick_first()
+                if c:
+                    out.append({"call_id": c.get("call_id"), "previous_item_id": c.get("id"), "output": to_str(it)})
+                continue
+            cid = it.get("call_id") or it.get("id") or it.get("tool_call_id") or ""
+            nm = it.get("name") or ""
+            resp = it.get("response")
+            if resp is None:
+                resp = it.get("result") or it.get("output") or it.get("content")
+            c = by_call.get(cid) or by_id.get(cid) or (pick_name(nm) if nm else pick_first())
+            if c:
+                out.append({"call_id": c.get("call_id"), "previous_item_id": c.get("id"), "output": to_str(resp)})
+        return out
+
+    if isinstance(results, dict):
+        for k, v in results.items():
+            if not isinstance(k, str):
+                continue
+            c = by_call.get(k) or by_id.get(k) or pick_name(k)
+            if c:
+                out.append({"call_id": c.get("call_id"), "previous_item_id": c.get("id"), "output": to_str(v)})
+        return out
+
+    c = pick_first()
+    if c:
+        out.append({"call_id": c.get("call_id"), "previous_item_id": c.get("id"), "output": to_str(results)})
+    return out
+
+def build_function_responses_payload(results, last_tool_calls: List[Dict]) -> List[Dict]:
+    """
+    Produce neutral list of dicts for Google:
+      [{"id": "...", "name": "...", "response": {...}}]
+    Provider converts to gtypes.FunctionResponse downstream.
+    """
+    calls = list(last_tool_calls or [])
+    by_id = {c.get("id") or "": c for c in calls if c.get("id")}
+    by_name: dict[str, list] = {}
+    for c in calls:
+        nm = (c.get("function") or {}).get("name") or ""
+        if nm:
+            by_name.setdefault(nm, []).append(c)
+
+    used_ids: set[str] = set()
+
+    def pick_id_for_name(name: str) -> str:
+        arr = by_name.get(name) or []
+        for cand in arr:
+            cid = cand.get("id") or ""
+            if cid and cid not in used_ids:
+                used_ids.add(cid)
+                return cid
+        return ""
+
+    def to_resp_dict(val):
+        if isinstance(val, dict):
+            return val
+        return {"result": str(val)}
+
+    out: list = []
+
+    if isinstance(results, dict) and "function_responses" in results:
+        items = results.get("function_responses") or []
+        for it in items:
+            fid = it.get("id") or ""
+            nm = it.get("name") or ""
+            resp = it.get("response")
+            if resp is None:
+                resp = it.get("result") or it.get("output") or it.get("content") or {}
+            out.append({"id": fid, "name": nm, "response": to_resp_dict(resp)})
+        return out
+
+    if isinstance(results, list):
+        for it in results:
+            if not isinstance(it, dict):
+                if calls:
+                    ref = calls[0]
+                    cid = ref.get("id") or ""
+                    nm = (ref.get("function") or {}).get("name") or ""
+                    used_ids.add(cid)
+                    out.append({"id": cid, "name": nm, "response": to_resp_dict(it)})
+                continue
+            fid = it.get("id") or it.get("call_id") or it.get("tool_call_id") or ""
+            nm = it.get("name") or ""
+            resp = it.get("response")
+            if resp is None:
+                resp = it.get("result") or it.get("output") or it.get("content") or {}
+            if not fid and nm:
+                fid = pick_id_for_name(nm)
+            if fid:
+                used_ids.add(fid)
+            out.append({"id": fid, "name": nm, "response": to_resp_dict(resp)})
+        return out
+
+    if isinstance(results, dict):
+        for k, v in results.items():
+            if not isinstance(k, str):
+                continue
+            if k in by_id:
+                nm = (by_id[k].get("function") or {}).get("name") or ""
+                used_ids.add(k)
+                out.append({"id": k, "name": nm, "response": to_resp_dict(v)})
+            else:
+                nm = k
+                fid = pick_id_for_name(nm)
+                out.append({"id": fid, "name": nm, "response": to_resp_dict(v)})
+        return out
+
+    return out

pygpt_net/core/realtime/shared/turn.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.31 23:00:00                  #
+# ================================================== #
+
+from enum import Enum
+
+class TurnMode(str, Enum):
+    MANUAL = "manual"
+    AUTO = "auto"  # future (server VAD / automatic activity detection)
+
+def apply_turn_mode_openai(session_payload: dict, mode: TurnMode):
+    """
+    Mutate OpenAI session.update payload to reflect turn mode.
+    Manual: turn_detection=None (default).
+    Auto: enable server VAD if available.
+    """
+    sess = session_payload.setdefault("session", {})
+    if mode == TurnMode.AUTO:
+        sess["turn_detection"] = {"type": "server_vad"}
+    else:
+        sess["turn_detection"] = None
+
+def apply_turn_mode_google(live_cfg: dict, mode: TurnMode):
+    """
+    Mutate Google Live connect config to reflect turn mode.
+    Manual: automatic_activity_detection.disabled=True
+    Auto: disabled=False (server handles VAD).
+    """
+    ri = live_cfg.setdefault("realtime_input_config", {})
+    aad = ri.setdefault("automatic_activity_detection", {})
+    aad["disabled"] = (mode != TurnMode.AUTO)

pygpt_net/core/realtime/shared/types.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.31 23:00:00                  #
+# ================================================== #
+
+from typing import Optional, Callable, Awaitable
+
+TextCallback = Callable[[str], Awaitable[None]]
+AudioCallback = Callable[[bytes, str, Optional[int], Optional[int], bool], Awaitable[None]]
+StopCallback = Callable[[], bool]

pygpt_net/core/realtime/worker.py

@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.30 06:00:00                  #
+# ================================================== #
+
+import asyncio
+from typing import Optional
+
+from PySide6.QtCore import Slot, QRunnable, QObject, Signal
+
+from pygpt_net.core.events import RealtimeEvent
+from pygpt_net.item.ctx import CtxItem
+
+from .options import RealtimeOptions
+
+class RealtimeSignals(QObject):
+    """Realtime signals"""
+    response = Signal(object)  # RealtimeEvent
+
+class RealtimeWorker(QRunnable):
+    """
+    QRunnable worker that runs a provider-specific realtime session (websocket).
+
+    - RT_OUTPUT_READY is emitted when the audio output is ready (STREAM_BEGIN).
+    - RT_OUTPUT_TEXT_DELTA is emitted for text deltas.
+    - RT_OUTPUT_AUDIO_DELTA is emitted for audio chunks to be handled by the main-thread AudioDispatcher.
+    - RT_OUTPUT_AUDIO_END is emitted when the session ends.
+    - RT_OUTPUT_AUDIO_ERROR is emitted on error.
+    """
+    def __init__(
+            self,
+            window,
+            ctx: CtxItem,
+            opts: RealtimeOptions
+    ):
+        """
+        Initialize the worker.
+
+        :param window: Window instance
+        :param ctx: CtxItem
+        :param opts: RealtimeOptions
+        """
+        super().__init__()
+        self.window = window
+        self.ctx = ctx
+        self.opts = opts
+
+    def get_client(self, provider: str):
+        """
+        Get the appropriate client based on the provider
+
+        :param provider: Provider name
+        :return: Client instance
+        """
+        provider = (provider or "openai").lower()
+        if provider == "google":
+            return self.window.core.api.google.realtime.handler
+        elif provider == "openai":
+            return self.window.core.api.openai.realtime.handler
+        else:
+            raise RuntimeError(f"Unsupported realtime provider: {provider}")
+
+    @Slot()
+    def run(self):
+        loop = None  # ensure defined for cleanup
+
+        # STREAM_BEGIN -> UI
+        try:
+            event = RealtimeEvent(RealtimeEvent.RT_OUTPUT_READY, {
+                "ctx": self.ctx,
+            })
+            self.opts.rt_signals.response.emit(event) if self.opts.rt_signals else None
+        except Exception:
+            pass
+
+        try:
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+
+            async def _amain():
+                # Text deltas -> UI
+                async def on_text(delta: str):
+                    if not delta:
+                        return
+                    event = RealtimeEvent(RealtimeEvent.RT_OUTPUT_TEXT_DELTA, {
+                        "ctx": self.ctx,
+                        "chunk": delta,
+                    })
+                    self.opts.rt_signals.response.emit(event) if self.opts.rt_signals else None
+
+                # Audio -> enqueue to main-thread
+                async def on_audio(
+                        data: bytes,
+                        mime: str,
+                        rate: Optional[int],
+                        channels: Optional[int],
+                        final: bool = False
+                ):
+                    event = RealtimeEvent(RealtimeEvent.RT_OUTPUT_AUDIO_DELTA, {
+                        "payload":  {
+                            "ctx": self.ctx,
+                            "data": data or b"",
+                            "mime": mime or "audio/pcm",
+                            "rate": int(rate) if rate is not None else None,
+                            "channels": int(channels) if channels is not None else None,
+                            "final": bool(final),
+                            "provider": self.opts.provider,
+                            "model": self.opts.model,
+                        }
+                    })
+                    self.opts.rt_signals.response.emit(event) if self.opts.rt_signals else None
+
+                def _should_stop() -> bool:
+                    try:
+                        return bool(self.window.controller.kernel.stopped())
+                    except Exception:
+                        return False
+
+                # run the client
+                client = self.get_client(self.opts.provider)
+                await client.run(self.ctx, self.opts, on_text, on_audio, _should_stop)
+
+            loop.run_until_complete(_amain())
+            # print("[rt] STREAM_END")
+
+        except Exception as e:
+            try:
+                event = RealtimeEvent(RealtimeEvent.RT_OUTPUT_AUDIO_ERROR, {"error": e})
+                self.opts.rt_signals.response.emit(event) if self.opts.rt_signals else None
+            finally:
+                try:
+                    event = RealtimeEvent(RealtimeEvent.RT_OUTPUT_AUDIO_END, {"ctx": self.ctx})
+                    self.opts.rt_signals.response.emit(event) if self.opts.rt_signals else None
+                except Exception:
+                    pass
+        finally:
+            # Robust asyncio teardown to avoid hangs on subsequent runs
+            if loop is not None:
+                try:
+                    pending = [t for t in asyncio.all_tasks(loop) if not t.done()]
+                    for t in pending:
+                        t.cancel()
+                    if pending:
+                        loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True))
+                except Exception:
+                    pass
+                try:
+                    loop.run_until_complete(loop.shutdown_asyncgens())
+                except Exception:
+                    pass
+                try:
+                    loop.close()
+                except Exception:
+                    pass
+                try:
+                    asyncio.set_event_loop(None)
+                except Exception:
+                    pass

pygpt_net/core/tokens/tokens.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2025.08.15 23:00:00                  #
+# Updated Date: 2025.08.28 09:00:00                  #
 # ================================================== #
 
 from typing import Tuple, List
@@ -36,8 +36,8 @@ from pygpt_net.item.ctx import CtxItem
 
 CHAT_MODES = [
     MODE_CHAT,
-    MODE_VISION,
-    MODE_LANGCHAIN,
+    # MODE_VISION,
+    # MODE_LANGCHAIN,
     MODE_ASSISTANT,
     MODE_LLAMA_INDEX,
     MODE_AGENT,
@@ -328,7 +328,7 @@ class Tokens:
         model_id = self.window.core.models.get_id(model)
         mode = self.window.core.config.get('mode')
         tokens = 0
-        if mode in [MODE_CHAT, MODE_VISION, MODE_AUDIO, MODE_RESEARCH]:
+        if mode in [MODE_CHAT, MODE_AUDIO, MODE_RESEARCH]:
             tokens += self.from_prompt(system_prompt, "", model_id)
             tokens += self.from_text("system", model_id)
             tokens += self.from_prompt(input_prompt, "", model_id)

pygpt_net/core/types/__init__.py

@@ -11,6 +11,7 @@
 
 from .agent import *
 from .base import *
+from .image import *
 from .mode import *
 from .model import *
 from .openai import *

pygpt_net/core/types/image.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.07.13 01:00:00                  #
+# ================================================== #
+
+IMAGE_AVAILABLE_RESOLUTIONS = {
+    "gpt-image": {
+        "auto": "auto",
+        "1024x1024": "1024x1024",
+        "1536x1024": "1536x1024",
+        "1024x1536": "1024x1536"
+    },
+    "dall-e-3": {
+        "1792x1024": "1792x1024",
+        "1024x1792": "1024x1792",
+        "1024x1024": "1024x1024"
+    },
+    "dall-e-2": {
+        "1024x1024": "1024x1024",
+        "512x512": "512x512",
+        "256x256": "256x256"
+    },
+    "imagen-3.0": {
+        "1024x1024": "1024x1024",
+        "896x1280": "896x1280",
+        "1280x896": "1280x896",
+        "768x1408": "768x1408",
+        "1408x768": "1408x768"
+    },
+    "imagen-4.0": {
+        "1024x1024": "1024x1024",
+        "896x1280": "896x1280",
+        "1280x896": "1280x896",
+        "768x1408": "768x1408",
+        "1408x768": "1408x768",
+        "2048x2048": "2048x2048",
+        "1792x2560": "1792x2560",
+        "2560x1792": "2560x1792",
+        "1536x2816": "1536x2816",
+        "2816x1536": "2816x1536"
+    }
+}