pygpt-net 2.4.28__py3-none-any.whl → 2.4.34__py3-none-any.whl

pygpt_net/provider/gpt/__init__.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.21 20:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 from httpx_socks import SyncProxyTransport
@@ -15,6 +15,7 @@ from openai import OpenAI, DefaultHttpxClient
 
 from pygpt_net.core.types import (
     MODE_ASSISTANT,
+    MODE_AUDIO,
     MODE_CHAT,
     MODE_COMPLETION,
     MODE_IMAGE,
@@ -22,6 +23,7 @@ from pygpt_net.core.types import (
 )
 from pygpt_net.core.bridge.context import BridgeContext
 
+from .audio import Audio
 from .assistants import Assistants
 from .chat import Chat
 from .completion import Completion
@@ -40,6 +42,7 @@ class Gpt:
         """
         self.window = window
         self.assistants = Assistants(window)
+        self.audio = Audio(window)
         self.chat = Chat(window)
         self.completion = Completion(window)
         self.image = Image(window)
@@ -114,7 +117,7 @@ class Gpt:
             used_tokens = self.completion.get_used_tokens()
 
         # chat
-        elif mode == MODE_CHAT:
+        elif mode in [MODE_CHAT, MODE_AUDIO]:
             response = self.chat.send(
                 context=context,
                 extra=extra,
@@ -187,6 +190,15 @@ class Gpt:
                     ctx.tool_calls = self.window.core.command.unpack_tool_calls(
                         response.choices[0].message.tool_calls,
                     )
+        # audio
+        elif mode in [MODE_AUDIO]:
+            if response.choices[0]:
+                if response.choices[0].message and response.choices[0].message.audio:
+                    ctx.audio_output = response.choices[0].message.audio.data
+                    ctx.audio_id = response.choices[0].message.audio.id
+                    ctx.audio_expires_ts = response.choices[0].message.audio.expires_at
+                    ctx.is_audio = True
+                    output = response.choices[0].message.audio.transcript  # from transcript
 
         ctx.set_output(output, ai_name)
         ctx.set_tokens(

pygpt_net/provider/gpt/audio.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2024.11.26 19:00:00                  #
+# ================================================== #
+
+import base64
+
+from pygpt_net.core.bridge.context import MultimodalContext
+
+
+class Audio:
+    def __init__(self, window=None):
+        """
+        Audio input wrapper
+
+        :param window: Window instance
+        """
+        self.window = window
+
+    def build_content(
+            self,
+            content,
+            multimodal_ctx: MultimodalContext = None,
+    ) -> list:
+        """
+        Build audio content
+
+        :param content: previous content or input prompt
+        :param multimodal_ctx: multimodal context
+        :return: List of contents
+        """
+        if not isinstance(content, list):
+            if content:
+                content = [
+                    {
+                        "type": "text",
+                        "text": str(content),
+                    }
+                ]
+            else:
+                content = []  # if empty input return empty list
+
+        # abort if no audio input provided
+        if not multimodal_ctx.is_audio_input:
+            return content
+
+        encoded = base64.b64encode(multimodal_ctx.audio_data).decode('utf-8')
+        audio_format = multimodal_ctx.audio_format  # wav by default
+        audio_data = {
+            "type": "input_audio",
+            "input_audio": {
+                "data": encoded,
+                "format": audio_format,
+            }
+        }
+        content.append(audio_data)
+        return content

pygpt_net/provider/gpt/chat.py

@@ -6,20 +6,22 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.23 00:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 import json
-import re
+import time
 
 from pygpt_net.core.types import (
     MODE_CHAT,
     MODE_VISION,
+    MODE_AUDIO,
 )
-from pygpt_net.core.bridge.context import BridgeContext
+from pygpt_net.core.bridge.context import BridgeContext, MultimodalContext
 from pygpt_net.item.ctx import CtxItem
 from pygpt_net.item.model import ModelItem
 
+from .utils import sanitize_name
 
 class Chat:
     def __init__(self, window=None):
@@ -31,7 +33,11 @@ class Chat:
         self.window = window
         self.input_tokens = 0
 
-    def send(self, context: BridgeContext, extra: dict = None):
+    def send(
+            self,
+            context: BridgeContext,
+            extra: dict = None
+    ):
         """
         Call OpenAI API for chat
 
@@ -43,9 +49,11 @@ class Chat:
         stream = context.stream
         max_tokens = int(context.max_tokens or 0)
         system_prompt = context.system_prompt
+        mode = context.mode
         model = context.model
         functions = context.external_functions
         attachments = context.attachments
+        multimodal_ctx = context.multimodal_ctx
 
         ctx = context.ctx
         if ctx is None:
@@ -64,6 +72,7 @@ class Chat:
             attachments=attachments,
             ai_name=ai_name,
             user_name=user_name,
+            multimodal_ctx=multimodal_ctx,
         )
         msg_tokens = self.window.core.tokens.from_messages(
             messages,
@@ -79,25 +88,24 @@ class Chat:
         # extra API kwargs
         response_kwargs = {}
 
-        # tools / functions
+        # tools / functions >>>>>>>>>> TMP DISABLED FOR AUDIO <<<<<<<<<<
         tools = []
-        if functions is not None and isinstance(functions, list):
-            for function in functions:
-                if str(function['name']).strip() == '' or function['name'] is None:
-                    continue
-                params = {}
-                if function['params'] is not None and function['params'] != "":
-                    params = json.loads(function['params'])  # unpack JSON from string
-                tools.append(
-                    {
+        if mode not in [MODE_AUDIO]:
+            if functions is not None and isinstance(functions, list):
+                for function in functions:
+                    if str(function['name']).strip() == '' or function['name'] is None:
+                        continue
+                    params = {}
+                    if function['params'] is not None and function['params'] != "":
+                        params = json.loads(function['params'])  # unpack JSON from string
+                    tools.append({
                         "type": "function",
                         "function": {
                             "name": function['name'],
                             "parameters": params,
                             "description": function['desc'],
                         }
-                    }
-                )
+                    })
 
         # fix: o1 compatibility
         if model.id is not None and not model.id.startswith("o1"):
@@ -117,11 +125,24 @@ class Chat:
         if model.id is not None and model.id.startswith("o1"):
             stream = False
 
+        # audio mode
+        if mode in [MODE_AUDIO]:
+            stream = False
+            voice_id = "alloy"
+            tmp_voice = self.window.core.plugins.get_option("audio_output", "openai_voice")
+            if tmp_voice:
+                voice_id = tmp_voice
+            response_kwargs["modalities"] = ["text", "audio"]
+            response_kwargs["audio"] = {
+                "voice": voice_id,
+                "format": "wav"
+            }
+
         response = client.chat.completions.create(
             messages=messages,
             model=model.id,
             stream=stream,
-            **response_kwargs
+            **response_kwargs,
         )
         return response
 
@@ -133,7 +154,8 @@ class Chat:
             history: list = None,
             attachments: dict = None,
             ai_name: str = None,
-            user_name: str = None
+            user_name: str = None,
+            multimodal_ctx: MultimodalContext = None,
     ) -> list:
         """
         Build list of chat messages
@@ -145,6 +167,7 @@ class Chat:
         :param attachments: attachments
         :param ai_name: AI name
         :param user_name: username
+        :param multimodal_ctx: Multimodal context
         :return: messages list
         """
         messages = []
@@ -190,26 +213,49 @@ class Chat:
             )
             for item in items:
                 # input
-                role_name = "system"
-                if not allowed_system:
-                    role_name = "user"
                 if item.final_input is not None and item.final_input != "":
-                    messages.append({"role": role_name, "name": self.sanitize_name(user_name), "content": item.final_input})
+                    messages.append({
+                        "role": "user",
+                        "name": sanitize_name(user_name),
+                        "content": item.final_input,
+                    })
 
                 # output
-                role_name = "system"
-                if not allowed_system:
-                    role_name = "assistant"
                 if item.final_output is not None and item.final_output != "":
-                    messages.append({"role": role_name, "name": self.sanitize_name(ai_name), "content": item.final_output})
-
-        # use vision if available in current model
+                    msg = {
+                        "role": "assistant",
+                        "name": sanitize_name(ai_name),
+                        "content": item.final_output,
+                    }
+                    # append previous audio ID
+                    if MODE_AUDIO in model.mode and item.audio_id:
+                        # at first check expires_at - expired audio throws error in API
+                        current_timestamp = time.time()
+                        audio_timestamp = int(item.audio_expires_ts) if item.audio_expires_ts else 0
+                        if audio_timestamp and audio_timestamp > current_timestamp:
+                            msg["audio"] = {
+                                "id": item.audio_id
+                            }
+                    messages.append(msg)
+
+        # use vision and audio if available in current model
         content = str(prompt)
         if MODE_VISION in model.mode:
-            content = self.window.core.gpt.vision.build_content(prompt, attachments)
+            content = self.window.core.gpt.vision.build_content(
+                content=content,
+                attachments=attachments,
+            )
+        if MODE_AUDIO in model.mode:
+            content = self.window.core.gpt.audio.build_content(
+                content=content,
+                multimodal_ctx=multimodal_ctx,
+            )
 
         # append current prompt
-        messages.append({"role": "user", "content": content})
+        messages.append({
+            "role": "user",
+            "content": content,
+        })
 
         # input tokens: update
         self.input_tokens += self.window.core.tokens.from_messages(
@@ -218,20 +264,6 @@ class Chat:
         )
         return messages
 
-    def sanitize_name(self, name: str) -> str:
-        """
-        Sanitize name
-
-        :param name: name
-        :return: sanitized name
-        """
-        if name is None:
-            return ""
-        # allowed characters: a-z, A-Z, 0-9, _, and -
-        name = name.strip().lower()
-        sanitized_name = re.sub(r'[^a-z0-9_-]', '_', name)
-        return sanitized_name[:64]  # limit to 64 characters
-
     def reset_tokens(self):
         """Reset input tokens counter"""
         self.input_tokens = 0

pygpt_net/provider/gpt/utils.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2024.11.26 19:00:00                  #
+# ================================================== #
+
+import re
+
+
+def sanitize_name(name: str) -> str:
+    """
+    Sanitize name
+
+    :param name: name
+    :return: sanitized name
+    """
+    if name is None:
+        return ""
+    # allowed characters: a-z, A-Z, 0-9, _, and -
+    name = name.strip().lower()
+    sanitized_name = re.sub(r'[^a-z0-9_-]', '_', name)
+    return sanitized_name[:64]  # limit to 64 characters

pygpt_net/provider/gpt/vision.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.23 00:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 import base64
@@ -109,10 +109,16 @@ class Vision:
 
         # append initial (system) message
         if system_prompt is not None and system_prompt != "":
-            messages.append({"role": "system", "content": system_prompt})
+            messages.append({
+                "role": "system",
+                "content": system_prompt,
+            })
         else:
             if system_prompt is not None and system_prompt != "":
-                messages.append({"role": "system", "content": system_prompt})
+                messages.append({
+                    "role": "system",
+                    "content": system_prompt,
+                })
 
         # append messages from context (memory)
         if self.window.core.config.get('use_context'):
@@ -126,15 +132,24 @@ class Vision:
             for item in items:
                 # input
                 if item.final_input is not None and item.final_input != "":
-                    messages.append({"role": "user", "content": item.final_input})
+                    messages.append({
+                        "role": "user",
+                        "content": item.final_input,
+                    })
 
                 # output
                 if item.final_output is not None and item.final_output != "":
-                    messages.append({"role": "assistant", "content": item.final_output})
+                    messages.append({
+                        "role": "assistant",
+                        "content": item.final_output,
+                    })
 
         # append current prompt
         content = self.build_content(prompt, attachments)
-        messages.append({"role": "user", "content": content})
+        messages.append({
+            "role": "user",
+            "content": content,
+        })
 
         # input tokens: update
         self.input_tokens += self.window.core.tokens.from_messages(
@@ -143,20 +158,27 @@ class Vision:
         )
         return messages
 
-    def build_content(self, prompt: str, attachments: dict = None) -> list:
+    def build_content(
+            self,
+            content: str or list,
+            attachments: dict = None,
+    ) -> list:
         """
-        Build vision contents
+        Build vision content
 
-        :param prompt: prompt (user input)
+        :param content: content (str or list)
         :param attachments: attachments (dict, optional)
         :return: List of contents
         """
-        content = [
-            {
-                "type": "text",
-                "text": str(prompt)
-            }
-        ]
+        if not isinstance(content, list):
+            content = [
+                {
+                    "type": "text",
+                    "text": str(content)
+                }
+            ]
+
+        prompt = content[0]['text']
 
         self.attachments = {}  # reset attachments, only current prompt
         self.urls = []

pygpt_net/provider/loaders/base.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.04.17 01:00:00                  #
+# Updated Date: 2024.11.26 04:00:00                  #
 # ================================================== #
 
 from llama_index.core.readers.base import BaseReader
@@ -75,6 +75,15 @@ class BaseLoader:
             return args.get("url")
         return ""
 
+    def is_supported_attachment(self, source: str) -> bool:
+        """
+        Check if attachment is supported by loader
+
+        :param source: attachment source
+        :return: True if supported
+        """
+        return False
+
     def get(self) -> BaseReader:
         """
         Get reader instance

pygpt_net/provider/loaders/web_yt.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.03.12 06:00:00                  #
+# Updated Date: 2024.11.26 04:00:00                  #
 # ================================================== #
 
 from llama_index.core.readers.base import BaseReader
@@ -52,3 +52,21 @@ class Loader(BaseLoader):
         args = {}
         args["ytlinks"] = [kwargs.get("url")]  # list of links
         return args
+
+    def is_supported_attachment(self, source: str) -> bool:
+        """
+        Check if attachment is supported by loader
+
+        :param source: attachment source
+        :return: True if supported
+        """
+        yt_prefix = [
+            "https://youtube.com",
+            "https://youtu.be",
+            "https://www.youtube.com",
+            "https://m.youtube.com",
+        ]
+        for prefix in yt_prefix:
+            if source.startswith(prefix):
+                return True
+        return False

pygpt_net/tools/code_interpreter/__init__.py

@@ -233,6 +233,7 @@ class CodeInterpreter(BaseTool):
 
     def restart_kernel(self):
         """Restart kernel"""
+        print("Sending restart event...")
         commands = [
             {
                 "cmd": "ipython_kernel_restart",

pygpt_net/tools/image_viewer/ui/dialogs.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.03.26 15:00:00                  #
+# Updated Date: 2024.11.26 02:00:00                  #
 # ================================================== #
 
 from PySide6.QtCore import Qt
@@ -17,6 +17,8 @@ from pygpt_net.ui.widget.dialog.base import BaseDialog
 from pygpt_net.ui.widget.image.display import ImageLabel
 from pygpt_net.utils import trans
 
+import pygpt_net.icons_rc
+
 class DialogSpawner:
     def __init__(self, window=None):
         """

pygpt_net/ui/dialog/preset.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.21 20:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 from PySide6.QtCore import Qt
@@ -15,6 +15,7 @@ from PySide6.QtWidgets import QPushButton, QHBoxLayout, QLabel, QVBoxLayout, QSp
 from pygpt_net.core.types import (
     MODE_AGENT,
     MODE_AGENT_LLAMA,
+    MODE_AUDIO,
     MODE_CHAT,
     MODE_COMPLETION,
     MODE_EXPERT,
@@ -108,6 +109,7 @@ class Preset(BaseConfigDialog):
             MODE_LLAMA_INDEX,
             MODE_AGENT_LLAMA,
             MODE_AGENT,
+            MODE_AUDIO,
             MODE_EXPERT,
         ]
         rows_mode = QVBoxLayout()

pygpt_net/ui/dialog/url.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2024.11.26 02:00:00                  #
+# ================================================== #
+
+from pygpt_net.ui.widget.dialog.url import UrlDialog
+from pygpt_net.utils import trans
+
+
+class Url:
+    def __init__(self, window=None):
+        """
+        URL dialog
+
+        :param window: Window instance
+        """
+        self.window = window
+
+    def setup(self):
+        """Setup URL dialog"""
+        id = 'url'
+        self.window.ui.dialog[id] = UrlDialog(self.window, id)
+        self.window.ui.dialog[id].setWindowTitle(trans("dialog.url.title"))

pygpt_net/ui/dialogs.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.14 01:00:00                  #
+# Updated Date: 2024.11.26 02:00:00                  #
 # ================================================== #
 
 import threading
@@ -35,6 +35,7 @@ from pygpt_net.ui.dialog.settings import Settings
 from pygpt_net.ui.dialog.snap import Snap
 from pygpt_net.ui.dialog.start import Start
 from pygpt_net.ui.dialog.update import Update
+from pygpt_net.ui.dialog.url import Url
 from pygpt_net.ui.dialog.workdir import Workdir
 from pygpt_net.ui.widget.dialog.alert import AlertDialog
 from pygpt_net.ui.widget.dialog.confirm import ConfirmDialog
@@ -66,6 +67,7 @@ class Dialogs:
         self.snap = Snap(self.window)
         self.start = Start(self.window)
         self.update = Update(self.window)
+        self.url = Url(self.window)
         self.workdir = Workdir(self.window)
 
     def setup(self):
@@ -85,6 +87,8 @@ class Dialogs:
         self.snap.setup()
         self.start.setup()
         self.update.setup()
+        self.url.setup()
+
         self.image.setup()
         self.license.setup()
         self.logger.setup()