pygpt-net 2.4.39__py3-none-any.whl → 2.4.40__py3-none-any.whl

pygpt_net/plugin/cmd_web/websearch.py

@@ -6,10 +6,11 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.21 02:00:00                  #
+# Updated Date: 2024.12.13 08:00:00                  #
 # ================================================== #
 
 import re
+
 from bs4 import BeautifulSoup
 
 from pygpt_net.core.events import KernelEvent
@@ -194,19 +195,22 @@ class WebSearch:
             query: str,
             page_no: int = 1,
             summarize_prompt: str = ""
-    ) -> (str, int, int, str):
+    ) -> (str, int, int, str, str):
         """
         Get result from search query
 
         :param query: query to search
         :param page_no: page number
         :param summarize_prompt: custom prompt
-        :return: result, total_found, current, url
+        :return: result, total_found, current, url, thumb image
         """
         self.log("Using web query: " + query)
         urls = self.get_urls(query)
 
         # get options
+        is_summary = True
+        if self.plugin.get_option_value("raw"):
+            is_summary = False
         max_per_page = int(self.plugin.get_option_value("max_page_content_length"))
         chunk_size = int(self.plugin.get_option_value("chunk_size"))
         max_result_size = int(self.plugin.get_option_value("max_result_length"))
@@ -216,6 +220,7 @@ class WebSearch:
         i = 1
         current = 1
         url = ""
+        img = None
         for url in urls:
             if url is None or url == "":
                 continue
@@ -236,19 +241,27 @@ class WebSearch:
             if 0 < max_per_page < len(content):
                 content = content[:max_per_page]
 
-            chunks = self.to_chunks(content, chunk_size)  # it returns list of chunks
-            self.debug(
-                "Plugin: cmd_web: URL: {}".format(url)
-            )
-            result = self.get_summary(
-                chunks,
-                str(query),
-                summarize_prompt,
-            )
+            # get summary
+            if is_summary:
+                chunks = self.to_chunks(content, chunk_size)  # it returns list of chunks
+                self.debug(
+                    "Plugin: cmd_web: URL: {}".format(url)
+                )
+                result = self.get_summary(
+                    chunks,
+                    str(query),
+                    summarize_prompt,
+                )
+            else:
+                # no summary
+                result = str(content)
+
             # if result then stop
             if result is not None and result != "":
+                # get thumbnail image
+                if self.plugin.get_option_value("img_thumbnail"):
+                    img = self.plugin.window.core.web.get_main_image(url)
                 self.log("Summary generated (chars: {})".format(len(result)))
-
                 # index webpage if auto-index is enabled
                 self.index_url(url)
                 break
@@ -262,7 +275,7 @@ class WebSearch:
                 "Plugin: cmd_web: summary length: {}".format(len(result))
             )
 
-        if len(result) > max_result_size:
+        if 0 < max_result_size < len(result):
             result = result[:max_result_size]
 
         self.debug(
@@ -273,25 +286,33 @@ class WebSearch:
             result, \
             total_found, \
             current, \
-            url
+            url, \
+            img
 
-    def open_url(self, url: str, summarize_prompt: str = "") -> (str, str):
+    def open_url(self, url: str, summarize_prompt: str = "") -> (str, str, str):
         """
         Get result from specified URL
 
         :param url: URL to visit
         :param summarize_prompt: custom prompt
-        :return: result, url
+        :return: result, url, thumb image
         """
         self.log("Using URL: " + url)
-
         # get options
+        is_summary = True
+        if self.plugin.get_option_value("raw"):
+            is_summary = False
         max_per_page = int(self.plugin.get_option_value("max_page_content_length"))
         chunk_size = int(self.plugin.get_option_value("chunk_size"))
         max_result_size = int(self.plugin.get_option_value("max_result_length"))
 
+        img = None
         self.log("URL: " + url)
         content = self.query_url(url)
+
+        if content is None:
+            return None, url, img
+
         self.log("Content found (chars: {}). Please wait...".format(len(content)))
 
         # index webpage if auto-index is enabled
@@ -300,14 +321,23 @@ class WebSearch:
 
         if 0 < max_per_page < len(content):
             content = content[:max_per_page]
-        chunks = self.to_chunks(
-            content,
-            chunk_size,
-        )  # it returns list of chunks
 
         self.debug("Plugin: cmd_web: URL: {}".format(url))  # log
 
-        result = self.get_summary(chunks, "", summarize_prompt)
+        # get summary
+        if is_summary:
+            chunks = self.to_chunks(
+                content,
+                chunk_size,
+            )  # it returns list of chunks
+            result = self.get_summary(chunks, "", summarize_prompt)
+        else:
+            # no summary
+            result = str(content)
+
+        # get thumbnail image
+        if self.plugin.get_option_value("img_thumbnail"):
+            img = self.plugin.window.core.web.get_main_image(url)
 
         if result is not None and result != "":
             self.log("Summary generated (chars: {})".format(len(result)))
@@ -318,21 +348,21 @@ class WebSearch:
                 "Plugin: cmd_web: summary length: {}".format(len(result))
             )
 
-        if len(result) > max_result_size:
+        if 0 < max_result_size < len(result):
             result = result[:max_result_size]
 
         self.debug(
             "Plugin: cmd_web: result length: {}".format(len(result))
         )
 
-        return result, url
+        return result, url, img
 
-    def open_url_raw(self, url: str) -> (str, str):
+    def open_url_raw(self, url: str) -> (str, str, str):
         """
         Get raw content from specified URL
 
         :param url: URL to visit
-        :return: result, url
+        :return: result, url, thumb image
         """
         self.log("Using URL: " + url)
 
@@ -343,8 +373,11 @@ class WebSearch:
         result = self.query_url(url)
         self.log("Content found (chars: {}). Please wait...".format(len(result)))
 
+        img = None
         # index webpage if auto-index is enabled
         if result:
+            if self.plugin.get_option_value("img_thumbnail"):
+                img = self.plugin.window.core.web.get_main_image(url)
             self.index_url(url)
 
         # strip if too long
@@ -355,7 +388,7 @@ class WebSearch:
             "Plugin: cmd_web: result length: {}".format(len(result))
         )
 
-        return result, url
+        return result, url, img
 
     def index_url(self, url: str):
         """

pygpt_net/plugin/cmd_web/worker.py

@@ -6,12 +6,13 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.18 21:00:00                  #
+# Updated Date: 2024.12.13 08:00:00                  #
 # ================================================== #
 
 import json
 
 from PySide6.QtCore import Slot
+
 from pygpt_net.plugin.base.worker import BaseWorker, BaseSignals
 
 
@@ -40,7 +41,7 @@ class Worker(BaseWorker):
             response = None
             try:
                 if item["cmd"] == "web_search":
-                    response = self.cmd_web_search(item)
+                    response = self.cmd_web_urls(item)  # return URLs
 
                 elif item["cmd"] == "web_url_open":
                     response = self.cmd_web_url_open(item)
@@ -57,6 +58,12 @@ class Worker(BaseWorker):
                 elif item["cmd"] == "web_index_query":
                     response = self.cmd_web_index_query(item)
 
+                elif item["cmd"] == "web_extract_links":
+                    response = self.cmd_web_extract_links(item)
+
+                elif item["cmd"] == "web_extract_images":
+                    response = self.cmd_web_extract_images(item)
+
                 if response:
                     responses.append(response)
 
@@ -91,7 +98,7 @@ class Worker(BaseWorker):
             prompt = self.get_param(item, "summarize_prompt")
 
         query = self.get_param(item, "query", "")
-        content, total_found, current, url = self.websearch.make_query(
+        content, total_found, current, url, img = self.websearch.make_query(
             query,
             page,
             prompt,
@@ -105,8 +112,12 @@ class Worker(BaseWorker):
             'total_found': total_found,
         }
         if url:
-            self.ctx.urls.append(url)
+            self.ctx.urls_before.append(url)
+        if img:
+            result["thumb_img"] = img
+            self.ctx.images_before.append(img)
 
+        self.ctx.save_reply = True  # leave links in context
         return self.make_response(item, result)
 
     def cmd_web_url_open(self, item: dict) -> dict:
@@ -122,7 +133,7 @@ class Worker(BaseWorker):
         url = self.get_param(item, "url", "")
 
         self.msg = "Opening Web URL: '{}'".format(url)
-        content, url = self.websearch.open_url(
+        content, url, img = self.websearch.open_url(
             url,
             prompt,
         )
@@ -130,9 +141,12 @@ class Worker(BaseWorker):
             'url': url,
             'content': content,
         }
-        context = "From: " + url + ":\n--------------------------------\n" + content
+        context = "From: " + url + ":\n--------------------------------\n" + str(content)
         if url:
-            self.ctx.urls.append(url)
+            self.ctx.urls_before.append(url)
+        if img:
+            result["thumb_img"] = img
+            self.ctx.images_before.append(img)
 
         extra = {
             "context": context,
@@ -148,16 +162,19 @@ class Worker(BaseWorker):
         """
         url = self.get_param(item, "url", "")
         self.msg = "Opening Web URL: '{}'".format(url)
-        content, url = self.websearch.open_url_raw(
+        content, url, img  = self.websearch.open_url_raw(
             url,
         )
         result = {
             'url': url,
             'content': content,
         }
-        context = "From: " + url + ":\n--------------------------------\n" + content
+        context = "From: " + url + ":\n--------------------------------\n" + str(content)
         if url:
-            self.ctx.urls.append(url)
+            self.ctx.urls_before.append(url)
+        if img:
+            result["thumb_img"] = img
+            self.ctx.images_before.append(img)
 
         extra = {
             "context": context,
@@ -201,7 +218,7 @@ class Worker(BaseWorker):
         }
         if urls:
             for url in urls:
-                self.ctx.urls.append(url)
+                self.ctx.urls_before.append(url)
 
         return self.make_response(item, result)
 
@@ -247,7 +264,7 @@ class Worker(BaseWorker):
             'errors': errors,
         }
         if url and (url.startswith("http://") or url.startswith("https://")):
-            self.ctx.urls.append(url)
+            self.ctx.urls_before.append(url)
 
         extra = {
             "url": url,
@@ -327,9 +344,58 @@ class Worker(BaseWorker):
 
         # add URL to context
         if url and (url.startswith("http://") or url.startswith("https://")):
-            self.ctx.urls.append(url)
+            self.ctx.urls_before.append(url)
 
         extra = {
             "context": context,
         }
         return self.make_response(item, result, extra=extra)
+
+    def cmd_web_extract_links(self, item: dict) -> dict:
+        """
+        Web extract links command
+
+        :param item: command item
+        :return: response item
+        """
+        url = ""
+        if self.has_param(item, "url"):
+            url = self.get_param(item, "url")
+        if not url:
+            return self.make_response(item, "No URL provided")
+        links = self.plugin.window.core.web.get_links(url)
+        result = {
+            'links': links,
+        }
+        self.ctx.urls_before.append(url)
+        return self.make_response(item, result)
+
+    def cmd_web_extract_images(self, item: dict) -> dict:
+        """
+        Web extract images command
+
+        :param item: command item
+        :return: response item
+        """
+        download = False
+        url = ""
+        if self.has_param(item, "url"):
+            url = self.get_param(item, "url")
+        if self.has_param(item, "download"):
+            download = bool(self.get_param(item, "download"))
+        if not url:
+            return self.make_response(item, "No URL provided")
+        images = self.plugin.window.core.web.get_images(url)
+        result = {
+            'images': images,
+        }
+        if images and download:
+            for img in images:
+                try:
+                    path = self.plugin.window.core.web.download_image(img)
+                    if path:
+                        self.ctx.images_before.append(path)
+                except Exception as e:
+                    print(e)
+        self.ctx.urls_before.append(url)
+        return self.make_response(item, result)

pygpt_net/provider/core/config/patch.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.12.09 03:00:00                  #
+# Updated Date: 2024.12.13 19:00:00                  #
 # ================================================== #
 
 import copy
@@ -1755,6 +1755,27 @@ class Patch:
                     data["layout.split"] = False
                 updated = True
 
+            # < 2.4.40
+            if old < parse_version("2.4.40"):
+                print("Migrating config from < 2.4.40...")
+                if 'cmd_web' in data['plugins'] \
+                        and 'max_result_length' in data['plugins']['cmd_web']:
+                    del data['plugins']['cmd_web']['max_result_length']
+                if 'cmd_web' in data['plugins'] \
+                        and 'cmd.web_search' in data['plugins']['cmd_web']:
+                    del data['plugins']['cmd_web']['cmd.web_search']
+                if 'cmd_web' in data['plugins'] \
+                        and 'cmd.web_url_open' in data['plugins']['cmd_web']:
+                    del data['plugins']['cmd_web']['cmd.web_url_open']
+                if 'cmd_web' in data['plugins'] \
+                        and 'cmd.web_url_raw' in data['plugins']['cmd_web']:
+                    del data['plugins']['cmd_web']['cmd.web_url_raw']
+                self.window.core.updater.patch_css('web-blocks.css', True)  # force update
+                self.window.core.updater.patch_css('web-blocks.light.css', True)  # force update
+                self.window.core.updater.patch_css('web-chatgpt.css', True)  # force update
+                self.window.core.updater.patch_css('web-chatgpt_wide.css', True)  # force update
+                updated = True
+
         # update file
         migrated = False
         if updated:

pygpt_net/tools/__init__.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.04.19 01:00:00                  #
+# Updated Date: 2024.12.09 23:00:00                  #
 # ================================================== #
 
 from pygpt_net.ui.widget.dialog.base import BaseDialog
@@ -42,6 +42,14 @@ class Tools:
         if id in self.tools:
             return self.tools[id]
 
+    def get_all(self) -> dict:
+        """
+        Get all tools
+
+        :return: dict with tools
+        """
+        return self.tools
+
     def setup(self):
         """Setup tools"""
         self.setup_dialogs()

pygpt_net/tools/base.py

@@ -6,11 +6,13 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.14 01:00:00                  #
+# Updated Date: 2024.12.12 04:00:00                  #
 # ================================================== #
 
 from PySide6.QtCore import QObject
+from PySide6.QtWidgets import QWidget
 
+from pygpt_net.core.tabs.tab import Tab
 from pygpt_net.ui.widget.dialog.base import BaseDialog
 
 class BaseTool(QObject):
@@ -25,6 +27,9 @@ class BaseTool(QObject):
         super(BaseTool, self).__init__()
         self.window = None
         self.id = ""
+        self.has_tab = False
+        self.tab_title = ""
+        self.tab_icon = ":/icons/build.svg"
 
     def setup(self):
         """Setup tool"""
@@ -83,6 +88,15 @@ class BaseTool(QObject):
         """
         return None
 
+    def as_tab(self, tab: Tab) -> QWidget or None:
+        """
+        Spawn and return tab instance
+
+        :param tab: Parent tab instance
+        :return: Tab widget instance
+        """
+        return None
+
     def get_lang_mappings(self) -> dict:
         """
         Get language mappings