pygpt-net 2.4.37__py3-none-any.whl → 2.4.42__py3-none-any.whl

pygpt_net/plugin/cmd_history/config.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.18 19:00:00                  #
+# Updated Date: 2024.12.14 19:00:00                  #
 # ================================================== #
 
 from pygpt_net.plugin.base.config import BaseConfig, BasePlugin
@@ -34,7 +34,7 @@ class Config(BaseConfig):
         plugin.add_option(
             "model_summarize",
             type="combo",
-            value="gpt-3.5-turbo",
+            value="gpt-4o-mini",
             label="Model",
             description="Model used for summarize, default: gpt-3.5-turbo",
             tooltip="Summarize model",

pygpt_net/plugin/cmd_web/__init__.py

@@ -6,14 +6,12 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.20 21:00:00                  #
+# Updated Date: 2024.12.15 01:00:00                  #
 # ================================================== #
 
 import ssl
 from urllib.request import Request, urlopen
 
-from requests import HTTPError
-
 from pygpt_net.plugin.base.plugin import BasePlugin
 from pygpt_net.provider.web.base import BaseProvider
 from pygpt_net.core.events import Event
@@ -38,8 +36,11 @@ class Plugin(BasePlugin):
             "web_urls",
             "web_url_open",
             "web_url_raw",
+            "web_request",
             "web_index",
             "web_index_query",
+            "web_extract_links",
+            "web_extract_images",
         ]
         self.order = 100
         self.use_locale = True
@@ -94,7 +95,7 @@ class Plugin(BasePlugin):
 
         :return: providers dict
         """
-        return self.window.core.web.get_providers("search_engine")
+        return self.window.core.web.get_providers(self.window.core.web.PROVIDER_SEARCH_ENGINE)
 
     def get_provider_options(self) -> list:
         """Get provider options"""
@@ -269,26 +270,66 @@ class Plugin(BasePlugin):
 
         :param data: event data dict
         """
+        max_urls = self.get_option_value("max_open_urls")
+        is_summary = True
+        if self.get_option_value("raw"):
+            is_summary = False
+
         for option in self.allowed_cmds:
             if not self.has_cmd(option):
                 continue
 
-            # special case for web_index
             if option == "web_index":
                 data['cmd'].append(self.prepare_idx_syntax())  # prepare params
                 continue
             elif option == "web_index_query":
                 data['cmd'].append(self.prepare_idx_query_syntax())  # prepare params
                 continue
+            elif option == "web_url_open":
+                cmd = self.get_cmd(option)
+                try:
+                    cmd["instruction"] = cmd["instruction"].format(
+                        max_urls=max_urls,
+                    )
+                    if not is_summary:
+                        for param in list(cmd["params"]):
+                            if param["name"] == "summarize_prompt":
+                                cmd["params"].remove(param)
+                    data['cmd'].append(cmd)
+                    continue
+                except Exception as e:
+                    pass
+            elif option == "web_url_raw":
+                cmd = self.get_cmd(option)
+                try:
+                    cmd["instruction"] = cmd["instruction"].format(
+                        max_urls=max_urls,
+                    )
+                    if not is_summary:
+                        for param in list(cmd["params"]):
+                            if param["name"] == "summarize_prompt":
+                                cmd["params"].remove(param)
+                    data['cmd'].append(cmd)
+                    continue
+                except Exception as e:
+                    pass
             elif option == "web_search":
                 max_pages = self.get_option_value("num_pages")
                 cmd = self.get_cmd(option)
                 try:
-                    cmd["instruction"] = cmd["instruction"].format(max_pages=max_pages)
+                    cmd["instruction"] = cmd["instruction"].format(
+                        max_pages=max_pages,
+                        max_urls=max_urls,
+                    )
+                    if not is_summary:
+                        for param in list(cmd["params"]):
+                            if param["name"] == "summarize_prompt":
+                                cmd["params"].remove(param)
                     data['cmd'].append(cmd)
                     continue
                 except Exception as e:
                     pass
+
             data['cmd'].append(self.get_cmd(option))  # append command
 
     def cmd(self, ctx: CtxItem, cmds: list):
@@ -389,8 +430,6 @@ class Plugin(BasePlugin):
                     req,
                     timeout=self.get_option_value('timeout'),
                 ).read()
-        except HTTPError as e:
-            data = str(e)
         except Exception as e:
-            raise e
+            data = str(e)
         return data

pygpt_net/plugin/cmd_web/config.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.18 19:00:00                  #
+# Updated Date: 2024.12.15 01:00:00                  #
 # ================================================== #
 
 from pygpt_net.plugin.base.config import BaseConfig, BasePlugin
@@ -41,6 +41,15 @@ class Config(BaseConfig):
             min=1,
             max=None,
         )
+        plugin.add_option(
+            "max_open_urls",
+            type="int",
+            value=1,
+            label="Number of max URLs to open at once",
+            description="Number of max URLs to open at once",
+            min=1,
+            max=None,
+        )
         plugin.add_option(
             "max_page_content_length",
             type="int",
@@ -59,6 +68,14 @@ class Config(BaseConfig):
             min=1,
             max=None,
         )
+        plugin.add_option(
+            "raw",
+            type="bool",
+            value=True,
+            label="Use raw content (without summarization)",
+            description="Return raw content from web search instead of summarized content",
+            tooltip="Use raw content for search",
+        )
         plugin.add_option(
             "disable_ssl",
             type="bool",
@@ -67,6 +84,14 @@ class Config(BaseConfig):
             description="Disables SSL verification when crawling web pages",
             tooltip="Disable SSL verify",
         )
+        plugin.add_option(
+            "img_thumbnail",
+            type="bool",
+            value=True,
+            label="Show thumbnail images",
+            description="Enable fetching thumbnails from opened websites",
+            tooltip="Enable fetching thumbnails from opened websites",
+        )
         plugin.add_option(
             "timeout",
             type="int",
@@ -86,7 +111,7 @@ class Config(BaseConfig):
         plugin.add_option(
             "max_result_length",
             type="int",
-            value=1500,
+            value=50000,
             label="Max result length",
             description="Max length of summarized result (characters)",
             min=0,
@@ -104,10 +129,10 @@ class Config(BaseConfig):
         plugin.add_option(
             "model_tmp_query",
             type="combo",
-            value="gpt-3.5-turbo",
+            value="gpt-4o-mini",
             label="Model for query in-memory index",
             description="Model used for query in-memory index for `web_index_query` command, "
-                        "default: gpt-3.5-turbo",
+                        "default: gpt-4o-mini",
             tooltip="Query model",
             use="models",
             tab="indexing",
@@ -125,19 +150,21 @@ class Config(BaseConfig):
         )
         plugin.add_option(
             "idx",
-            type="text",
+            type="combo",
             value="base",
             label="Index to use",
             description="ID of index to use for web page indexing (persistent index)",
             tooltip="Index name",
+            use="idx",
             tab="indexing",
         )
         plugin.add_option(
             "summary_model",
-            type="text",
-            value="gpt-3.5-turbo-1106",
+            type="combo",
+            value="gpt-4o-mini",
             label="Model used for web page summarize",
-            description="Model used for web page summarize, default: gpt-3.5-turbo-1106",
+            description="Model used for web page summarize, default: gpt-4o-mini",
+            use="models",
             advanced=True,
         )
         plugin.add_option(
@@ -163,12 +190,40 @@ class Config(BaseConfig):
         )
 
         # commands
+        plugin.add_cmd(
+            "web_url_open",
+            instruction="read and get text content from ANY website URL. Always open a max of {max_urls} URLs at a time.",
+            params=[
+                {
+                    "name": "url",
+                    "type": "str",
+                    "description": "URL to website",
+                    "required": True,
+                },
+            ],
+            enabled=True,
+            description="If enabled, model will be able to open URL and read text content from it",
+        )
+        plugin.add_cmd(
+            "web_url_raw",
+            instruction="read and get raw HTML body from ANY website URL. Always open a max of {max_urls} URLs at a time.",
+            params=[
+                {
+                    "name": "url",
+                    "type": "str",
+                    "description": "URL to website",
+                    "required": True,
+                },
+            ],
+            enabled=True,
+            description="If enabled, model will be able to open specified URL and get raw HTML body from it",
+        )
         plugin.add_cmd(
             "web_search",
-            instruction="search the Web for more info, prepare a query for the search engine itself, start from page 1. "
-                        "If no results, then try the next page. "
-                        "Use a custom summary prompt if necessary, otherwise, a default summary will be used. "
-                        "Max pages: {max_pages}",
+            instruction="search the Web for list of URLs, prepare search query itself, list of "
+                        "URLs will be returned, 10 links per page max. After receiving the list of URLs, "
+                        "choose the best matched URLs and use the `web_url_open` command to read the content. "
+                        "Always open a max of {max_urls} URLs at a time.",
             params=[
                 {
                     "name": "query",
@@ -183,39 +238,85 @@ class Config(BaseConfig):
                     "required": False,
                 },
                 {
-                    "name": "summarize_prompt",
-                    "type": "str",
-                    "description": "summary prompt",
+                    "name": "num_links",
+                    "type": "int",
+                    "description": "links per page",
                     "required": False,
                 },
             ],
             enabled=True,
-            description="If enabled, model will be able to search the Web",
+            description="If enabled, model will be able to search the Web and get founded URLs list",
         )
         plugin.add_cmd(
-            "web_url_open",
-            instruction="read and get summarized content from ANY website URL. Use a custom summary prompt if necessary, "
-                        "otherwise default summary will be used",
+            "web_request",
+            instruction="send HTTP request to specified URL or API endpoint, using `requests` library. Tip: to "
+                        "send raw data, use `data` parameter. To send data in `application/x-www-form-urlencoded` format, "
+                        "use `data_form` parameter. To send data in `application/json` format, use `data_json` parameter. "
+                        "Sending data in `data_form` and `data_json` parameters will automatically set `Content-Type`. To "
+                        "upload files, use `files` parameter. To send cookies, use `cookies` parameter. To set custom headers, "
+                        "use `headers` parameter. You can combine all parameters.",
             params=[
                 {
                     "name": "url",
                     "type": "str",
-                    "description": "URL to website",
+                    "description": "URL to website or API endpoint",
                     "required": True,
                 },
                 {
-                    "name": "summarize_prompt",
+                    "name": "method",
+                    "type": "str",
+                    "description": "HTTP method, default: GET",
+                    "required": False,
+                },
+                {
+                    "name": "headers",
+                    "type": "dict",
+                    "description": "HTTP headers",
+                    "required": False,
+                },
+                {
+                    "name": "params",
+                    "type": "dict",
+                    "description": "GET parameters",
+                    "required": False,
+                },
+                {
+                    "name": "data",
                     "type": "str",
-                    "description": "summary prompt",
+                    "description": "raw data to send in POST requests",
+                    "required": False,
+                },
+                {
+                    "name": "data_form",
+                    "type": "dict",
+                    "description": "data to send in POST `application/x-www-form-urlencoded` requests",
+                    "required": False,
+                },
+                {
+                    "name": "data_json",
+                    "type": "dict",
+                    "description": "JSON data to send in POST `application/json` requests",
+                    "required": False,
+                },
+                {
+                    "name": "cookies",
+                    "type": "dict",
+                    "description": "Cookies",
+                    "required": False,
+                },
+                {
+                    "name": "files",
+                    "type": "dict",
+                    "description": "Files to upload, key is form field name, value is absolute path to file",
                     "required": False,
                 },
             ],
             enabled=True,
-            description="If enabled, model will be able to open URL and summarize content",
+            description="If enabled, model will be able to send any HTTP request to specified URL or API endpoint",
         )
         plugin.add_cmd(
-            "web_url_raw",
-            instruction="read and get raw HTML/txt content (not summarized) from ANY website URL",
+            "web_extract_links",
+            instruction="open webpage and get list of all links from it",
             params=[
                 {
                     "name": "url",
@@ -225,34 +326,27 @@ class Config(BaseConfig):
                 },
             ],
             enabled=True,
-            description="If enabled, model will be able to open specified URL and get raw content",
+            description="If enabled, model will be able to open URL and get list of all links from it",
         )
         plugin.add_cmd(
-            "web_urls",
-            instruction="search the Web for list of URLs, prepare search query itself, list of "
-                        "URLs will be returned, 10 links per page max.",
+            "web_extract_images",
+            instruction="open webpage and get list of all images from it",
             params=[
                 {
-                    "name": "query",
+                    "name": "url",
                     "type": "str",
-                    "description": "search query",
+                    "description": "URL to website",
                     "required": True,
                 },
                 {
-                    "name": "page",
-                    "type": "int",
-                    "description": "page number",
-                    "required": False,
-                },
-                {
-                    "name": "num_links",
-                    "type": "int",
-                    "description": "links per page",
+                    "name": "download",
+                    "type": "bool",
+                    "description": "Download images to disk if user wants to, default: False",
                     "required": False,
                 },
             ],
             enabled=True,
-            description="If enabled, model will be able to search the Web and get founded URLs list",
+            description="If enabled, model will be able to open URL and get list of all images from it",
         )
         plugin.add_cmd(
             "web_index",

pygpt_net/plugin/cmd_web/websearch.py

@@ -6,10 +6,12 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.21 02:00:00                  #
+# Updated Date: 2024.12.14 22:00:00                  #
 # ================================================== #
 
 import re
+from typing import Optional, Tuple, Any, List
+
 from bs4 import BeautifulSoup
 
 from pygpt_net.core.events import KernelEvent
@@ -31,7 +33,7 @@ class WebSearch:
             query: str,
             num: int,
             offset: int = 1
-    ) -> list:
+    ) -> List[str]:
         """
         Execute search query and return list of urls
 
@@ -42,7 +44,7 @@ class WebSearch:
         """
         return self.plugin.get_provider().search(query, num, offset)
 
-    def get_urls(self, query: str) -> list:
+    def get_urls(self, query: str) -> List[str]:
         """
         Search the web and returns URLs
 
@@ -103,7 +105,7 @@ class WebSearch:
             self,
             text: str,
             chunk_size: int
-    ) -> list:
+    ) -> List[str]:
         """
         Split text into chunks
 
@@ -119,7 +121,7 @@ class WebSearch:
             self,
             chunks: list,
             query: str,
-            summarize_prompt: str = None
+            summarize_prompt: Optional[str] = None
     ) -> str:
         """
         Get summarized text from chunks
@@ -194,19 +196,22 @@ class WebSearch:
             query: str,
             page_no: int = 1,
             summarize_prompt: str = ""
-    ) -> (str, int, int, str):
+    ) -> Tuple[str, int, int, str, str]:
         """
         Get result from search query
 
         :param query: query to search
         :param page_no: page number
         :param summarize_prompt: custom prompt
-        :return: result, total_found, current, url
+        :return: result, total_found, current, url, thumb image
         """
         self.log("Using web query: " + query)
         urls = self.get_urls(query)
 
         # get options
+        is_summary = True
+        if self.plugin.get_option_value("raw"):
+            is_summary = False
         max_per_page = int(self.plugin.get_option_value("max_page_content_length"))
         chunk_size = int(self.plugin.get_option_value("chunk_size"))
         max_result_size = int(self.plugin.get_option_value("max_result_length"))
@@ -216,6 +221,7 @@ class WebSearch:
         i = 1
         current = 1
         url = ""
+        img = None
         for url in urls:
             if url is None or url == "":
                 continue
@@ -236,19 +242,27 @@ class WebSearch:
             if 0 < max_per_page < len(content):
                 content = content[:max_per_page]
 
-            chunks = self.to_chunks(content, chunk_size)  # it returns list of chunks
-            self.debug(
-                "Plugin: cmd_web: URL: {}".format(url)
-            )
-            result = self.get_summary(
-                chunks,
-                str(query),
-                summarize_prompt,
-            )
+            # get summary
+            if is_summary:
+                chunks = self.to_chunks(content, chunk_size)  # it returns list of chunks
+                self.debug(
+                    "Plugin: cmd_web: URL: {}".format(url)
+                )
+                result = self.get_summary(
+                    chunks,
+                    str(query),
+                    summarize_prompt,
+                )
+            else:
+                # no summary
+                result = str(content)
+
             # if result then stop
             if result is not None and result != "":
+                # get thumbnail image
+                if self.plugin.get_option_value("img_thumbnail"):
+                    img = self.plugin.window.core.web.helpers.get_main_image(url)
                 self.log("Summary generated (chars: {})".format(len(result)))
-
                 # index webpage if auto-index is enabled
                 self.index_url(url)
                 break
@@ -262,7 +276,7 @@ class WebSearch:
                 "Plugin: cmd_web: summary length: {}".format(len(result))
             )
 
-        if len(result) > max_result_size:
+        if 0 < max_result_size < len(result):
             result = result[:max_result_size]
 
         self.debug(
@@ -273,25 +287,37 @@ class WebSearch:
             result, \
             total_found, \
             current, \
-            url
+            url, \
+            img
 
-    def open_url(self, url: str, summarize_prompt: str = "") -> (str, str):
+    def open_url(
+            self,
+            url: str,
+            summarize_prompt: str = ""
+    ) -> Tuple[Optional[str], str, Optional[str]]:
         """
         Get result from specified URL
 
         :param url: URL to visit
         :param summarize_prompt: custom prompt
-        :return: result, url
+        :return: result, url, thumb image
         """
         self.log("Using URL: " + url)
-
         # get options
+        is_summary = True
+        if self.plugin.get_option_value("raw"):
+            is_summary = False
         max_per_page = int(self.plugin.get_option_value("max_page_content_length"))
         chunk_size = int(self.plugin.get_option_value("chunk_size"))
         max_result_size = int(self.plugin.get_option_value("max_result_length"))
 
+        img = None
         self.log("URL: " + url)
         content = self.query_url(url)
+
+        if content is None:
+            return None, url, img
+
         self.log("Content found (chars: {}). Please wait...".format(len(content)))
 
         # index webpage if auto-index is enabled
@@ -300,14 +326,23 @@ class WebSearch:
 
         if 0 < max_per_page < len(content):
             content = content[:max_per_page]
-        chunks = self.to_chunks(
-            content,
-            chunk_size,
-        )  # it returns list of chunks
 
         self.debug("Plugin: cmd_web: URL: {}".format(url))  # log
 
-        result = self.get_summary(chunks, "", summarize_prompt)
+        # get summary
+        if is_summary:
+            chunks = self.to_chunks(
+                content,
+                chunk_size,
+            )  # it returns list of chunks
+            result = self.get_summary(chunks, "", summarize_prompt)
+        else:
+            # no summary
+            result = str(content)
+
+        # get thumbnail image
+        if self.plugin.get_option_value("img_thumbnail"):
+            img = self.plugin.window.core.web.helpers.get_main_image(url)
 
         if result is not None and result != "":
             self.log("Summary generated (chars: {})".format(len(result)))
@@ -318,21 +353,24 @@ class WebSearch:
                 "Plugin: cmd_web: summary length: {}".format(len(result))
             )
 
-        if len(result) > max_result_size:
+        if 0 < max_result_size < len(result):
             result = result[:max_result_size]
 
         self.debug(
             "Plugin: cmd_web: result length: {}".format(len(result))
         )
 
-        return result, url
+        return result, url, img
 
-    def open_url_raw(self, url: str) -> (str, str):
+    def open_url_raw(
+            self,
+            url: str
+    ) -> Tuple[str, str, Optional[str]]:
         """
         Get raw content from specified URL
 
         :param url: URL to visit
-        :return: result, url
+        :return: result, url, thumb image
         """
         self.log("Using URL: " + url)
 
@@ -343,8 +381,11 @@ class WebSearch:
         result = self.query_url(url)
         self.log("Content found (chars: {}). Please wait...".format(len(result)))
 
+        img = None
         # index webpage if auto-index is enabled
         if result:
+            if self.plugin.get_option_value("img_thumbnail"):
+                img = self.plugin.window.core.web.helpers.get_main_image(url)
             self.index_url(url)
 
         # strip if too long
@@ -355,7 +396,7 @@ class WebSearch:
             "Plugin: cmd_web: result length: {}".format(len(result))
         )
 
-        return result, url
+        return result, url, img
 
     def index_url(self, url: str):
         """
@@ -381,7 +422,7 @@ class WebSearch:
         except Exception as e:
             self.error(e)
 
-    def error(self, err: any):
+    def error(self, err: Any):
         """
         Log error message