pygpt-net 2.6.55__py3-none-any.whl → 2.6.57__py3-none-any.whl

pygpt_net/plugin/cmd_web/worker.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2025.08.11 14:00:00                  #
+# Updated Date: 2025.09.22 15:00:00                  #
 # ================================================== #
 
 import json
@@ -32,6 +32,73 @@ class Worker(BaseWorker):
         self.ctx = None
         self.msg = None
 
+    # --- helpers ---
+
+    def _to_list(self, value):
+        """Normalize any value to a flat list of strings."""
+        # Accept list-like values
+        if value is None:
+            return []
+        if isinstance(value, list):
+            result = []
+            for v in value:
+                result.extend(self._to_list(v))
+            return result
+        if isinstance(value, (tuple, set)):
+            result = []
+            for v in value:
+                result.extend(self._to_list(v))
+            return result
+        # Accept string (also split by whitespace/newlines to support multiple URLs pasted together)
+        if isinstance(value, str):
+            # Split by any whitespace; do not split by comma/semicolon to avoid breaking data: URLs etc.
+            parts = value.split()
+            return [p.strip() for p in parts if p and isinstance(p, str)]
+        # Fallback: try to cast to str
+        try:
+            s = str(value).strip()
+            return [s] if s else []
+        except Exception:
+            return []
+
+    def _extract_urls(self, item: dict, allow_args: bool = True) -> list:
+        """Extract and normalize URLs from item (supports url, urls; also from args if present)."""
+        urls = []
+        try:
+            if self.has_param(item, "urls"):
+                urls.extend(self._to_list(self.get_param(item, "urls")))
+            if self.has_param(item, "url"):
+                urls.extend(self._to_list(self.get_param(item, "url")))
+            if allow_args and self.has_param(item, "args"):
+                args = self.get_param(item, "args")
+                if isinstance(args, dict):
+                    if "urls" in args:
+                        urls.extend(self._to_list(args.get("urls")))
+                    if "url" in args:
+                        urls.extend(self._to_list(args.get("url")))
+        except Exception:
+            pass
+
+        # Deduplicate and cleanup
+        cleaned = []
+        seen = set()
+        for u in urls:
+            if not u:
+                continue
+            if not isinstance(u, str):
+                try:
+                    u = str(u)
+                except Exception:
+                    continue
+            u = u.strip()
+            if not u:
+                continue
+            if u in seen:
+                continue
+            seen.add(u)
+            cleaned.append(u)
+        return cleaned
+
     @Slot()
     def run(self):
         try:
@@ -111,7 +178,7 @@ class Worker(BaseWorker):
             page,
             prompt,
         )
-        self.msg = "Web search finished: '{}'".format(query )
+        self.msg = f"Web search finished: '{query}'"
         result = {
             'query': query,
             'content': content,
@@ -123,7 +190,8 @@ class Worker(BaseWorker):
             self.ctx.urls_before.append(url)
         if img:
             result["thumb_img"] = img
-            self.ctx.images_before.append(img)
+            if img not in self.ctx.images_before:
+                self.ctx.images_before.append(img)
 
         self.ctx.save_reply = True  # leave links in context
         return self.make_response(item, result)
@@ -138,28 +206,43 @@ class Worker(BaseWorker):
         prompt = None
         if self.has_param(item, "summarize_prompt"):
             prompt = self.get_param(item, "summarize_prompt")
-        url = self.get_param(item, "url", "")
 
-        self.msg = "Opening Web URL: '{}'".format(url)
-        content, url, img = self.websearch.open_url(
-            url,
-            prompt,
-        )
-        result = {
-            'url': url,
-            'content': content,
-        }
-        context = "From: " + url + ":\n--------------------------------\n" + str(content)
-        if url:
-            self.ctx.urls_before.append(url)
-        if img:
-            result["thumb_img"] = img
-            self.ctx.images_before.append(img)
+        # Get URLs with robust fallback (urls/url; also supports lists)
+        urls = self._extract_urls(item, allow_args=False)
+
+        results = []
+        context = ""
+        for url in urls:
+            try:
+                self.msg = f"Opening Web URL: '{url}'"
+                content, url_resolved, img = self.websearch.open_url(
+                    url,
+                    prompt,
+                )
+                # Always include the URL the result refers to
+                result = {
+                    'url': url_resolved or url,
+                    'content': content,
+                }
+                context += f"From: {url_resolved or url} + :\n--------------------------------\n{content}"
+                if url_resolved or url:
+                    self.ctx.urls_before.append(url_resolved or url)
+                if img:
+                    result["thumb_img"] = img
+                    if img not in self.ctx.images_before:
+                        self.ctx.images_before.append(img)
+                results.append(result)
+            except Exception as e:
+                results.append({
+                    'url': url,
+                    'content': f"Error: {e}",
+                })
+                self.log(f"Error opening URL '{url}': {e}")
 
         extra = {
             "context": context,
         }
-        return self.make_response(item, result, extra=extra)
+        return self.make_response(item, results, extra=extra)
 
     def cmd_web_url_raw(self, item: dict) -> dict:
         """
@@ -168,26 +251,40 @@ class Worker(BaseWorker):
         :param item: command item
         :return: response item
         """
-        url = self.get_param(item, "url", "")
-        self.msg = "Opening Web URL: '{}'".format(url)
-        content, url, img  = self.websearch.open_url_raw(
-            url,
-        )
-        result = {
-            'url': url,
-            'content': content,
-        }
-        context = "From: " + url + ":\n--------------------------------\n" + str(content)
-        if url:
-            self.ctx.urls_before.append(url)
-        if img:
-            result["thumb_img"] = img
-            self.ctx.images_before.append(img)
+        # Get URLs with robust fallback (urls/url; also supports lists)
+        urls = self._extract_urls(item, allow_args=False)
+
+        results = []
+        context = ""
+        for url in urls:
+            try:
+                self.msg = f"Opening Web URL: '{url}'"
+                content, url_resolved, img = self.websearch.open_url_raw(
+                    url,
+                )
+                result = {
+                    'url': url_resolved or url,
+                    'content': content,
+                }
+                context += f"From: {url_resolved or url} + :\n--------------------------------\n{content}"
+                if url_resolved or url:
+                    self.ctx.urls_before.append(url_resolved or url)
+                if img:
+                    result["thumb_img"] = img
+                    if img not in self.ctx.images_before:
+                        self.ctx.images_before.append(img)
+                results.append(result)
+            except Exception as e:
+                results.append({
+                    'url': url,
+                    'content': f"Error: {e}",
+                })
+                self.log(f"Error opening URL '{url}': {e}")
 
         extra = {
             "context": context,
         }
-        return self.make_response(item, result, extra=extra)
+        return self.make_response(item, results, extra=extra)
 
     def cmd_web_urls(self, item: dict) -> dict:
         """
@@ -216,7 +313,7 @@ class Worker(BaseWorker):
             num,
             offset,
         )
-        self.msg = "Web search finished: '{}'".format(query)
+        self.msg = f"Web search finished: '{query}'"
         result = {
             'query': query,
             'urls': json.dumps(urls),
@@ -226,7 +323,8 @@ class Worker(BaseWorker):
         }
         if urls:
             for url in urls:
-                self.ctx.urls_before.append(url)
+                if url not in self.ctx.urls_before:
+                    self.ctx.urls_before.append(url)
 
         return self.make_response(item, result)
 
@@ -245,39 +343,38 @@ class Worker(BaseWorker):
         if self.has_param(item, "args"):
             args = self.get_param(item, "args")
 
-        url = ""
-        if self.has_param(item, "url"):
-            url = self.get_param(item, "url")  # from default param
-        if "url" in args:
-            url = args["url"]  # override from args
-
-        if not url:
+        # Extract URLs with fallback: url(s) from item and args
+        urls = self._extract_urls(item, allow_args=True)
+        if not urls:
             return self.make_response(item, "No URL provided")
 
-        self.msg = "Indexing URL: '{}'".format(url)
         idx_name = self.plugin.get_option_value("idx")
-        self.status("Please wait... indexing: {}...".format(url))
-
-        # index URL via Llama-index
-        num, errors = self.plugin.window.core.idx.index_urls(
-            idx=idx_name,
-            urls=[url],
-            type=type,
-            extra_args=args,
-        )
-        result = {
-            'url': url,
-            'num_indexed': num,
-            'index': idx_name,
-            'errors': errors,
-        }
-        if url and (url.startswith("http://") or url.startswith("https://")):
-            self.ctx.urls_before.append(url)
+        results = []
 
-        extra = {
-            "url": url,
-        }
-        return self.make_response(item, result, extra=extra)
+        for url in urls:
+            self.msg = f"Indexing URL: '{url}'"
+            self.status(f"Please wait... indexing: {url}...")
+
+            # index URL via Llama-index per URL to keep per-URL result
+            num, errors = self.plugin.window.core.idx.index_urls(
+                idx=idx_name,
+                urls=[url],
+                type=type,
+                extra_args=args,
+            )
+            result = {
+                'url': url,
+                'num_indexed': num,
+                'index': idx_name,
+                'errors': errors,
+            }
+            if url and (url.startswith("http://") or url.startswith("https://")):
+                if url not in self.ctx.urls_before:
+                    self.ctx.urls_before.append(url)
+            results.append(result)
+
+        # Return list of results with URLs included
+        return self.make_response(item, results)
 
     def cmd_web_index_query(self, item: dict) -> dict:
         """
@@ -295,69 +392,85 @@ class Worker(BaseWorker):
         if self.has_param(item, "args"):
             args = self.get_param(item, "args")
 
-        url = ""
-        if self.has_param(item, "url"):
-            url = self.get_param(item, "url")  # from default param
-        if isinstance(args, dict) and "url" in args:
-            url = args["url"]  # override from args
-
-        if not url:
+        # Extract URLs with fallback: url(s) from item and args
+        urls = self._extract_urls(item, allow_args=True)
+        if not urls:
             result = "No URL provided"
             return self.make_response(item, result)
 
         if self.has_param(item, "query"):
             query = self.get_param(item, "query")
 
-        result = "No data"
-        context = None
-        if query is not None:
-            # query file using temp index (created on the fly)
-            self.log("Querying web: {}".format(url))
+        results = []
+        context = ""
 
-            # get tmp query model
-            model = self.plugin.window.core.models.from_defaults()
-            tmp_model = self.plugin.get_option_value("model_tmp_query")
-            if self.plugin.window.core.models.has(tmp_model):
-                model = self.plugin.window.core.models.get(tmp_model)
+        # get tmp query model (shared for all URLs)
+        model = self.plugin.window.core.models.from_defaults()
+        tmp_model = self.plugin.get_option_value("model_tmp_query")
+        if self.plugin.window.core.models.has(tmp_model):
+            model = self.plugin.window.core.models.get(tmp_model)
 
-            answer = self.plugin.window.core.idx.chat.query_web(
-                ctx=self.ctx,
-                type=type,
-                url=url,
-                args=args,
-                query=query,
-                model=model,
-            )
-            self.log("Response from temporary in-memory index: {}".format(answer))
-            if answer:
-                from_str = type
-                if url:
-                    from_str += ", URL: " + url
-                result = answer
-                context = "From: " + from_str + ":\n--------------------------------\n" + answer
-
-            # + auto-index to main index using Llama-index
-            if self.plugin.get_option_value("auto_index"):
-                self.msg = "Indexing URL: '{}'".format(url)
-                idx_name = self.plugin.get_option_value("idx")
-                # show status
-                self.status("Please wait... indexing: {}...".format(url))
-                # index URL via Llama-index
-                num, errors = self.plugin.window.core.idx.index_urls(
-                    idx=idx_name,
-                    urls=[url],
-                    type=type,
-                    extra_args=args,
-                )
+        if query is not None:
+            for url in urls:
+                try:
+                    # query file using temp index (created on the fly)
+                    self.log(f"Querying web: {url}")
+
+                    answer = self.plugin.window.core.idx.chat.query_web(
+                        ctx=self.ctx,
+                        type=type,
+                        url=url,
+                        args=args,
+                        query=query,
+                        model=model,
+                    )
+                    self.log(f"Response from temporary in-memory index: {answer}")
+                    content = answer if answer else "No data"
+                    from_str = type
+                    if url:
+                        from_str += ", URL: " + url
+                    context += "From: " + from_str + ":\n--------------------------------\n" + content
+
+                    # + auto-index to main index using Llama-index
+                    if self.plugin.get_option_value("auto_index"):
+                        self.msg = f"Indexing URL: '{url}'"
+                        idx_name = self.plugin.get_option_value("idx")
+                        # show status
+                        self.status(f"Please wait... indexing: {url}...")
+                        # index URL via Llama-index
+                        self.plugin.window.core.idx.index_urls(
+                            idx=idx_name,
+                            urls=[url],
+                            type=type,
+                            extra_args=args,
+                        )
 
-        # add URL to context
-        if url and (url.startswith("http://") or url.startswith("https://")):
-            self.ctx.urls_before.append(url)
+                    # add URL to context
+                    if url and (url.startswith("http://") or url.startswith("https://")):
+                        self.ctx.urls_before.append(url)
+
+                    results.append({
+                        'url': url,
+                        'content': content,
+                    })
+                except Exception as e:
+                    results.append({
+                        'url': url,
+                        'content': f"Error: {e}",
+                    })
+                    self.log(f"Error querying/indexing URL '{url}': {e}")
+        else:
+            # No query provided - return info per URL to keep URL in response
+            for url in urls:
+                results.append({
+                    'url': url,
+                    'content': "No data",
+                })
 
         extra = {
-            "context": context,
+            "context": context if context else None,
         }
-        return self.make_response(item, result, extra=extra)
+        return self.make_response(item, results, extra=extra)
 
     def cmd_web_extract_links(self, item: dict) -> dict:
         """
@@ -366,17 +479,30 @@ class Worker(BaseWorker):
         :param item: command item
         :return: response item
         """
-        url = ""
-        if self.has_param(item, "url"):
-            url = self.get_param(item, "url")
-        if not url:
+        urls = self._extract_urls(item, allow_args=False)
+        if not urls:
             return self.make_response(item, "No URL provided")
-        links = self.plugin.window.core.web.helpers.get_links(url)
-        result = {
-            'links': links,
-        }
-        self.ctx.urls_before.append(url)
-        return self.make_response(item, result)
+
+        results = []
+        for url in urls:
+            try:
+                links = self.plugin.window.core.web.helpers.get_links(url)
+                result = {
+                    'url': url,
+                    'links': links,
+                }
+                if url not in self.ctx.urls_before:
+                    self.ctx.urls_before.append(url)
+                results.append(result)
+            except Exception as e:
+                results.append({
+                    'url': url,
+                    'links': [],
+                    'content': f"Error: {e}",
+                })
+                self.log(f"Error extracting links from '{url}': {e}")
+
+        return self.make_response(item, results)
 
     def cmd_web_extract_images(self, item: dict) -> dict:
         """
@@ -386,27 +512,43 @@ class Worker(BaseWorker):
         :return: response item
         """
         download = False
-        url = ""
-        if self.has_param(item, "url"):
-            url = self.get_param(item, "url")
         if self.has_param(item, "download"):
             download = bool(self.get_param(item, "download"))
-        if not url:
+
+        urls = self._extract_urls(item, allow_args=False)
+        if not urls:
             return self.make_response(item, "No URL provided")
-        images = self.plugin.window.core.web.helpers.get_images(url)
-        result = {
-            'images': images,
-        }
-        if images and download:
-            for img in images:
-                try:
-                    path = self.plugin.window.core.web.helpers.download_image(img)
-                    if path:
-                        self.ctx.images_before.append(path)
-                except Exception as e:
-                    print(e)
-        self.ctx.urls_before.append(url)
-        return self.make_response(item, result)
+
+        results = []
+        for url in urls:
+            try:
+                images = self.plugin.window.core.web.helpers.get_images(url)
+                result = {
+                    'url': url,
+                    'images': images,
+                }
+                if images and download:
+                    for img in images:
+                        try:
+                            path = self.plugin.window.core.web.helpers.download_image(img)
+                            if path:
+                                if path not in self.ctx.images_before:
+                                    self.ctx.images_before.append(path)
+                        except Exception as e:
+                            # Keep downloading independent per image
+                            print(e)
+                if url not in self.ctx.urls_before:
+                    self.ctx.urls_before.append(url)
+                results.append(result)
+            except Exception as e:
+                results.append({
+                    'url': url,
+                    'images': [],
+                    'content': f"Error: {e}",
+                })
+                self.log(f"Error extracting images from '{url}': {e}")
+
+        return self.make_response(item, results)
 
     def cmd_web_request(self, item: dict) -> dict:
         """
@@ -415,7 +557,6 @@ class Worker(BaseWorker):
         :param item: command item
         :return: response item
         """
-        url = ""
         method = "GET"
         data = None
         raw = None
@@ -425,10 +566,10 @@ class Worker(BaseWorker):
         cookies = None
         files = None
 
-        if self.has_param(item, "url"):
-            url = self.get_param(item, "url")
-        if not url:
+        urls = self._extract_urls(item, allow_args=False)
+        if not urls:
             return self.make_response(item, "No URL provided")
+
         if self.has_param(item, "method"):
             method = self.get_param(item, "method")
         if self.has_param(item, "data_form"):
@@ -446,24 +587,37 @@ class Worker(BaseWorker):
         if self.has_param(item, "files"):
             files = self.get_param(item, "files")
 
-        code, response = self.window.core.web.helpers.request(
-            url=url,
-            method=method,
-            headers=headers,
-            params=params,
-            data=data if data else raw,
-            json=json,
-            cookies=cookies,
-            files=files,
-            timeout=self.plugin.get_option_value("timeout"),
-            disable_ssl_verify=self.plugin.get_option_value("disable_ssl"),
-            allow_redirects=True,
-            stream=False,
-            user_agent=self.plugin.get_option_value("user_agent"))
-        result = {
-            'url': url,
-            'code': code,
-            'response': response,
-        }
-        self.ctx.urls_before.append(url)
-        return self.make_response(item, result)
+        results = []
+        for url in urls:
+            try:
+                code, response = self.window.core.web.helpers.request(
+                    url=url,
+                    method=method,
+                    headers=headers,
+                    params=params,
+                    data=data if data else raw,
+                    json=json,
+                    cookies=cookies,
+                    files=files,
+                    timeout=self.plugin.get_option_value("timeout"),
+                    disable_ssl_verify=self.plugin.get_option_value("disable_ssl"),
+                    allow_redirects=True,
+                    stream=False,
+                    user_agent=self.plugin.get_option_value("user_agent"))
+                result = {
+                    'url': url,
+                    'code': code,
+                    'response': response,
+                }
+                if url not in self.ctx.urls_before:
+                    self.ctx.urls_before.append(url)
+                results.append(result)
+            except Exception as e:
+                results.append({
+                    'url': url,
+                    'code': None,
+                    'response': f"Error: {e}",
+                })
+                self.log(f"Error during web request to '{url}': {e}")
+
+        return self.make_response(item, results)