webscout 1.2.2__py3-none-any.whl → 1.2.4__py3-none-any.whl

DeepWEBS/documents/query_results_extractor.py

@@ -2,61 +2,79 @@ from bs4 import BeautifulSoup
 from pathlib import Path
 from DeepWEBS.utilsdw.logger import logger
 
-
 class QueryResultsExtractor:
     def __init__(self) -> None:
         self.query_results = []
         self.related_questions = []
 
     def load_html(self, html_path):
-        with open(html_path, "r", encoding="utf-8") as f:
-            html = f.read()
-        self.soup = BeautifulSoup(html, "html.parser")
+        try:
+            with open(html_path, "r", encoding="utf-8") as f:
+                html = f.read()
+            self.soup = BeautifulSoup(html, "html.parser")
+        except FileNotFoundError:
+            logger.error(f"File not found: {html_path}")
+        except Exception as e:
+            logger.error(f"Error loading HTML: {e}")
 
     def extract_query_results(self):
-        self.query = self.soup.find("textarea").text.strip()
-        query_result_elements = self.soup.find_all("div", class_="g")
-        for idx, result in enumerate(query_result_elements):
-            site = result.find("cite").find_previous("span").text.strip()
-            url = result.find("a")["href"]
-            title = result.find("h3").text.strip()
-
-            abstract_element_conditions = [
-                {"data-sncf": "1"},
-                {"class_": "ITZIwc"},
-            ]
-            for condition in abstract_element_conditions:
-                abstract_element = result.find("div", condition)
-                if abstract_element is not None:
-                    abstract = abstract_element.text.strip()
-                    break
-            else:
-                abstract = ""
-
-            logger.mesg(
-                f"{title}\n" f"  - {site}\n" f"  - {url}\n" f"  - {abstract}\n" f"\n"
-            )
-            self.query_results.append(
-                {
-                    "title": title,
-                    "site": site,
-                    "url": url,
-                    "abstract": abstract,
-                    "index": idx,
-                    "type": "web",
-                }
-            )
-        logger.success(f"- {len(query_result_elements)} query results")
+        try:
+            self.query = self.soup.find("textarea").text.strip()
+            query_result_elements = self.soup.find_all("div", class_="g")
+            for idx, result in enumerate(query_result_elements):
+                try:
+                    site = result.find("cite").find_previous("span").text.strip()
+                    url = result.find("a")["href"]
+                    title = result.find("h3").text.strip()
+                    abstract_element_conditions = [
+                        {"data-sncf": "1"},
+                        {"class_": "ITZIwc"},
+                    ]
+                    for condition in abstract_element_conditions:
+                        abstract_element = result.find("div", condition)
+                        if abstract_element is not None:
+                            abstract = abstract_element.text.strip()
+                            break
+                    else:
+                        abstract = ""
+                    logger.mesg(
+                        f"{title}\n"
+                        f" - {site}\n"
+                        f" - {url}\n"
+                        f" - {abstract}\n"
+                        f"\n"
+                    )
+                    self.query_results.append(
+                        {
+                            "title": title,
+                            "site": site,
+                            "url": url,
+                            "abstract": abstract,
+                            "index": idx,
+                            "type": "web",
+                        }
+                    )
+                except Exception as e:
+                    logger.error(f"Error extracting query result: {e}")
+            logger.success(f"- {len(query_result_elements)} query results")
+        except Exception as e:
+            logger.error(f"Error extracting query results: {e}")
 
     def extract_related_questions(self):
-        related_question_elements = self.soup.find_all(
-            "div", class_="related-question-pair"
-        )
-        for question_element in related_question_elements:
-            question = question_element.find("span").text.strip()
-            print(question)
-            self.related_questions.append(question)
-        logger.success(f"- {len(self.related_questions)} related questions")
+        try:
+            related_question_elements = self.soup.find_all(
+                "div", class_="related-question-pair"
+            )
+            for question_element in related_question_elements:
+                try:
+                    question = question_element.find("span").text.strip()
+                    print(question)
+                    self.related_questions.append(question)
+                except Exception as e:
+                    logger.error(f"Error extracting related question: {e}")
+            logger.success(f"- {len(self.related_questions)} related questions")
+        except Exception as e:
+            logger.error(f"Error extracting related questions: {e}")
 
     def extract(self, html_path):
         self.load_html(html_path)
@@ -75,4 +93,7 @@ if __name__ == "__main__":
     html_filename = "python_tutorials"
     html_path = html_path_root / f"{html_filename}.html"
     extractor = QueryResultsExtractor()
-    extractor.extract(html_path)
+    try:
+        extractor.extract(html_path)
+    except Exception as e:
+        logger.error(f"Error in main function: {e}")

DeepWEBS/networks/google_searcher.py

@@ -1,11 +1,12 @@
 import requests
 from pathlib import Path
+from typing import Optional
+import random
 from DeepWEBS.utilsdw.enver import enver
 from DeepWEBS.utilsdw.logger import logger
 from DeepWEBS.networks.filepath_converter import QueryToFilepathConverter
 from DeepWEBS.networks.network_configs import REQUESTS_HEADERS
 
-
 class GoogleSearcher:
     def __init__(self):
         self.url = "https://www.google.com/search"
@@ -13,36 +14,39 @@ class GoogleSearcher:
         self.enver.set_envs(proxies=True)
         self.filepath_converter = QueryToFilepathConverter()
 
-    def send_request(self, result_num=10, safe=False):
-        self.request_response = requests.get(
-            url=self.url,
+    def send_request(self, query: str, result_num: int = 10, safe: bool = False) -> requests.Response:
+        params = {
+            "q": query,
+            "num": result_num,
+        }
+        response = requests.get(
+            self.url,
             headers=REQUESTS_HEADERS,
-            params={
-                "q": self.query,
-                "num": result_num,
-            },
+            params=params,
             proxies=self.enver.requests_proxies,
         )
+        response.raise_for_status()  # Raise an exception for non-2xx status codes
+        return response
+
+    def save_response(self, response: requests.Response, html_path: Path) -> None:
+        html_path.parent.mkdir(parents=True, exist_ok=True)
+        logger.note(f"Saving to: [{html_path}]")
+        with html_path.open("wb") as wf:
+            wf.write(response.content)
 
-    def save_response(self):
-        if not self.html_path.exists():
-            self.html_path.parent.mkdir(parents=True, exist_ok=True)
-        logger.note(f"Saving to: [{self.html_path}]")
-        with open(self.html_path, "wb") as wf:
-            wf.write(self.request_response.content)
+    def search(self, query: str, result_num: int = 10, safe: bool = False, overwrite: bool = False) -> Path:
+        html_path = self.filepath_converter.convert(query)
+        logger.note(f"Searching: [{query}]")
 
-    def search(self, query, result_num=10, safe=False, overwrite=False):
-        self.query = query
-        self.html_path = self.filepath_converter.convert(self.query)
-        logger.note(f"Searching: [{self.query}]")
-        if self.html_path.exists() and not overwrite:
-            logger.success(f"HTML existed: {self.html_path}")
+        if html_path.exists() and not overwrite:
+            logger.success(f"HTML existed: {html_path}")
         else:
-            self.send_request(result_num=result_num, safe=safe)
-            self.save_response()
-        return self.html_path
+            response = self.send_request(query, result_num, safe)
+            self.save_response(response, html_path)
 
+        return html_path
 
 if __name__ == "__main__":
     searcher = GoogleSearcher()
-    searcher.search("python tutorials")
+    html_path = searcher.search("python tutorials")
+    print(f"HTML file saved at: {html_path}")

DeepWEBS/networks/webpage_fetcher.py

@@ -1,100 +1,92 @@
 import concurrent.futures
+import random
 import requests
 import tldextract
 from pathlib import Path
+from typing import List, Tuple, Dict
+
 from DeepWEBS.utilsdw.enver import enver
 from DeepWEBS.utilsdw.logger import logger
 from DeepWEBS.networks.filepath_converter import UrlToFilepathConverter
 from DeepWEBS.networks.network_configs import IGNORE_HOSTS, REQUESTS_HEADERS
 
-
 class WebpageFetcher:
     def __init__(self):
         self.enver = enver
         self.enver.set_envs(proxies=True)
         self.filepath_converter = UrlToFilepathConverter()
 
-    def is_ignored_host(self, url):
-        self.host = tldextract.extract(url).registered_domain
-        if self.host in IGNORE_HOSTS:
-            return True
-        else:
-            return False
+    def is_ignored_host(self, url: str) -> bool:
+        host = tldextract.extract(url).registered_domain
+        return host in IGNORE_HOSTS
 
-    def send_request(self):
+    def send_request(self, url: str) -> requests.Response:
         try:
-            self.request_response = requests.get(
-                url=self.url,
-                headers=REQUESTS_HEADERS,
+            user_agent = random.choice(REQUESTS_HEADERS["User-Agent"])
+            response = requests.get(
+                url=url,
+                headers={"User-Agent": user_agent},
                 proxies=self.enver.requests_proxies,
                 timeout=15,
             )
-        except:
-            logger.warn(f"Failed to fetch: [{self.url}]")
-            self.request_response = None
-
-    def save_response(self):
-        if not self.html_path.exists():
-            self.html_path.parent.mkdir(parents=True, exist_ok=True)
-        logger.success(f"Saving to: [{self.html_path}]")
-
-        if self.request_response is None:
+            response.raise_for_status()
+            return response
+        except requests.exceptions.RequestException as e:
+            logger.warn(f"Failed to fetch: [{url}] | {e}")
+            return None
+
+    def save_response(self, response: requests.Response, html_path: Path) -> None:
+        if response is None:
             return
-        else:
-            with open(self.html_path, "wb") as wf:
-                wf.write(self.request_response.content)
 
-    def fetch(self, url, overwrite=False, output_parent=None):
-        self.url = url
-        logger.note(f"Fetching: [{self.url}]")
-        self.html_path = self.filepath_converter.convert(self.url, parent=output_parent)
+        html_path.parent.mkdir(parents=True, exist_ok=True)
+        logger.success(f"Saving to: [{html_path}]")
+        with html_path.open("wb") as wf:
+            wf.write(response.content)
+
+    def fetch(self, url: str, overwrite: bool = False, output_parent: str = None) -> Path:
+        logger.note(f"Fetching: [{url}]")
+        html_path = self.filepath_converter.convert(url, parent=output_parent)
 
-        if self.is_ignored_host(self.url):
-            logger.warn(f"Ignore host: [{self.host}]")
-            return self.html_path
+        if self.is_ignored_host(url):
+            logger.warn(f"Ignored host: [{tldextract.extract(url).registered_domain}]")
+            return html_path
 
-        if self.html_path.exists() and not overwrite:
-            logger.success(f"HTML existed: [{self.html_path}]")
+        if html_path.exists() and not overwrite:
+            logger.success(f"HTML existed: [{html_path}]")
         else:
-            self.send_request()
-            self.save_response()
-        return self.html_path
+            response = self.send_request(url)
+            self.save_response(response, html_path)
 
+        return html_path
 
 class BatchWebpageFetcher:
     def __init__(self):
         self.done_count = 0
         self.total_count = 0
-        self.url_and_html_path_list = []
+        self.url_and_html_path_list: List[Dict[str, str]] = []
 
-    def fecth_single_webpage(self, url, overwrite=False, output_parent=None):
+    def fetch_single_webpage(self, url: str, overwrite: bool = False, output_parent: str = None) -> Tuple[str, Path]:
         webpage_fetcher = WebpageFetcher()
-        html_path = webpage_fetcher.fetch(
-            url=url, overwrite=overwrite, output_parent=output_parent
-        )
-        self.url_and_html_path_list.append({"url": url, "html_path": html_path})
+        html_path = webpage_fetcher.fetch(url, overwrite, output_parent)
+        self.url_and_html_path_list.append({"url": url, "html_path": str(html_path)})
         self.done_count += 1
         logger.success(f"> [{self.done_count}/{self.total_count}] Fetched: {url}")
+        return url, html_path
 
-    def fetch(self, urls, overwrite=False, output_parent=None):
+    def fetch(self, urls: List[str], overwrite: bool = False, output_parent: str = None) -> List[Dict[str, str]]:
         self.urls = urls
         self.total_count = len(self.urls)
+
         with concurrent.futures.ThreadPoolExecutor() as executor:
             futures = [
-                executor.submit(
-                    self.fecth_single_webpage,
-                    url=url,
-                    overwrite=overwrite,
-                    output_parent=output_parent,
-                )
+                executor.submit(self.fetch_single_webpage, url, overwrite, output_parent)
                 for url in urls
             ]
+            concurrent.futures.wait(futures)
 
-            for idx, future in enumerate(concurrent.futures.as_completed(futures)):
-                result = future.result()
         return self.url_and_html_path_list
 
-
 if __name__ == "__main__":
     urls = [
         "https://stackoverflow.com/questions/295135/turn-a-string-into-a-valid-filename",
@@ -102,6 +94,4 @@ if __name__ == "__main__":
         "https://docs.python.org/zh-cn/3/tutorial/interpreter.html",
     ]
     batch_webpage_fetcher = BatchWebpageFetcher()
-    batch_webpage_fetcher.fetch(
-        urls=urls, overwrite=True, output_parent="python tutorials"
-    )
+    batch_webpage_fetcher.fetch(urls=urls, overwrite=True, output_parent="python tutorials")

webscout/DWEBS.py

@@ -1,179 +1,197 @@
-
-from pydantic import BaseModel, Field
-from typing import Union
-
-from DeepWEBS.utilsdw.logger import logger
-from DeepWEBS.networks.google_searcher import GoogleSearcher
-from DeepWEBS.networks.webpage_fetcher import BatchWebpageFetcher
-from DeepWEBS.documents.query_results_extractor import QueryResultsExtractor
-from DeepWEBS.documents.webpage_content_extractor import BatchWebpageContentExtractor
-from DeepWEBS.utilsdw.logger import logger
-import argparse
-
-class DeepWEBS:
-    def __init__(self):
-        pass
-
-    class DeepSearch(BaseModel):
-        queries: list = Field(
-            default=[""],
-            description="(list[str]) Queries to search",
-        )
-        result_num: int = Field(
-            default=10,
-            description="(int) Number of search results",
-        )
-        safe: bool = Field(
-            default=False,
-            description="(bool) Enable SafeSearch",
-        )
-        types: list = Field(
-            default=["web"],
-            description="(list[str]) Types of search results: `web`, `image`, `videos`, `news`",
-        )
-        extract_webpage: bool = Field(
-            default=False,
-            description="(bool) Enable extracting main text contents from webpage, will add `text` filed in each `query_result` dict",
-        )
-        overwrite_query_html: bool = Field(
-            default=False,
-            description="(bool) Overwrite HTML file of query results",
-        )
-        overwrite_webpage_html: bool = Field(
-            default=False,
-            description="(bool) Overwrite HTML files of webpages from query results",
-        )
-
-    def queries_to_search_results(self, item: DeepSearch):
-        google_searcher = GoogleSearcher()
-        queries_search_results = []
-        for query in item.queries:
-            query_results_extractor = QueryResultsExtractor()
-            if not query.strip():
-                continue
-            query_html_path = google_searcher.search(
-                query=query,
-                result_num=item.result_num,
-                safe=item.safe,
-                overwrite=item.overwrite_query_html,
-            )
-            query_search_results = query_results_extractor.extract(query_html_path)
-            queries_search_results.append(query_search_results)
-        logger.note(queries_search_results)
-
-        if item.extract_webpage:
-            queries_search_results = self.extract_webpages(
-                queries_search_results,
-                overwrite_webpage_html=item.overwrite_webpage_html,
-            )
-        return queries_search_results
-
-    def extract_webpages(self, queries_search_results, overwrite_webpage_html=False):
-        for query_idx, query_search_results in enumerate(queries_search_results):
-            # Fetch webpages with urls
-            batch_webpage_fetcher = BatchWebpageFetcher()
-            urls = [
-                query_result["url"]
-                for query_result in query_search_results["query_results"]
-            ]
-            url_and_html_path_list = batch_webpage_fetcher.fetch(
-                urls,
-                overwrite=overwrite_webpage_html,
-                output_parent=query_search_results["query"],
-            )
-
-            # Extract webpage contents from htmls
-            html_paths = [
-                str(url_and_html_path["html_path"])
-                for url_and_html_path in url_and_html_path_list
-            ]
-            batch_webpage_content_extractor = BatchWebpageContentExtractor()
-            html_path_and_extracted_content_list = (
-                batch_webpage_content_extractor.extract(html_paths)
-            )
-
-            # Build the map of url to extracted_content
-            html_path_to_url_dict = {
-                str(url_and_html_path["html_path"]): url_and_html_path["url"]
-                for url_and_html_path in url_and_html_path_list
-            }
-            url_to_extracted_content_dict = {
-                html_path_to_url_dict[
-                    html_path_and_extracted_content["html_path"]
-                ]: html_path_and_extracted_content["extracted_content"]
-                for html_path_and_extracted_content in html_path_and_extracted_content_list
-            }
-
-            # Write extracted contents (as 'text' field) to query_search_results
-            for query_result_idx, query_result in enumerate(
-                query_search_results["query_results"]
-            ):
-                url = query_result["url"]
-                extracted_content = url_to_extracted_content_dict[url]
-                queries_search_results[query_idx]["query_results"][query_result_idx][
-                    "text"
-                ] = extracted_content
-
-        return queries_search_results
-
-
-class ArgParser(argparse.ArgumentParser):
-    def __init__(self, *args, **kwargs):
-        super(ArgParser, self).__init__(*args, **kwargs)
-
-        self.add_argument(
-            "-q",
-            "--queries",
-            type=str,
-            nargs="+",
-            required=True,
-            help="Queries to search",
-        )
-        self.add_argument(
-            "-n",
-            "--result_num",
-            type=int,
-            default=10,
-            help="Number of search results",
-        )
-        self.add_argument(
-            "-s",
-            "--safe",
-            default=False,
-            action="store_true",
-            help="Enable SafeSearch",
-        )
-        self.add_argument(
-            "-t",
-            "--types",
-            type=str,
-            nargs="+",
-            default=["web"],
-            choices=["web", "image", "videos", "news"],
-            help="Types of search results",
-        )
-        self.add_argument(
-            "-e",
-            "--extract_webpage",
-            default=False,
-            action="store_true",
-            help="Enable extracting main text contents from webpage",
-        )
-        self.add_argument(
-            "-o",
-            "--overwrite_query_html",
-            default=False,
-            action="store_true",
-            help="Overwrite HTML file of query results",
-        )
-        self.add_argument(
-            "-w",
-            "--overwrite_webpage_html",
-            default=False,
-            action="store_true",
-            help="Overwrite HTML files of webpages from query results",
-        )
-
-        self.args = self.parse_args()
-
-
+
+from pydantic import BaseModel, Field
+from typing import Union
+
+from DeepWEBS.utilsdw.logger import logger
+from DeepWEBS.networks.google_searcher import GoogleSearcher
+from DeepWEBS.networks.webpage_fetcher import BatchWebpageFetcher
+from DeepWEBS.documents.query_results_extractor import QueryResultsExtractor
+from DeepWEBS.documents.webpage_content_extractor import BatchWebpageContentExtractor
+from DeepWEBS.utilsdw.logger import logger
+import argparse
+
+class DeepWEBS:
+    def __init__(self):
+        pass
+
+    class DeepSearch(BaseModel):
+        queries: list = Field(
+            default=[""],
+            description="(list[str]) Queries to search",
+        )
+        result_num: int = Field(
+            default=10,
+            description="(int) Number of search results",
+        )
+        safe: bool = Field(
+            default=False,
+            description="(bool) Enable SafeSearch",
+        )
+        types: list = Field(
+            default=["web"],
+            description="(list[str]) Types of search results: `web`, `image`, `videos`, `news`",
+        )
+        extract_webpage: bool = Field(
+            default=False,
+            description="(bool) Enable extracting main text contents from webpage, will add `text` filed in each `query_result` dict",
+        )
+        overwrite_query_html: bool = Field(
+            default=False,
+            description="(bool) Overwrite HTML file of query results",
+        )
+        overwrite_webpage_html: bool = Field(
+            default=False,
+            description="(bool) Overwrite HTML files of webpages from query results",
+        )
+
+    def queries_to_search_results(self, item: DeepSearch):
+        google_searcher = GoogleSearcher()
+        queries_search_results = []
+        for query in item.queries:
+            query_results_extractor = QueryResultsExtractor()
+            if not query.strip():
+                continue
+            try:
+                query_html_path = google_searcher.search(
+                    query=query,
+                    result_num=item.result_num,
+                    safe=item.safe,
+                    overwrite=item.overwrite_query_html,
+                )
+            except Exception as e:
+                logger.error(f"Failed to search for query '{query}': {e}")
+                continue
+
+            try:
+                query_search_results = query_results_extractor.extract(query_html_path)
+            except Exception as e:
+                logger.error(f"Failed to extract search results for query '{query}': {e}")
+                continue
+
+            queries_search_results.append(query_search_results)
+        logger.note(queries_search_results)
+
+        if item.extract_webpage:
+            queries_search_results = self.extract_webpages(
+                queries_search_results,
+                overwrite_webpage_html=item.overwrite_webpage_html,
+            )
+        return queries_search_results
+
+    def extract_webpages(self, queries_search_results, overwrite_webpage_html=False):
+        for query_idx, query_search_results in enumerate(queries_search_results):
+            try:
+                # Fetch webpages with urls
+                batch_webpage_fetcher = BatchWebpageFetcher()
+                urls = [
+                    query_result["url"]
+                    for query_result in query_search_results["query_results"]
+                ]
+                url_and_html_path_list = batch_webpage_fetcher.fetch(
+                    urls,
+                    overwrite=overwrite_webpage_html,
+                    output_parent=query_search_results["query"],
+                )
+            except Exception as e:
+                logger.error(f"Failed to fetch webpages for query '{query_search_results['query']}': {e}")
+                continue
+
+            # Extract webpage contents from htmls
+            html_paths = [
+                str(url_and_html_path["html_path"])
+                for url_and_html_path in url_and_html_path_list
+            ]
+            batch_webpage_content_extractor = BatchWebpageContentExtractor()
+            try:
+                html_path_and_extracted_content_list = (
+                    batch_webpage_content_extractor.extract(html_paths)
+                )
+            except Exception as e:
+                logger.error(f"Failed to extract webpage contents for query '{query_search_results['query']}': {e}")
+                continue
+
+            # Build the map of url to extracted_content
+            html_path_to_url_dict = {
+                str(url_and_html_path["html_path"]): url_and_html_path["url"]
+                for url_and_html_path in url_and_html_path_list
+            }
+            url_to_extracted_content_dict = {
+                html_path_to_url_dict[
+                    html_path_and_extracted_content["html_path"]
+                ]: html_path_and_extracted_content["extracted_content"]
+                for html_path_and_extracted_content in html_path_and_extracted_content_list
+            }
+
+            # Write extracted contents (as 'text' field) to query_search_results
+            for query_result_idx, query_result in enumerate(
+                query_search_results["query_results"]
+            ):
+                url = query_result["url"]
+                extracted_content = url_to_extracted_content_dict.get(url, "")
+                queries_search_results[query_idx]["query_results"][query_result_idx][
+                    "text"
+                ] = extracted_content
+
+        return queries_search_results
+
+
+class ArgParser(argparse.ArgumentParser):
+    def __init__(self, *args, **kwargs):
+        super(ArgParser, self).__init__(*args, **kwargs)
+
+        self.add_argument(
+            "-q",
+            "--queries",
+            type=str,
+            nargs="+",
+            required=True,
+            help="Queries to search",
+        )
+        self.add_argument(
+            "-n",
+            "--result_num",
+            type=int,
+            default=10,
+            help="Number of search results",
+        )
+        self.add_argument(
+            "-s",
+            "--safe",
+            default=False,
+            action="store_true",
+            help="Enable SafeSearch",
+        )
+        self.add_argument(
+            "-t",
+            "--types",
+            type=str,
+            nargs="+",
+            default=["web"],
+            choices=["web", "image", "videos", "news"],
+            help="Types of search results",
+        )
+        self.add_argument(
+            "-e",
+            "--extract_webpage",
+            default=False,
+            action="store_true",
+            help="Enable extracting main text contents from webpage",
+        )
+        self.add_argument(
+            "-o",
+            "--overwrite_query_html",
+            default=False,
+            action="store_true",
+            help="Overwrite HTML file of query results",
+        )
+        self.add_argument(
+            "-w",
+            "--overwrite_webpage_html",
+            default=False,
+            action="store_true",
+            help="Overwrite HTML files of webpages from query results",
+        )
+
+        self.args = self.parse_args()
+
+

webscout/__init__.py

@@ -9,6 +9,7 @@ from .webscout_search import WEBS
 from .webscout_search_async import AsyncWEBS
 from .version import __version__
 from .DWEBS import DeepWEBS
+from .offlineAI import GPT4ALL
 __all__ = ["WEBS", "AsyncWEBS", "__version__", "cli"]
 
 logging.getLogger("webscout").addHandler(logging.NullHandler())

webscout/offlineAI.py

@@ -0,0 +1,206 @@
+from webscout.AIutel import Optimizers
+from webscout.AIutel import Conversation
+from webscout.AIutel import AwesomePrompts
+from webscout.AIbase import Provider
+from gpt4all import GPT4All
+from gpt4all.gpt4all import empty_chat_session
+from gpt4all.gpt4all import append_extension_if_missing
+
+
+import logging
+
+my_logger = logging.getLogger("gpt4all")
+my_logger.setLevel(logging.CRITICAL)
+
+
+class GPT4ALL(Provider):
+    def __init__(
+        self,
+        model: str,
+        is_conversation: bool = True,
+        max_tokens: int = 800,
+        temperature: float = 0.7,
+        presence_penalty: int = 0,
+        frequency_penalty: int = 1.18,
+        top_p: float = 0.4,
+        intro: str = None,
+        filepath: str = None,
+        update_file: bool = True,
+        history_offset: int = 10250,
+        act: str = None,
+    ):
+        """Instantiates GPT4ALL
+
+        Args:
+            model (str, optional): Path to LLM model (.gguf or .bin).
+            is_conversation (bool, optional): Flag for chatting conversationally. Defaults to True.
+            max_tokens (int, optional): Maximum number of tokens to be generated upon completion. Defaults to 800.
+            temperature (float, optional): Charge of the generated text's randomness. Defaults to 0.7.
+            presence_penalty (int, optional): Chances of topic being repeated. Defaults to 0.
+            frequency_penalty (int, optional): Chances of word being repeated. Defaults to 1.18.
+            top_p (float, optional): Sampling threshold during inference time. Defaults to 0.4.
+            intro (str, optional): Conversation introductory prompt. Defaults to None.
+            filepath (str, optional): Path to file containing conversation history. Defaults to None.
+            update_file (bool, optional): Add new prompts and responses to the file. Defaults to True.
+            history_offset (int, optional): Limit conversation history to this number of last texts. Defaults to 10250.
+            act (str|int, optional): Awesome prompt key or index. (Used as intro). Defaults to None.
+        """
+        self.is_conversation = is_conversation
+        self.max_tokens_to_sample = max_tokens
+        self.model = model
+        self.temperature = temperature
+        self.presence_penalty = presence_penalty
+        self.frequency_penalty = frequency_penalty
+        self.top_p = top_p
+        self.last_response = {}
+
+        self.__available_optimizers = (
+            method
+            for method in dir(Optimizers)
+            if callable(getattr(Optimizers, method)) and not method.startswith("__")
+        )
+        Conversation.intro = (
+            AwesomePrompts().get_act(
+                act, raise_not_found=True, default=None, case_insensitive=True
+            )
+            if act
+            else intro or Conversation.intro
+        )
+        self.conversation = Conversation(
+            is_conversation, self.max_tokens_to_sample, filepath, update_file
+        )
+        self.conversation.history_offset = history_offset
+
+        def get_model_name_path():
+            import os
+            from pathlib import Path
+
+            initial_model_path = Path(append_extension_if_missing(model))
+            if initial_model_path.exists:
+                if not initial_model_path.is_absolute():
+                    initial_model_path = Path(os.getcwd()) / initial_model_path
+                return os.path.split(initial_model_path.as_posix())
+            else:
+                raise FileNotFoundError(
+                    "File does not exist " + initial_model_path.as_posix()
+                )
+
+        model_dir, model_name = get_model_name_path()
+
+        self.gpt4all = GPT4All(
+            model_name=model_name,
+            model_path=model_dir,
+            allow_download=False,
+            verbose=False,
+        )
+
+    def ask(
+        self,
+        prompt: str,
+        stream: bool = False,
+        raw: bool = False,
+        optimizer: str = None,
+        conversationally: bool = False,
+    ) -> dict:
+        """Chat with AI
+
+        Args:
+            prompt (str): Prompt to be send.
+            stream (bool, optional): Flag for streaming response. Defaults to False.
+            raw (bool, optional): Stream back raw response as received. Defaults to False.
+            optimizer (str, optional): Prompt optimizer name - `[code, shell_command]`. Defaults to None.
+            conversationally (bool, optional): Chat conversationally when using optimizer. Defaults to False.
+        Returns:
+           dict : {}
+        ```json
+        {
+           "text" : "How may I help you today?"
+        }
+        ```
+        """
+        conversation_prompt = self.conversation.gen_complete_prompt(prompt)
+        if optimizer:
+            if optimizer in self.__available_optimizers:
+                conversation_prompt = getattr(Optimizers, optimizer)(
+                    conversation_prompt if conversationally else prompt
+                )
+            else:
+                raise Exception(
+                    f"Optimizer is not one of {self.__available_optimizers}"
+                )
+
+        def for_stream():
+            response = self.gpt4all.generate(
+                prompt=conversation_prompt,
+                max_tokens=self.max_tokens_to_sample,
+                temp=self.temperature,
+                top_p=self.top_p,
+                repeat_penalty=self.frequency_penalty,
+                streaming=True,
+            )
+
+            message_load: str = ""
+            for token in response:
+                message_load += token
+                resp: dict = dict(text=message_load)
+                yield token if raw else resp
+                self.last_response.update(resp)
+
+            self.conversation.update_chat_history(
+                prompt, self.get_message(self.last_response)
+            )
+            self.gpt4all.current_chat_session = empty_chat_session()
+
+        def for_non_stream():
+            for _ in for_stream():
+                pass
+            return self.last_response
+
+        return for_stream() if stream else for_non_stream()
+
+    def chat(
+        self,
+        prompt: str,
+        stream: bool = False,
+        optimizer: str = None,
+        conversationally: bool = False,
+    ) -> str:
+        """Generate response `str`
+        Args:
+            prompt (str): Prompt to be send.
+            stream (bool, optional): Flag for streaming response. Defaults to False.
+            optimizer (str, optional): Prompt optimizer name - `[code, shell_command]`. Defaults to None.
+            conversationally (bool, optional): Chat conversationally when using optimizer. Defaults to False.
+        Returns:
+            str: Response generated
+        """
+
+        def for_stream():
+            for response in self.ask(
+                prompt, True, optimizer=optimizer, conversationally=conversationally
+            ):
+                yield self.get_message(response)
+
+        def for_non_stream():
+            return self.get_message(
+                self.ask(
+                    prompt,
+                    False,
+                    optimizer=optimizer,
+                    conversationally=conversationally,
+                )
+            )
+
+        return for_stream() if stream else for_non_stream()
+
+    def get_message(self, response: dict) -> str:
+        """Retrieves message only from response
+
+        Args:
+            response (str): Response generated by `self.ask`
+
+        Returns:
+            str: Message extracted
+        """
+        assert isinstance(response, dict), "Response should be of dict data-type only"
+        return response["text"]

webscout/version.py

@@ -1,2 +1,2 @@
-__version__ = "1.2.2"
+__version__ = "1.2.4"
 

{webscout-1.2.2.dist-info → webscout-1.2.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: webscout
-Version: 1.2.2
+Version: 1.2.4
 Summary: Search for words, documents, images, videos, news, maps and text translation using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models
 Author: OEvortex
 Author-email: helpingai5@gmail.com
@@ -45,6 +45,8 @@ Requires-Dist: sse-starlette
 Requires-Dist: termcolor
 Requires-Dist: tiktoken
 Requires-Dist: tldextract
+Requires-Dist: gpt4all
+Requires-Dist: orjson
 Provides-Extra: dev
 Requires-Dist: ruff >=0.1.6 ; extra == 'dev'
 Requires-Dist: pytest >=7.4.2 ; extra == 'dev'
@@ -69,6 +71,7 @@ Also containes AI models that you can use
   - [Regions](#regions)
   - [DeepWEBS: Advanced Web Searches](#deepwebs-advanced-web-searches)
     - [Activating DeepWEBS](#activating-deepwebs)
+    - [Point to remember before using `DeepWEBS`](#point-to-remember-before-using-deepwebs)
     - [Usage Example](#usage-example)
   - [WEBS and AsyncWEBS classes](#webs-and-asyncwebs-classes)
   - [Exceptions](#exceptions)
@@ -91,6 +94,7 @@ Also containes AI models that you can use
     - [6. `BlackBox` - Search/chat With BlackBox](#6-blackbox---searchchat-with-blackbox)
     - [7. `PERPLEXITY` - Search With PERPLEXITY](#7-perplexity---search-with-perplexity)
     - [8. `OpenGPT` - chat With OPENGPT](#8-opengpt---chat-with-opengpt)
+    - [9. `GPT4ALL` - chat offline with Language models using gpt4all from webscout](#9-gpt4all---chat-offline-with-language-models-using-gpt4all-from-webscout)
   - [usage of special .LLM file from webscout (webscout.LLM)](#usage-of-special-llm-file-from-webscout-webscoutllm)
     - [`LLM`](#llm)
 
@@ -224,26 +228,33 @@ ___
 
 To utilize the `DeepWEBS` feature, you must first create an instance of the `DeepWEBS` . This is designed to be used independently of the `WEBS` , offering a focused approach to web searches.
 
+### Point to remember before using `DeepWEBS`
+As `DeepWEBS` is designed to extract relevant information directly from webpages and Search engine, It extracts html from webpages and saves them to folder named files in `DeepWEBS` that can be found at `C:\Users\Username\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\LocalCache\local-packages\Python311\site-packages\DeepWEBS`
+
 ### Usage Example
 
 Here's a basic example of how to use the `DeepWEBS` :
 ```python
 from webscout import DeepWEBS
 
-
 def perform_web_search(query):
-    D = DeepWEBS() 
-    item = D.DeepSearch(
-        queries=[query],  # Query to search
-        result_num=5,  # Number of search results
-        safe=True,  # Enable SafeSearch
-        types=["web"],  # Search type:  web
+    # Initialize the DeepWEBS class
+    D = DeepWEBS()
+    
+    # Set up the search parameters
+    search_params = D.DeepSearch(
+        queries=[query], # Query to search
+        result_num=5, # Number of search results
+        safe=True, # Enable SafeSearch
+        types=["web"], # Search type: web
         extract_webpage=True, # True for extracting webpages
         overwrite_query_html=False,
         overwrite_webpage_html=False,
     )
-    results = D.queries_to_search_results(item)
-
+    
+    # Execute the search and retrieve results
+    results = D.queries_to_search_results(search_params)
+    
     return results
 
 def print_search_results(results):
@@ -260,8 +271,13 @@ def print_search_results(results):
         print("No search results found.")
 
 def main():
+    # Prompt the user for a search query
     query = input("Enter your search query: ")
+    
+    # Perform the web search
     results = perform_web_search(query)
+    
+    # Print the search results
     print_search_results(results)
 
 if __name__ == "__main__":
@@ -592,6 +608,37 @@ prompt = "tell me about india"
 response_str = opengpt.chat(prompt)
 print(response_str)
 ```
+### 9. `GPT4ALL` - chat offline with Language models using gpt4all from webscout
+```python
+from webscout import GPT4ALL
+
+# Initialize the GPT4ALL class with your model path and other optional parameters
+gpt4all_instance = GPT4ALL(
+    model="path/to/your/model/file", # Replace with the actual path to your model file
+    is_conversation=True,
+    max_tokens=800,
+    temperature=0.7,
+    presence_penalty=0,
+    frequency_penalty=1.18,
+    top_p=0.4,
+    intro="Hello, how can I assist you today?",
+    filepath="path/to/conversation/history/file", # Optional, for conversation history
+    update_file=True,
+    history_offset=10250,
+    act=None # Optional, for using an awesome prompt as intro
+)
+
+# Generate a response from the AI model
+response = gpt4all_instance.chat(
+    prompt="What is the weather like today?",
+    stream=False, # Set to True if you want to stream the response
+    optimizer=None, # Optional, specify an optimizer if needed
+    conversationally=False # Set to True for conversationally generated responses
+)
+
+# Print the generated response
+print(response)
+```
 
 ## usage of special .LLM file from webscout (webscout.LLM)
 

{webscout-1.2.2.dist-info → webscout-1.2.4.dist-info}/RECORD

@@ -1,33 +1,34 @@
 DeepWEBS/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 DeepWEBS/documents/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-DeepWEBS/documents/query_results_extractor.py,sha256=ss6qplHctefwb88dePM6fdSiYBM4nob99PFpglfBlAI,2881
+DeepWEBS/documents/query_results_extractor.py,sha256=whd0NKLpcxW_6q3SkBOhMukr1K_c1PPYN92rf5EHRPM,4049
 DeepWEBS/documents/webpage_content_extractor.py,sha256=P4yHCkPTiBvMbORd8SKVt64rQFPJuj3iixcQoRU34Lw,5272
 DeepWEBS/networks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 DeepWEBS/networks/filepath_converter.py,sha256=JKMBew1TYe4TVoGTqgTWerq2Pam49_9u9TVUFCTDQyk,3183
-DeepWEBS/networks/google_searcher.py,sha256=iO9IBtycTDzoOqy15RzUUcD9rzCbQiB0BJvebpo41s8,1738
+DeepWEBS/networks/google_searcher.py,sha256=-AdIpVkRgemsARnOt8WPkF2Id1baVlqDHyqX2qz8Aew,1966
 DeepWEBS/networks/network_configs.py,sha256=-Hb78_7SBx32h219FnU14qcHTvBdDUf_QAU6-RTL_e0,726
-DeepWEBS/networks/webpage_fetcher.py,sha256=DpxyuYJpvVlDO09JcSsPzCByjhw-b_ouy_40c5WaEOQ,3823
+DeepWEBS/networks/webpage_fetcher.py,sha256=d5paDTB3wa_w6YWmLV7RkpAj8Lh8ztuUuyfe8RuTjQg,3846
 DeepWEBS/utilsdw/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 DeepWEBS/utilsdw/enver.py,sha256=vstxg_5P3Rwo1en6oPcuc2SBiATJqxi4C7meGmw5w0M,1754
 DeepWEBS/utilsdw/logger.py,sha256=Z0nFUcEGyU8r28yKiIyvEtO26xxpmJgbvNToTfwZecc,8174
 webscout/AI.py,sha256=CwUCeGnNRL9STd5bAZSyIiLysorBMu065HrkY8UCzAQ,49618
 webscout/AIbase.py,sha256=vQi2ougu5bG-QdmoYmxCQsOg7KTEgG7EF6nZh5qqUGw,2343
 webscout/AIutel.py,sha256=cvsuw57hq3GirAiT-PjqwhAiLPf1urOzDb2szJ4bwmo,24124
-webscout/DWEBS.py,sha256=hHoQCOdQJGrPT7A7cXQCfColrUOGITKW5NWd4iJ8dMI,6343
+webscout/DWEBS.py,sha256=QT-7-dUgWhQ_H7EVZD53AVyXxyskoPMKCkFIpzkN56Q,7332
 webscout/HelpingAI.py,sha256=YeZw0zYVHMcBFFPNdd3_Ghpm9ebt_EScQjHO_IIs4lg,8103
 webscout/LLM.py,sha256=XByJPiATLA_57FBWKw18Xx_PGRCPOj-GJE96aQH1k2Y,3309
-webscout/__init__.py,sha256=wRx1a7Jo-Vs9VwCq1ib5VEf2OeKK4qbtK1DaWu7Oj_s,425
+webscout/__init__.py,sha256=auv4OtSXPzH_Bcocya1179UvX4CTLmUqVg3cVXszjaA,457
 webscout/__main__.py,sha256=ZtTRgsRjUi2JOvYFLF1ZCh55Sdoz94I-BS-TlJC7WDU,126
 webscout/cli.py,sha256=F888fdrFUQgczMBN4yMOSf6Nh-IbvkqpPhDsbnA2FtQ,17059
 webscout/exceptions.py,sha256=4AOO5wexeL96nvUS-badcckcwrPS7UpZyAgB9vknHZE,276
 webscout/models.py,sha256=5iQIdtedT18YuTZ3npoG7kLMwcrKwhQ7928dl_7qZW0,692
+webscout/offlineAI.py,sha256=ieF9fQU-bWFZz5aBAQ8ZNxaCj1O1mI_w5AaAM9E3e8Y,7607
 webscout/utils.py,sha256=c_98M4oqpb54pUun3fpGGlCerFD6ZHUbghyp5b7Mwgo,2605
-webscout/version.py,sha256=Jux9ej4Nrsn-ptJdJyWt9TN1AGVnychpYPsy66KzEHU,25
+webscout/version.py,sha256=w3Y48JpCJLB-DvbXBfEkRgyEnrQoRiXGnyHDTl9pG5M,25
 webscout/webscout_search.py,sha256=3_lli-hDb8_kCGwscK29xuUcOS833ROgpNhDzrxh0dk,3085
 webscout/webscout_search_async.py,sha256=Y5frH0k3hLqBCR-8dn7a_b7EvxdYxn6wHiKl3jWosE0,40670
-webscout-1.2.2.dist-info/LICENSE.md,sha256=mRVwJuT4SXC5O93BFdsfWBjlXjGn2Np90Zm5SocUzM0,3150
-webscout-1.2.2.dist-info/METADATA,sha256=cICkVd_iLzMB31F__L75233nwc-rDLHzeCXrv1XG9w0,21013
-webscout-1.2.2.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-webscout-1.2.2.dist-info/entry_points.txt,sha256=8-93eRslYrzTHs5E-6yFRJrve00C9q-SkXJD113jzRY,197
-webscout-1.2.2.dist-info/top_level.txt,sha256=OD5YKy6Y3hldL7SmuxsiEDxAG4LgdSSWwzYk22MF9fk,18
-webscout-1.2.2.dist-info/RECORD,,
+webscout-1.2.4.dist-info/LICENSE.md,sha256=mRVwJuT4SXC5O93BFdsfWBjlXjGn2Np90Zm5SocUzM0,3150
+webscout-1.2.4.dist-info/METADATA,sha256=Zh6yfh9n8U_C2QZUYkpluwAk04H7Hj2bcsyd0EHfP9w,23100
+webscout-1.2.4.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+webscout-1.2.4.dist-info/entry_points.txt,sha256=8-93eRslYrzTHs5E-6yFRJrve00C9q-SkXJD113jzRY,197
+webscout-1.2.4.dist-info/top_level.txt,sha256=OD5YKy6Y3hldL7SmuxsiEDxAG4LgdSSWwzYk22MF9fk,18
+webscout-1.2.4.dist-info/RECORD,,