webscout 3.4__py3-none-any.whl → 3.6__py3-none-any.whl

{webscout-3.4.dist-info → webscout-3.6.dist-info}/RECORD

@@ -1,21 +1,9 @@
-DeepWEBS/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-DeepWEBS/documents/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-DeepWEBS/documents/query_results_extractor.py,sha256=whd0NKLpcxW_6q3SkBOhMukr1K_c1PPYN92rf5EHRPM,4049
-DeepWEBS/documents/webpage_content_extractor.py,sha256=P4yHCkPTiBvMbORd8SKVt64rQFPJuj3iixcQoRU34Lw,5272
-DeepWEBS/networks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-DeepWEBS/networks/filepath_converter.py,sha256=JKMBew1TYe4TVoGTqgTWerq2Pam49_9u9TVUFCTDQyk,3183
-DeepWEBS/networks/google_searcher.py,sha256=-AdIpVkRgemsARnOt8WPkF2Id1baVlqDHyqX2qz8Aew,1966
-DeepWEBS/networks/network_configs.py,sha256=-Hb78_7SBx32h219FnU14qcHTvBdDUf_QAU6-RTL_e0,726
-DeepWEBS/networks/webpage_fetcher.py,sha256=vRB9T3o-nMgrMkG2NPHTDctNeXaPSKCmBXqu189h2ZI,3590
-DeepWEBS/utilsdw/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-DeepWEBS/utilsdw/enver.py,sha256=vpI7s4_o_VL9govSryOv-z1zYK3pTEW3-H9QNN8JYtc,2472
-DeepWEBS/utilsdw/logger.py,sha256=Z0nFUcEGyU8r28yKiIyvEtO26xxpmJgbvNToTfwZecc,8174
 webscout/AIauto.py,sha256=xPGr_Z0h27XXNh4Wiufjn9TksDOqxqlaGcLUYKNP55w,18246
 webscout/AIbase.py,sha256=GoHbN8r0gq2saYRZv6LA-Fr9Jlcjv80STKFXUq2ZeGU,4710
-webscout/AIutel.py,sha256=Ghe9w1gqnCZTwjc3NzXymSamY3nP0zEep4NiATW32Qk,33454
-webscout/DWEBS.py,sha256=QT-7-dUgWhQ_H7EVZD53AVyXxyskoPMKCkFIpzkN56Q,7332
+webscout/AIutel.py,sha256=MMfUvTQXYDtaFXsXtwKgv9V_qMK6WgOxdx7Wagdm2Lw,33542
+webscout/DWEBS.py,sha256=QLuT1IKu0lnwdl7W6c-ctBAO7Jj0Zk3PYm6-13BC7rU,25740
 webscout/LLM.py,sha256=LbGCZdJf8A5dwfoGS4tyy39tAh5BDdhMZP0ScKaaQfU,4184
-webscout/__init__.py,sha256=pOqM5UGmljQN4jTrL3kyrjMv66VuTPyzfNlXZra9KLQ,1872
+webscout/__init__.py,sha256=ugx2Z3KX710527ri6AWPRwme9HjuiQezAnwyltqzr4c,2038
 webscout/__main__.py,sha256=ZtTRgsRjUi2JOvYFLF1ZCh55Sdoz94I-BS-TlJC7WDU,126
 webscout/async_providers.py,sha256=holBv5SxanxVXc_92CBBaXHlB2IakB_fHnhyZaFjYF8,684
 webscout/cli.py,sha256=enw_dPTCG3sNC1TXt96XccnpRmF4Etr99nh-RbGYags,18784
@@ -27,14 +15,15 @@ webscout/transcriber.py,sha256=EddvTSq7dPJ42V3pQVnGuEiYQ7WjJ9uyeR9kMSxN7uY,20622
 webscout/utils.py,sha256=CxeXvp0rWIulUrEaPZMaNfg_tSuQLRSV8uuHA2chyKE,2603
 webscout/version.py,sha256=pTj22SSXb7rieyMXdGyEFmljJmZMa6FL_DaETjfeLwA,23
 webscout/voice.py,sha256=0QjXTHAQmCK07IDZXRc7JXem47cnPJH7u3X0sVP1-UQ,967
-webscout/webai.py,sha256=hnRfUI9AT3MgltP68bAmW5Tq4_aWcYytYeTFEsgS7u0,85991
+webscout/webai.py,sha256=qkvhYdyF5wNdmW4rNdH3RbfQxabEWlGvCyAk2SbH04k,86602
 webscout/webscout_search.py,sha256=lFAot1-Qil_YfXieeLakDVDEX8Ckcima4ueXdOYwiMc,42804
 webscout/webscout_search_async.py,sha256=dooKGwLm0cwTml55Vy6NHPPY-nymEqX2h8laX94Zg5A,14537
+webscout/websx_search.py,sha256=n-qVwiHozJEF-GFRPcAfh4k1d_tscTmDe1dNL-1ngcU,12094
 webscout/Local/__init__.py,sha256=RN6klpbabPGNX2YzPm_hdeUcQvieUwvJt22uAO2RKSM,238
 webscout/Local/_version.py,sha256=hC_EHWR519ZOsyRw9i6gXEfU5IAIR_B9d3THLVmkWXw,83
 webscout/Local/formats.py,sha256=BiZZSoN3e8S6-S-ykBL9ogSUs0vK11GaZ3ghc9U8GRk,18994
 webscout/Local/model.py,sha256=T_bzNNrxEyOyLyhp6fKwiuVBBkXC2a37LzJVCxFIxOU,30710
-webscout/Local/rawdog.py,sha256=LtA7bck2HyvWmovuaG86Iiquiz7XiMcxBlebo9IuGBY,35744
+webscout/Local/rawdog.py,sha256=ojY_O8Vb1KvR34OwWdfLgllgaAK_7HMf64ElMATvCXs,36689
 webscout/Local/samplers.py,sha256=qXwU4eLXER-2aCYzcJcTgA6BeFmi5GMpTDUX1C9pTN4,4372
 webscout/Local/thread.py,sha256=Lyf_N2CaGAn2usSWSiUXLPAgpWub8vUu_tgFgtnvZVA,27408
 webscout/Local/utils.py,sha256=CSt9IqHhVGk_nJEnKvSFbLhC5nNf01e0MtwpgMmF9pA,6197
@@ -43,26 +32,28 @@ webscout/Provider/Berlin4h.py,sha256=zMpmWmdFCbcE3UWB-F9xbbTWZTfx4GnjnRf6sDoaiC0
 webscout/Provider/Blackboxai.py,sha256=HUk0moEGsgGvidD1LF9tbfaKdx7bPnGU_SrYPdcfHU8,17182
 webscout/Provider/ChatGPTUK.py,sha256=qmuCb_a71GNE5LelOb5AKJUBndvj7soebiNey4VdDvE,8570
 webscout/Provider/Cohere.py,sha256=IXnRosYOaMAA65nvsKmN6ZkJGSdZFYQYBidzuNaCqX8,8711
-webscout/Provider/Deepseek.py,sha256=HKsC-ePLSPqcrQbafy-IzR0BNqId3LfiIEhk9j9oTs4,10285
+webscout/Provider/Deepinfra.py,sha256=kVnWARJdEtIeIsZwGw3POq8B2dO87bDcJso3uOeCeOA,18750
+webscout/Provider/Deepseek.py,sha256=pnOB44ObuOfAsoi_bUGUvha3tfwd0rTJ9rnX-14QkL4,10550
 webscout/Provider/Gemini.py,sha256=_4DHWvlWuNAmVHPwHB1RjmryjTZZCthLa6lvPEHLvkQ,8451
 webscout/Provider/Groq.py,sha256=QfgP3hKUcqq5vUA4Pzuu3HAgpJkKwLWNjjsnxtkCYd8,21094
 webscout/Provider/Koboldai.py,sha256=KwWx2yPlvT9BGx37iNvSbgzWkJ9I8kSOmeg7sL1hb0M,15806
 webscout/Provider/Leo.py,sha256=wbuDR-vFjLptfRC6yDlk74tINqNvCOzpISsK92lIgGg,19987
 webscout/Provider/Llama2.py,sha256=gVMotyiBaDSqliwuDtFefHoOBn9V5m5Ze_YVtV0trt8,17525
-webscout/Provider/OpenGPT.py,sha256=SJskNkUGNNb3zdZY50xokzW-rwcSlHw8EN6WVv70dg8,18890
+webscout/Provider/OpenGPT.py,sha256=ZymwLgNJSPlGZHW3msMlnRR7NxmALqJw9yuToqrRrhw,35515
 webscout/Provider/Openai.py,sha256=SjfVOwY94unVnXhvN0Fkome-q2-wi4mPJk_vCGq5Fjc,20617
 webscout/Provider/Perplexity.py,sha256=CPdKqkdlVejXDcf1uycNO4LPCVNUADSCetvyJEGepSw,8826
-webscout/Provider/Phind.py,sha256=NXiYNRs8h_6c3AGOUqFrvN01odBIQ_psSUBPaHiAUoE,19907
+webscout/Provider/Phind.py,sha256=bkgKVtggRJSbJAG1tXviW9BqDvcgqPBlSr88Q6rlFHw,39226
 webscout/Provider/Poe.py,sha256=ObUxa-Fa2Dq7sJcV0hc65m09StS9uWsB2-bR2rSjXDY,7510
 webscout/Provider/Reka.py,sha256=F0ZXENkhARprj5biK3mRxwiuPH0BW3ga7EWsi8agbtE,8917
 webscout/Provider/ThinkAnyAI.py,sha256=_qFjj0djxxrranyEY33w14oizyRjzlVwMv_hzvVtwNc,11616
+webscout/Provider/VTLchat.py,sha256=_sErGr-wOi16ZAfiGOo0bPsAEMkjzzwreEsIqjIZMIU,10041
 webscout/Provider/Xjai.py,sha256=BIlk2ouz9Kh_0Gg9hPvTqhI7XtcmWdg5vHSX_4uGrIs,9039
 webscout/Provider/Yepchat.py,sha256=2Eit-A7w1ph1GQKNQuur_yaDzI64r0yBGxCIjDefJxQ,19875
 webscout/Provider/Youchat.py,sha256=UVGBuGSjv4uRibn1xflmCjYcfrRTKnDvX3adhag6T98,7976
-webscout/Provider/__init__.py,sha256=nmZYPpXyp8s0xn4UO9IMhkV7-RfGqMdOa3CRmG0uuTg,1510
-webscout-3.4.dist-info/LICENSE.md,sha256=9P0imsudI7MEvZe2pOcg8rKBn6E5FGHQ-riYozZI-Bk,2942
-webscout-3.4.dist-info/METADATA,sha256=t8Hfgd5KshA4OOid1ovzYd83p890DBmZWGFWFn1pTFE,67227
-webscout-3.4.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-webscout-3.4.dist-info/entry_points.txt,sha256=Hh4YIIjvkqB9SVxZ2ri4DZUkgEu_WF_5_r_nZDIvfG8,73
-webscout-3.4.dist-info/top_level.txt,sha256=OD5YKy6Y3hldL7SmuxsiEDxAG4LgdSSWwzYk22MF9fk,18
-webscout-3.4.dist-info/RECORD,,
+webscout/Provider/__init__.py,sha256=RaMdtYv7eQJ2vB8jXUHrkfNbx2DgRjbwc6DI40cOH1A,1809
+webscout-3.6.dist-info/LICENSE.md,sha256=9P0imsudI7MEvZe2pOcg8rKBn6E5FGHQ-riYozZI-Bk,2942
+webscout-3.6.dist-info/METADATA,sha256=YhKU0lcCGrDmUA-L1wy7ETzzmkWmbY5MvEfZg_6GlcU,67341
+webscout-3.6.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
+webscout-3.6.dist-info/entry_points.txt,sha256=Hh4YIIjvkqB9SVxZ2ri4DZUkgEu_WF_5_r_nZDIvfG8,73
+webscout-3.6.dist-info/top_level.txt,sha256=nYIw7OKBQDr_Z33IzZUKidRD3zQEo8jOJYkMVMeN334,9
+webscout-3.6.dist-info/RECORD,,

{webscout-3.4.dist-info → webscout-3.6.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.42.0)
+Generator: setuptools (70.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{webscout-3.4.dist-info → webscout-3.6.dist-info}/top_level.txt

@@ -1,2 +1 @@
-DeepWEBS
 webscout

DeepWEBS/documents/query_results_extractor.py

@@ -1,99 +0,0 @@
-from bs4 import BeautifulSoup
-from pathlib import Path
-from DeepWEBS.utilsdw.logger import logger
-
-class QueryResultsExtractor:
-    def __init__(self) -> None:
-        self.query_results = []
-        self.related_questions = []
-
-    def load_html(self, html_path):
-        try:
-            with open(html_path, "r", encoding="utf-8") as f:
-                html = f.read()
-            self.soup = BeautifulSoup(html, "html.parser")
-        except FileNotFoundError:
-            logger.error(f"File not found: {html_path}")
-        except Exception as e:
-            logger.error(f"Error loading HTML: {e}")
-
-    def extract_query_results(self):
-        try:
-            self.query = self.soup.find("textarea").text.strip()
-            query_result_elements = self.soup.find_all("div", class_="g")
-            for idx, result in enumerate(query_result_elements):
-                try:
-                    site = result.find("cite").find_previous("span").text.strip()
-                    url = result.find("a")["href"]
-                    title = result.find("h3").text.strip()
-                    abstract_element_conditions = [
-                        {"data-sncf": "1"},
-                        {"class_": "ITZIwc"},
-                    ]
-                    for condition in abstract_element_conditions:
-                        abstract_element = result.find("div", condition)
-                        if abstract_element is not None:
-                            abstract = abstract_element.text.strip()
-                            break
-                    else:
-                        abstract = ""
-                    logger.mesg(
-                        f"{title}\n"
-                        f" - {site}\n"
-                        f" - {url}\n"
-                        f" - {abstract}\n"
-                        f"\n"
-                    )
-                    self.query_results.append(
-                        {
-                            "title": title,
-                            "site": site,
-                            "url": url,
-                            "abstract": abstract,
-                            "index": idx,
-                            "type": "web",
-                        }
-                    )
-                except Exception as e:
-                    logger.error(f"Error extracting query result: {e}")
-            logger.success(f"- {len(query_result_elements)} query results")
-        except Exception as e:
-            logger.error(f"Error extracting query results: {e}")
-
-    def extract_related_questions(self):
-        try:
-            related_question_elements = self.soup.find_all(
-                "div", class_="related-question-pair"
-            )
-            for question_element in related_question_elements:
-                try:
-                    question = question_element.find("span").text.strip()
-                    print(question)
-                    self.related_questions.append(question)
-                except Exception as e:
-                    logger.error(f"Error extracting related question: {e}")
-            logger.success(f"- {len(self.related_questions)} related questions")
-        except Exception as e:
-            logger.error(f"Error extracting related questions: {e}")
-
-    def extract(self, html_path):
-        self.load_html(html_path)
-        self.extract_query_results()
-        self.extract_related_questions()
-        self.search_results = {
-            "query": self.query,
-            "query_results": self.query_results,
-            "related_questions": self.related_questions,
-        }
-        return self.search_results
-
-
-if __name__ == "__main__":
-    html_path_root = Path(__file__).parents[1] / "files"
-    html_filename = "python_tutorials"
-    html_path = html_path_root / f"{html_filename}.html"
-    extractor = QueryResultsExtractor()
-    try:
-        extractor.extract(html_path)
-    except Exception as e:
-        logger.error(f"Error in main function: {e}")

DeepWEBS/documents/webpage_content_extractor.py

@@ -1,145 +0,0 @@
-import concurrent.futures
-import re
-from pathlib import Path
-from pprint import pprint
-from bs4 import BeautifulSoup
-from tiktoken import get_encoding as tiktoken_get_encoding
-from DeepWEBS.utilsdw.logger import logger
-from markdownify import markdownify
-from DeepWEBS.networks.network_configs import IGNORE_TAGS, IGNORE_CLASSES
-from termcolor import colored
-
-
-class WebpageContentExtractor:
-    def __init__(self):
-        self.tokenizer = tiktoken_get_encoding("cl100k_base")
-
-    def count_tokens(self, text):
-        tokens = self.tokenizer.encode(text)
-        token_count = len(tokens)
-        return token_count
-
-    def html_to_markdown(self, html_str, ignore_links=True):
-        if ignore_links:
-            markdown_str = markdownify(html_str, strip="a")
-        else:
-            markdown_str = markdownify(html_str)
-        markdown_str = re.sub(r"\n{3,}", "\n\n", markdown_str)
-
-        self.markdown_token_count = self.count_tokens(markdown_str)
-        logger.mesg(f'- Tokens: {colored(self.markdown_token_count,"light_green")}')
-
-        self.markdown_str = markdown_str
-
-        return self.markdown_str
-
-    def remove_elements_from_html(self, html_str):
-        soup = BeautifulSoup(html_str, "html.parser")
-        ignore_classes_with_parentheses = [f"({word})" for word in IGNORE_CLASSES]
-        ignore_classes_pattern = f'{"|".join(ignore_classes_with_parentheses)}'
-        removed_element_counts = 0
-        for element in soup.find_all():
-            class_str = ""
-            id_str = ""
-            try:
-                class_attr = element.get("class", [])
-                if class_attr:
-                    class_str = " ".join(list(class_attr))
-                if id_str:
-                    class_str = f"{class_str} {id_str}"
-            except:
-                pass
-
-            try:
-                id_str = element.get("id", "")
-            except:
-                pass
-
-            if (
-                (not element.text.strip())
-                or (element.name in IGNORE_TAGS)
-                or (re.search(ignore_classes_pattern, class_str, flags=re.IGNORECASE))
-                or (re.search(ignore_classes_pattern, id_str, flags=re.IGNORECASE))
-            ):
-                element.decompose()
-                removed_element_counts += 1
-
-        logger.mesg(
-            f"- Elements: "
-            f'{colored(len(soup.find_all()),"light_green")} / {colored(removed_element_counts,"light_red")}'
-        )
-
-        html_str = str(soup)
-        self.html_str = html_str
-
-        return self.html_str
-
-    def extract(self, html_path):
-        logger.note(f"Extracting content from: {html_path}")
-
-        if not Path(html_path).exists():
-            logger.warn(f"File not found: {html_path}")
-            return ""
-
-        encodings = ["utf-8", "latin-1"]
-        for encoding in encodings:
-            try:
-                with open(html_path, "r", encoding=encoding, errors="ignore") as rf:
-                    html_str = rf.read()
-                break
-            except UnicodeDecodeError:
-                pass
-        else:
-            logger.warn(f"No matching encodings: {html_path}")
-            return ""
-
-        html_str = self.remove_elements_from_html(html_str)
-        markdown_str = self.html_to_markdown(html_str)
-        return markdown_str
-
-
-class BatchWebpageContentExtractor:
-    def __init__(self) -> None:
-        self.html_path_and_extracted_content_list = []
-        self.done_count = 0
-
-    def extract_single_html(self, html_path):
-        webpage_content_extractor = WebpageContentExtractor()
-        extracted_content = webpage_content_extractor.extract(html_path)
-        self.html_path_and_extracted_content_list.append(
-            {"html_path": html_path, "extracted_content": extracted_content}
-        )
-        self.done_count += 1
-        logger.success(
-            f"> [{self.done_count}/{self.total_count}] Extracted: {html_path}"
-        )
-
-    def extract(self, html_paths):
-        self.html_path = html_paths
-        self.total_count = len(self.html_path)
-        with concurrent.futures.ThreadPoolExecutor() as executor:
-            futures = [
-                executor.submit(self.extract_single_html, html_path)
-                for html_path in self.html_path
-            ]
-            for idx, future in enumerate(concurrent.futures.as_completed(futures)):
-                result = future.result()
-
-        return self.html_path_and_extracted_content_list
-
-
-if __name__ == "__main__":
-    html_root = Path(__file__).parents[1] / "files" / "urls" / "python tutorials"
-    html_paths = [
-        html_root / html_filename
-        for html_filename in [
-            "docs.python.org_zh-cn_3_tutorial_interpreter.html",
-            "stackoverflow.com_questions_295135_turn-a-string-into-a-valid-filename.html",
-            "www.liaoxuefeng.com_wiki_1016959663602400_1017495723838528.html",
-        ]
-    ]
-    batch_webpage_content_extractor = BatchWebpageContentExtractor()
-    html_path_and_extracted_content_list = batch_webpage_content_extractor.extract(
-        html_paths
-    )
-    # pprint(html_path_and_extracted_content_list)

DeepWEBS/networks/filepath_converter.py

@@ -1,109 +0,0 @@
-import platform
-import re
-from pathlib import Path
-from urllib.parse import quote, unquote
-
-
-# What characters are forbidden in Windows and Linux directory names?
-#   https://stackoverflow.com/questions/1976007/what-characters-are-forbidden-in-windows-and-linux-directory-names
-
-INVALID_FILE_PATH_CHARS = [
-    "\\",
-    "/",
-    ":",
-    "*",
-    "?",
-    '"',
-    "<",
-    ">",
-    "|",
-    "\n",
-    "\t",
-    "\r",
-    *[chr(i) for i in range(32)],
-]
-
-WINDOWS_INVALID_FILE_PATH_NAMES = [
-    "con",
-    "prn",
-    "aux",
-    "nul",
-    *[f"com{i+1}" for i in range(10)],
-    *[f"lpt{i+1}" for i in range(10)],
-]
-
-
-class FilepathConverter:
-    def __init__(self, parent: str = None):
-        self.output_root = Path(__file__).parents[1] / "files"
-        self.parent = parent
-
-    def preprocess(self, input_string):
-        return input_string
-
-    def validate(self, input_string):
-        if not input_string:
-            return input_string
-        filename = input_string
-        for char in INVALID_FILE_PATH_CHARS:
-            filename = filename.replace(char, "_")
-        if platform.system() == "Windows":
-            filename_base = filename.split(".")[0]
-            if filename_base.lower() in WINDOWS_INVALID_FILE_PATH_NAMES:
-                filename_base = filename_base + "_"
-                filename = ".".join([filename_base, *filename.split(".")[1:]])
-        return filename
-
-    def append_extension(self, filename, accept_exts=[".html", ".htm"], ext=".html"):
-        if ext:
-            filename_ext = "." + filename.split(".")[-1]
-            if filename_ext.lower() not in accept_exts:
-                filename += ext
-        return filename
-
-    def convert(self, input_string, parent=None):
-        filename = self.preprocess(input_string)
-        filename = self.validate(filename)
-        filename = self.append_extension(filename)
-
-        parent = parent or self.parent
-        parent = self.validate(parent)
-        if parent:
-            filepath = self.output_root / parent / filename
-        else:
-            filepath = self.output_root / filename
-
-        self.filename = filename
-        self.filepath = filepath
-
-        return self.filepath
-
-
-class UrlToFilepathConverter(FilepathConverter):
-    def __init__(self, parent: str = None):
-        super().__init__(parent)
-        self.output_root = self.output_root / "urls"
-
-    def preprocess(self, url):
-        filename = unquote(url.split("//")[1])
-        return filename
-
-
-class QueryToFilepathConverter(FilepathConverter):
-    def __init__(self, parent: str = None):
-        super().__init__(parent)
-        self.output_root = self.output_root / "queries"
-
-
-if __name__ == "__main__":
-    query = "python"
-    query_converter = QueryToFilepathConverter()
-    print(query_converter.convert(query))
-
-    # url = "https://trafilatura.readthedocs.io/en/latest/quickstart.html"
-    url = (
-        "https://stackoverflow.com/questions/295135/turn-a-string-into-a-valid-filename"
-    )
-
-    url_converter = UrlToFilepathConverter(parent=query)
-    print(url_converter.convert(url))

DeepWEBS/networks/google_searcher.py

@@ -1,52 +0,0 @@
-import requests
-from pathlib import Path
-from typing import Optional
-import random
-from DeepWEBS.utilsdw.enver import enver
-from DeepWEBS.utilsdw.logger import logger
-from DeepWEBS.networks.filepath_converter import QueryToFilepathConverter
-from DeepWEBS.networks.network_configs import REQUESTS_HEADERS
-
-class GoogleSearcher:
-    def __init__(self):
-        self.url = "https://www.google.com/search"
-        self.enver = enver
-        self.enver.set_envs(proxies=True)
-        self.filepath_converter = QueryToFilepathConverter()
-
-    def send_request(self, query: str, result_num: int = 10, safe: bool = False) -> requests.Response:
-        params = {
-            "q": query,
-            "num": result_num,
-        }
-        response = requests.get(
-            self.url,
-            headers=REQUESTS_HEADERS,
-            params=params,
-            proxies=self.enver.requests_proxies,
-        )
-        response.raise_for_status()  # Raise an exception for non-2xx status codes
-        return response
-
-    def save_response(self, response: requests.Response, html_path: Path) -> None:
-        html_path.parent.mkdir(parents=True, exist_ok=True)
-        logger.note(f"Saving to: [{html_path}]")
-        with html_path.open("wb") as wf:
-            wf.write(response.content)
-
-    def search(self, query: str, result_num: int = 10, safe: bool = False, overwrite: bool = False) -> Path:
-        html_path = self.filepath_converter.convert(query)
-        logger.note(f"Searching: [{query}]")
-
-        if html_path.exists() and not overwrite:
-            logger.success(f"HTML existed: {html_path}")
-        else:
-            response = self.send_request(query, result_num, safe)
-            self.save_response(response, html_path)
-
-        return html_path
-
-if __name__ == "__main__":
-    searcher = GoogleSearcher()
-    html_path = searcher.search("python tutorials")
-    print(f"HTML file saved at: {html_path}")

DeepWEBS/networks/network_configs.py

@@ -1,30 +0,0 @@
-IGNORE_TAGS = ["script", "style", "button"]
-IGNORE_CLASSES = [
-    # common
-    "sidebar",
-    "footer",
-    "related",
-    "comment",
-    "topbar",
-    "offcanvas",
-    "navbar",
-    # 163.com
-    "post_(top)|(side)|(recommends)|(crumb)|(statement)|(next)|(jubao)",
-    "ntes\-.*nav",
-    "nav\-bottom",
-    # wikipedia.org
-    "language\-list",
-    "vector\-(header)|(column)|(sticky\-pinned)|(dropdown\-content)",
-    "navbox",
-    "catlinks",
-]
-
-IGNORE_HOSTS = [
-    "weibo.com",
-    "hymson.com",
-    "yahoo.com",
-]
-
-REQUESTS_HEADERS = {
-    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 Edg/111.0.1661.62",
-}

DeepWEBS/networks/webpage_fetcher.py

@@ -1,95 +0,0 @@
-import concurrent.futures
-import random
-import requests
-import tldextract
-from pathlib import Path
-from typing import List, Tuple, Dict
-
-from DeepWEBS.utilsdw.enver import enver
-from DeepWEBS.utilsdw.logger import logger
-from DeepWEBS.networks.filepath_converter import UrlToFilepathConverter
-from DeepWEBS.networks.network_configs import IGNORE_HOSTS, REQUESTS_HEADERS
-
-class WebpageFetcher:
-    def __init__(self):
-        self.enver = enver
-        self.enver.set_envs(proxies=True)
-        self.filepath_converter = UrlToFilepathConverter()
-
-    def is_ignored_host(self, url: str) -> bool:
-        host = tldextract.extract(url).registered_domain
-        return host in IGNORE_HOSTS
-
-    def send_request(self, url: str) -> requests.Response:
-        try:
-            user_agent = random.choice(REQUESTS_HEADERS["User-Agent"])
-            response = requests.get(
-                url=url,
-                headers={"User-Agent": user_agent},
-                proxies=self.enver.requests_proxies,
-                timeout=15,
-            )
-            response.raise_for_status()
-            return response
-        except requests.exceptions.RequestException as e:
-            logger.warn(f"Failed to fetch: [{url}] | {e}")
-            return None
-
-    def save_response(self, response: requests.Response, html_path: Path) -> None:
-        if response is None:
-            return
-
-        html_path.parent.mkdir(parents=True, exist_ok=True)
-        logger.success(f"Saving to: [{html_path}]")
-        with html_path.open("wb") as wf:
-            wf.write(response.content)
-
-    def fetch(self, url: str, overwrite: bool = False, output_parent: str = None) -> Path:
-        logger.note(f"Fetching: [{url}]")
-        html_path = self.filepath_converter.convert(url, parent=output_parent)
-
-        if self.is_ignored_host(url):
-            logger.warn(f"Ignored host: [{tldextract.extract(url).registered_domain}]")
-            return html_path
-
-        if html_path.exists() and not overwrite:
-            logger.success(f"HTML existed: [{html_path}]")
-        else:
-            response = self.send_request(url)
-            self.save_response(response, html_path)
-
-        return html_path
-
-class BatchWebpageFetcher:
-    def __init__(self):
-        self.done_count = 0
-        self.total_count = 0
-        self.url_and_html_path_list: List[Dict[str, str]] = []
-
-    def fetch_single_webpage(self, url: str, overwrite: bool = False, output_parent: str = None) -> Tuple[str, Path]:
-        webpage_fetcher = WebpageFetcher()
-        html_path = webpage_fetcher.fetch(url, overwrite, output_parent)
-        self.url_and_html_path_list.append({"url": url, "html_path": str(html_path)})
-        self.done_count += 1
-        logger.success(f"> [{self.done_count}/{self.total_count}] Fetched: {url}")
-        return url, html_path
-
-    def fetch(self, urls: List[str], overwrite: bool = False, output_parent: str = None) -> List[Dict[str, str]]:
-        self.urls = urls
-        self.total_count = len(self.urls)
-
-        with concurrent.futures.ProcessPoolExecutor() as executor:
-            futures = [
-                executor.submit(WebpageFetcher().fetch, url, overwrite, output_parent)
-                for url in urls
-            ]
-            concurrent.futures.wait(futures)
-
-        self.url_and_html_path_list = [
-            {"url": future.result().url, "html_path": str(future.result().html_path)}
-            for future in futures
-        ]
-
-        return self.url_and_html_path_list
-
-