py2ls 0.2.5.10__py3-none-any.whl → 0.2.5.14__py3-none-any.whl

py2ls/netfinder.py

@@ -1,7 +1,8 @@
-from bs4 import BeautifulSoup
+from bs4 import BeautifulSoup, NavigableString
 import scrapy
 import requests
 import os
+import chardet
 import pandas as pd
 import logging
 import json
@@ -116,8 +117,8 @@ def extract_text_from_content(
 
     def extract_text(element):
         texts = ""
-        if isinstance(element, str) and element.strip():
-            texts += element.strip()
+        if isinstance(element, NavigableString) and element.strip():  
+            texts += element.strip() + " " 
         elif hasattr(element, "children"):
             for child in element.children:
                 texts += extract_text(child)
@@ -172,6 +173,8 @@ def extract_text_from_content(
                 texts = ""
                 for tag in result_set:
                     texts = texts + " " + extract_text(tag) + " \n"
+                    # texts = texts + " " + tag.get_text(" ", strip=True)+ " \n"
+                    
                 text_list = [tx.strip() for tx in texts.split(" \n") if tx.strip()]
                 return text_list
             else:
@@ -332,7 +335,6 @@ def parse_cookies(cookies_str):
             cookies_dict[cookie_name] = cookie_value
 
     return cookies_dict
-
 def fetch_scrapy(
     url,
     parser="html.parser",
@@ -358,23 +360,60 @@ def fetch_scrapy(
     from scrapy.signalmanager import dispatcher
     from scrapy import signals
     from twisted.internet import reactor, defer
+    from twisted.internet.error import ReactorNotRestartable
     import scrapy
+    import logging
+
+    # Disable Scrapy's excessive logging
+    logging.getLogger('scrapy').setLevel(logging.WARNING)
+    logging.getLogger('twisted').setLevel(logging.WARNING)
 
     # Container for scraped content
     content = []
 
+    # Define the spider class inside the function
+    class FetchSpider(scrapy.Spider):
+        name = "fetch_spider"
+        
+        def __init__(self, url=None, parser=None, cookies=None, headers=None, *args, **kwargs):
+            super(FetchSpider, self).__init__(*args, **kwargs)
+            self.start_urls = [url]
+            self.parser = parser
+            self.cookies = cookies
+            self.headers = headers
+
+        def start_requests(self):
+            for url in self.start_urls:
+                yield scrapy.Request(
+                    url,
+                    cookies=self.cookies,
+                    headers=self.headers,
+                    callback=self.parse
+                )
+
+        def parse(self, response):
+            from bs4 import BeautifulSoup
+            soup = BeautifulSoup(response.text, self.parser)
+            yield {
+                "content": soup,
+                "url": response.url,
+                "status": response.status
+            }
+
     # Callback function for item scraped signal
     def handle_item(item, response, spider):
-        content.append(item["content"])
+        content.append(item)
 
     # Scrapy settings
     process_settings = settings or get_project_settings()
     process_settings.update(
         {
-            "USER_AGENT": "CustomUserAgent/1.0",  # Use a custom user agent
-            "DOWNLOAD_DELAY": 1,  # Prevent overloading servers
+            "USER_AGENT": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+            "DOWNLOAD_DELAY": 1,
             "COOKIES_ENABLED": bool(cookies),
-            "LOG_LEVEL": "ERROR",  # Minimize log verbosity
+            "LOG_LEVEL": "ERROR",
+            "RETRY_ENABLED": False,
+            "HTTPERROR_ALLOW_ALL": True,
         }
     )
 
@@ -394,26 +433,292 @@ def fetch_scrapy(
         )
         reactor.stop()
 
-    # Start the reactor if not already running
-    if not reactor.running:
-        crawl()
-        reactor.run()  # Blocks until the crawl finishes
-    else:
-        # Run the crawl if the reactor is already running
-        d = crawl()
-        d.addBoth(lambda _: reactor.stop())
+    # Handle reactor execution
+    try:
+        if not reactor.running:
+            crawl()
+            reactor.run(installSignalHandlers=0)
+        else:
+            # This case is problematic - reactor can't be restarted
+            raise RuntimeError("Reactor already running. Cannot run multiple crawls in same process.")
+    except ReactorNotRestartable:
+        raise RuntimeError("Scrapy reactor cannot be restarted. Create a new process for additional crawls.")
 
     # Return the first scraped content or None if empty
     return content[0] if content else None
-
-
+ 
+# def fetch_all(
+#     url,
+#     parser="lxml",
+#     driver="request",  # request or selenium
+#     by=By.TAG_NAME,
+#     timeout=10,
+#     retry=2,
+#     wait=0,
+#     wait_until=None,
+#     wait_until_kind=None,
+#     scroll_try=3,
+#     login_url=None,
+#     username=None,
+#     password=None,
+#     username_field="username",
+#     password_field="password",
+#     submit_field="submit",
+#     username_by=By.NAME,
+#     password_by=By.NAME,
+#     submit_by=By.NAME,
+#     # capability='eager', # eager or none
+#     proxy=None,  # Add proxy parameter
+#     javascript=True,  # Add JavaScript option
+#     disable_images=False,  # Add option to disable images
+#     iframe_name=None,
+#     login_dict=None,
+#     cookies=None,  # Add cookies parameter
+# ):  # Add option to handle iframe):  # lxml is faster, # parser="html.parser"
+#     try:
+#         # # Generate a random user-agent string
+#         # response = requests.get(url)
+#         # # get cookies
+#         # cookie=dict_from_cookiejar(response.cookies)
+#         # # get token from cookies
+#         # scrf_token=re.findall(r'csrf-token=(.*?);', response.headers.get('Set-Cookie'))[0]
+#         # headers = {"User-Agent": user_agent(), "X-CSRF-Token":scrf_token}
+
+#         headers = {"User-Agent": user_agent()}
+#         if isinstance(driver, int):
+#             drivers=["request", "selenium","scrapy"]
+#             driver=drivers[driver]
+#         if "req" in driver.lower():
+#             # response = requests.get(
+#             #     url, headers=headers, proxies=proxies_glob, timeout=30, stream=True
+#             # )
+
+#             # Handle cookies for requests
+#             if cookies:
+#                 from requests.cookies import RequestsCookieJar
+#                 cookie_jar = RequestsCookieJar()
+#                 if isinstance(cookies, str):
+#                     cookies=parse_cookies(cookies)
+#                 for cookie_name, cookie_value in cookies.items():
+#                     cookie_jar.set(cookie_name, cookie_value)
+#                 response = requests.get(
+#                     url, headers=headers, cookies=cookie_jar, timeout=timeout, stream=True
+#                 )
+#             else:
+#                 response = requests.get(url, headers=headers, timeout=timeout, stream=True)
+
+#             # If the response is a redirect, follow it
+#             while response.is_redirect:
+#                 logger.info(f"Redirecting to: {response.headers['Location']}")
+#                 response = requests.get(
+#                     response.headers["Location"],
+#                     headers=headers,
+#                     proxies=proxies_glob,
+#                     timeout=timeout,
+#                     stream=True,
+#                 )
+#             # Check for a 403 error
+#             if response.status_code == 403:
+#                 logger.warning("403 Forbidden error. Retrying...")
+#                 # Retry the request after a short delay
+#                 time.sleep(random.uniform(1, 3))
+#                 response = requests.get(
+#                     url, headers=headers, proxies=proxies_glob, timeout=timeout, stream=True
+#                 )
+#                 # Raise an error if retry also fails
+#                 response.raise_for_status()
+
+#             # Raise an error for other HTTP status codes
+#             response.raise_for_status()
+
+#             # Get the content type
+#             content_type = (
+#                 response.headers.get("content-type", "").split(";")[0].lower()
+#             )
+#             if response.encoding:
+#                 content = response.content.decode(response.encoding)
+#             else:
+#                 content = None
+#             # logger.info(f"Content type: {content_type}")
+
+#             # Check if content type is supported
+#             if content_type in CONTENT_PARSERS and content:
+#                 return content_type, CONTENT_PARSERS[content_type](content, parser)
+#             else:
+#                 logger.warning("Unsupported content type")
+#                 return None, None
+#         elif "se" in driver.lower():
+#             import random
+#             from selenium import webdriver
+#             from selenium.webdriver.chrome.service import Service
+#             from selenium.webdriver.common.by import By
+#             from selenium.webdriver.chrome.options import Options
+#             from selenium.webdriver.support.ui import WebDriverWait
+#             from selenium.webdriver.support import expected_conditions as EC
+#             from webdriver_manager.chrome import ChromeDriverManager
+#             from selenium.webdriver.common.desired_capabilities import (
+#                 DesiredCapabilities,
+#             )
+
+#             chrome_options = Options()
+#             chrome_options.add_argument("--headless")
+#             chrome_options.add_argument("--no-sandbox")
+#             chrome_options.add_argument("--disable-dev-shm-usage")
+#             chrome_options.add_argument(f"user-agent={user_agent()}")
+#             if proxy:
+#                 chrome_options.add_argument(f"--proxy-server={proxy}")
+#             if disable_images:
+#                 prefs = {"profile.managed_default_content_settings.images": 2}
+#                 chrome_options.add_experimental_option("prefs", prefs)
+#             # chrome_options.page_load_strategy = capability
+#             try:
+#                 # Try to install ChromeDriver using webdriver-manager
+
+#                 # driver_pah='/Users/macjianfeng/.wdm/drivers/chromedriver/mac64/129.0.6668.100/chromedriver-mac-arm64/chromedriver'
+#                 # service=Service(executable_path=driver_path)
+
+#                 service = Service(ChromeDriverManager().install())
+#                 driver_ = webdriver.Chrome(service=service, options=chrome_options)
+#             except Exception as e:
+#                 print(f"Error occurred: {e}")
+#                 print("Attempting to reinstall webdriver-manager...")
+#                 try: 
+#                     service = Service(ChromeDriverManager().install())
+#                     driver_ = webdriver.Chrome(service=service, options=chrome_options)
+#                 except Exception as reinstall_error:
+#                     print(
+#                         f"Reinstallation failed: {reinstall_error}\n之前发生过类似的问题, 更新了webdriver-manager以后得到解决"
+#                     )
+#                     try:
+#                         ips.upgrade("webdriver-manager", uninstall=True)
+#                         service = Service(ChromeDriverManager().install())
+#                         driver_ = webdriver.Chrome(service=service, options=chrome_options)
+#                     except Exception as e:
+#                         print(
+#                         f"Reinstallation failed: {reinstall_error}\n之前发生过类似的问题, 但是更新了'webdriver-manager'之后依然没有解决"
+#                     )
+
+#             # 隐式等等待
+#             if 3 < wait < 5:
+#                 wait_ = random.uniform(3, 5)
+#             elif 5 <= wait < 8:
+#                 wait_ = random.uniform(5, 8)
+#             elif 8 <= wait < 12:
+#                 wait_ = random.uniform(8, 10)
+#             else:
+#                 wait_ = 0
+#             driver_.implicitly_wait(wait_)
+
+#             if wait_until is not None and wait_until_kind is not None:
+#                 strategy = corr_by_kind(wait_until_kind)
+#                 WebDriverWait(driver_, timeout).until(
+#                     EC.presence_of_element_located((strategy, wait_until))
+#                 )
+#             if login_url and login_dict:
+#                 cookies = get_cookies(url=login_url, login=login_dict)
+#                 driver_.get(url)
+#                 for cookie_name, cookie_value in cookies.items():
+#                     driver_.add_cookie({"name": cookie_name, "value": cookie_value})
+#             else:
+#                 if cookies:
+#                     driver_.get(url)
+#                     if isinstance(cookies, str):
+#                         cookies=parse_cookies(cookies)
+#                     for cookie_name, cookie_value in cookies.items():
+#                         driver_.add_cookie({"name": cookie_name, "value": cookie_value})
+#             if not javascript:
+#                 driver_.execute_cdp_cmd(
+#                     "Emulation.setScriptExecutionDisabled", {"value": True}
+#                 )
+
+#             if login_url:
+#                 driver_.get(login_url)
+#                 WebDriverWait(driver_, timeout).until(
+#                     EC.presence_of_element_located((username_by, username_field))
+#                 ).send_keys(username)
+#                 WebDriverWait(driver_, timeout).until(
+#                     EC.presence_of_element_located((password_by, password_field))
+#                 ).send_keys(password)
+#                 WebDriverWait(driver_, timeout).until(
+#                     EC.element_to_be_clickable((submit_by, submit_field))
+#                 ).click()
+
+#             driver_.get(url)
+
+#             if iframe_name:
+#                 iframe = WebDriverWait(driver_, timeout).until(
+#                     EC.presence_of_element_located((By.NAME, iframe_name))
+#                 )
+#                 driver_.switch_to.frame(iframe)
+
+#             # WebDriverWait(driver, timeout).until(
+#             #     EC.presence_of_element_located((by, where))
+#             # )
+
+#             # # scroll down the page by a certain number of pixels
+#             scroll_smth_steps(driver_)
+
+#             # 设置轮询
+#             for attempt in range(scroll_try):
+#                 page_source = driver_.page_source
+#                 content = BeautifulSoup(page_source, "html.parser")
+#                 if content and content.find_all(by):
+#                     break
+#                 time.sleep(
+#                     random.uniform(2, 4)
+#                 )  # Wait for a random time before polling again
+
+#             driver_.quit()
+
+#             # content = BeautifulSoup(page_source, "html.parser")
+#             if content:
+#                 return "text/html", content
+#             else:
+#                 logger.warning("Selenium could not fetch content")
+#                 return None, None
+#         elif 'scr' in driver.lower():
+#             settings = {
+#                 "USER_AGENT": user_agent(),
+#                 "DOWNLOAD_DELAY": 1,  # Prevent overloading the server
+#                 "COOKIES_ENABLED": True if cookies else False,
+#                 "LOG_LEVEL": "WARNING",  # Reduce log verbosity
+#             }
+#             content=fetch_scrapy(url, 
+#                                  parser=parser, 
+#                                  cookies=cookies, 
+#                                  headers=headers, 
+#                                  settings=settings)
+#             return parser, content 
+
+#     except requests.RequestException as e:
+#         logger.error(f"Error fetching URL '{url}': {e}")
+#         return None, None
+ 
+def _clean_temp():
+    import os
+    import shutil
+    import tempfile
+    from pathlib import Path
+
+    # Get the parent folder of the tempdir
+    temp_dir = Path(tempfile.gettempdir()).parent  # moves from /T to parent dir
+
+    for subdir in temp_dir.iterdir():
+        if subdir.is_dir():
+            for d in subdir.iterdir():
+                if "com.google.Chrome.code_sign_clone" in d.name:
+                    try:
+                        print(f"Removing: {d}")
+                        shutil.rmtree(d)
+                    except Exception as e:
+                        print(f"Error removing {d}: {e}")
 def fetch_all(
     url,
     parser="lxml",
     driver="request",  # request or selenium
     by=By.TAG_NAME,
     timeout=10,
-    retry=2,
+    retry=3,  # Increased default retries
     wait=0,
     wait_until=None,
     wait_until_kind=None,
@@ -427,231 +732,222 @@ def fetch_all(
     username_by=By.NAME,
     password_by=By.NAME,
     submit_by=By.NAME,
-    # capability='eager', # eager or none
-    proxy=None,  # Add proxy parameter
-    javascript=True,  # Add JavaScript option
-    disable_images=False,  # Add option to disable images
+    proxy=None,
+    javascript=True,
+    disable_images=False,
     iframe_name=None,
     login_dict=None,
-    cookies=None,  # Add cookies parameter
-):  # Add option to handle iframe):  # lxml is faster, # parser="html.parser"
-    try:
-        # # Generate a random user-agent string
-        # response = requests.get(url)
-        # # get cookies
-        # cookie=dict_from_cookiejar(response.cookies)
-        # # get token from cookies
-        # scrf_token=re.findall(r'csrf-token=(.*?);', response.headers.get('Set-Cookie'))[0]
-        # headers = {"User-Agent": user_agent(), "X-CSRF-Token":scrf_token}
-
-        headers = {"User-Agent": user_agent()}
-        if "req" in driver.lower():
-            # response = requests.get(
-            #     url, headers=headers, proxies=proxies_glob, timeout=30, stream=True
-            # )
-
-            # Handle cookies for requests
-            if cookies:
-                from requests.cookies import RequestsCookieJar
-                cookie_jar = RequestsCookieJar()
-                if isinstance(cookies, str):
-                    cookies=parse_cookies(cookies)
-                for cookie_name, cookie_value in cookies.items():
-                    cookie_jar.set(cookie_name, cookie_value)
-                response = requests.get(
-                    url, headers=headers, cookies=cookie_jar, timeout=timeout, stream=True
-                )
-            else:
-                response = requests.get(url, headers=headers, timeout=timeout, stream=True)
+    cookies=None,
+    verify_ssl=True,  # Added SSL verification option
+    follow_redirects=True,  # Added redirect control
+):
+    """
+    Enhanced fetch function with better error handling and reliability.
+    
+    Returns:
+        tuple: (content_type, parsed_content) or (None, None) on failure
+    """
+    def _parse_content(content, content_type, parser):
+        """Helper function to parse content with fallback"""
+        try:
+            if content_type in CONTENT_PARSERS:
+                return CONTENT_PARSERS[content_type](content, parser)
+            
+            # Fallback parsing attempts
+            if content_type.startswith('text/'):
+                try:
+                    return BeautifulSoup(content, parser)
+                except:
+                    return content
+            return content
+        except Exception as e:
+            logger.warning(f"Content parsing failed: {e}")
+            return content
 
-            # If the response is a redirect, follow it
-            while response.is_redirect:
-                logger.info(f"Redirecting to: {response.headers['Location']}")
+    def _make_request(url, headers, cookies, timeout, verify_ssl, follow_redirects):
+        """Helper function for HTTP requests with retries"""
+        for attempt in range(retry):
+            try:
                 response = requests.get(
-                    response.headers["Location"],
+                    url,
                     headers=headers,
-                    proxies=proxies_glob,
+                    cookies=cookies,
                     timeout=timeout,
                     stream=True,
+                    verify=verify_ssl,
+                    allow_redirects=follow_redirects
                 )
-            # Check for a 403 error
-            if response.status_code == 403:
-                logger.warning("403 Forbidden error. Retrying...")
-                # Retry the request after a short delay
-                time.sleep(random.uniform(1, 3))
-                response = requests.get(
-                    url, headers=headers, proxies=proxies_glob, timeout=timeout, stream=True
-                )
-                # Raise an error if retry also fails
+                
+                # Handle redirects manually if needed
+                if not follow_redirects and response.is_redirect:
+                    logger.info(f"Redirect detected to: {response.headers['Location']}")
+                    return None, None
+                
                 response.raise_for_status()
-
-            # Raise an error for other HTTP status codes
-            response.raise_for_status()
-
-            # Get the content type
-            content_type = (
-                response.headers.get("content-type", "").split(";")[0].lower()
+                return response, None
+                
+            except requests.RequestException as e:
+                logger.warning(f"Attempt {attempt + 1} failed: {e}")
+                if attempt == retry - 1:
+                    return None, e
+                time.sleep(random.uniform(1, 3))
+    
+    # Convert driver integer to string if needed
+    if isinstance(driver, int):
+        drivers = ["request", "selenium", "scrapy"]
+        try:
+            driver = drivers[driver]
+        except IndexError:
+            driver = "request"
+    
+    headers = {"User-Agent": user_agent()}
+    
+    # Prepare cookies
+    cookie_jar = None
+    if cookies:
+        from requests.cookies import RequestsCookieJar
+        cookie_jar = RequestsCookieJar()
+        if isinstance(cookies, str):
+            cookies = parse_cookies(cookies)
+        for name, value in cookies.items():
+            cookie_jar.set(name, value)
+    
+    try:
+        if "req" in driver.lower():
+            response, error = _make_request(
+                url, headers, cookie_jar, timeout, verify_ssl, follow_redirects
             )
-            if response.encoding:
-                content = response.content.decode(response.encoding)
-            else:
-                content = None
-            # logger.info(f"Content type: {content_type}")
-
-            # Check if content type is supported
-            if content_type in CONTENT_PARSERS and content:
-                return content_type, CONTENT_PARSERS[content_type](content, parser)
-            else:
-                logger.warning("Unsupported content type")
+            if error:
                 return None, None
+            content_type = response.headers.get("content-type", "").split(";")[0].lower()
+            try:
+                detected = chardet.detect(response.content)
+                encoding = detected.get("encoding") or "utf-8"
+                content = response.content.decode(encoding, errors='replace')
+            except:
+                content = response.content.decode(response.encoding or 'utf-8', errors='replace')
+            
+            return content_type, _parse_content(content, content_type, parser)
+            
         elif "se" in driver.lower():
-            import random
             from selenium import webdriver
             from selenium.webdriver.chrome.service import Service
-            from selenium.webdriver.common.by import By
             from selenium.webdriver.chrome.options import Options
-            from selenium.webdriver.support.ui import WebDriverWait
-            from selenium.webdriver.support import expected_conditions as EC
             from webdriver_manager.chrome import ChromeDriverManager
-            from selenium.webdriver.common.desired_capabilities import (
-                DesiredCapabilities,
-            )
-
+            from selenium.common.exceptions import WebDriverException
+            
             chrome_options = Options()
             chrome_options.add_argument("--headless")
             chrome_options.add_argument("--no-sandbox")
+            chrome_options.add_argument("--disable-gpu")
             chrome_options.add_argument("--disable-dev-shm-usage")
+            chrome_options.add_argument(f'--user-data-dir={os.path.expanduser("~/selenium_profile")}')
             chrome_options.add_argument(f"user-agent={user_agent()}")
+            
             if proxy:
                 chrome_options.add_argument(f"--proxy-server={proxy}")
             if disable_images:
-                prefs = {"profile.managed_default_content_settings.images": 2}
-                chrome_options.add_experimental_option("prefs", prefs)
-            # chrome_options.page_load_strategy = capability
+                chrome_options.add_experimental_option(
+                    "prefs", {"profile.managed_default_content_settings.images": 2}
+                )
+            
+            driver_instance = None
             try:
-                # Try to install ChromeDriver using webdriver-manager
-
-                # driver_pah='/Users/macjianfeng/.wdm/drivers/chromedriver/mac64/129.0.6668.100/chromedriver-mac-arm64/chromedriver'
-                # service=Service(executable_path=driver_path)
-
+                # Try with latest ChromeDriver first
                 service = Service(ChromeDriverManager().install())
-                driver_ = webdriver.Chrome(service=service, options=chrome_options)
-            except Exception as e:
-                print(f"Error occurred: {e}")
-                print("Attempting to reinstall webdriver-manager...")
-                try: 
-                    service = Service(ChromeDriverManager().install())
-                    driver_ = webdriver.Chrome(service=service, options=chrome_options)
-                except Exception as reinstall_error:
-                    print(
-                        f"Reinstallation failed: {reinstall_error}\n之前发生过类似的问题, 更新了webdriver-manager以后得到解决"
+                driver_instance = webdriver.Chrome(service=service, options=chrome_options)
+                
+                # Configure wait times
+                if 3 < wait < 5:
+                    wait_time = random.uniform(3, 5)
+                elif 5 <= wait < 8:
+                    wait_time = random.uniform(5, 8)
+                elif 8 <= wait < 12:
+                    wait_time = random.uniform(8, 10)
+                else:
+                    wait_time = 0
+                
+                driver_instance.implicitly_wait(wait_time)
+                
+                # Handle login if needed
+                if login_url and login_dict:
+                    cookies = get_cookies(url=login_url, login=login_dict)
+                    driver_instance.get(url)
+                    for name, value in cookies.items():
+                        driver_instance.add_cookie({"name": name, "value": value})
+                elif cookies:
+                    driver_instance.get(url)
+                    if isinstance(cookies, str):
+                        cookies = parse_cookies(cookies)
+                    for name, value in cookies.items():
+                        driver_instance.add_cookie({"name": name, "value": value})
+                
+                if not javascript:
+                    driver_instance.execute_cdp_cmd(
+                        "Emulation.setScriptExecutionDisabled", {"value": True}
+                    )
+                
+                # Navigate to target URL
+                driver_instance.get(url)
+                
+                # Handle iframes if needed
+                if iframe_name:
+                    iframe = WebDriverWait(driver_instance, timeout).until(
+                        EC.presence_of_element_located((By.NAME, iframe_name))
                     )
+                    driver_instance.switch_to.frame(iframe)
+                
+                # Scroll to trigger dynamic content
+                scroll_smth_steps(driver_instance)
+                
+                # Get page source with retries
+                content = None
+                for attempt in range(scroll_try):
                     try:
-                        ips.upgrade("webdriver-manager", uninstall=True)
-                        service = Service(ChromeDriverManager().install())
-                        driver_ = webdriver.Chrome(service=service, options=chrome_options)
+                        page_source = driver_instance.page_source
+                        content = BeautifulSoup(page_source, parser)
+                        if content and content.find_all(by):
+                            break
                     except Exception as e:
-                        print(
-                        f"Reinstallation failed: {reinstall_error}\n之前发生过类似的问题, 但是更新了'webdriver-manager'之后依然没有解决"
-                    )
-
-            # 隐式等等待
-            if 3 < wait < 5:
-                wait_ = random.uniform(3, 5)
-            elif 5 <= wait < 8:
-                wait_ = random.uniform(5, 8)
-            elif 8 <= wait < 12:
-                wait_ = random.uniform(8, 10)
-            else:
-                wait_ = 0
-            driver_.implicitly_wait(wait_)
-
-            if wait_until is not None and wait_until_kind is not None:
-                strategy = corr_by_kind(wait_until_kind)
-                WebDriverWait(driver_, timeout).until(
-                    EC.presence_of_element_located((strategy, wait_until))
-                )
-            if login_url and login_dict:
-                cookies = get_cookies(url=login_url, login=login_dict)
-                driver_.get(url)
-                for cookie_name, cookie_value in cookies.items():
-                    driver_.add_cookie({"name": cookie_name, "value": cookie_value})
-            else:
-                if cookies:
-                    driver_.get(url)
-                    if isinstance(cookies, str):
-                        cookies=parse_cookies(cookies)
-                    for cookie_name, cookie_value in cookies.items():
-                        driver_.add_cookie({"name": cookie_name, "value": cookie_value})
-            if not javascript:
-                driver_.execute_cdp_cmd(
-                    "Emulation.setScriptExecutionDisabled", {"value": True}
-                )
-
-            if login_url:
-                driver_.get(login_url)
-                WebDriverWait(driver_, timeout).until(
-                    EC.presence_of_element_located((username_by, username_field))
-                ).send_keys(username)
-                WebDriverWait(driver_, timeout).until(
-                    EC.presence_of_element_located((password_by, password_field))
-                ).send_keys(password)
-                WebDriverWait(driver_, timeout).until(
-                    EC.element_to_be_clickable((submit_by, submit_field))
-                ).click()
-
-            driver_.get(url)
-
-            if iframe_name:
-                iframe = WebDriverWait(driver_, timeout).until(
-                    EC.presence_of_element_located((By.NAME, iframe_name))
-                )
-                driver_.switch_to.frame(iframe)
-
-            # WebDriverWait(driver, timeout).until(
-            #     EC.presence_of_element_located((by, where))
-            # )
-
-            # # scroll down the page by a certain number of pixels
-            scroll_smth_steps(driver_)
-
-            # 设置轮询
-            for attempt in range(scroll_try):
-                page_source = driver_.page_source
-                content = BeautifulSoup(page_source, "html.parser")
-                if content and content.find_all(by):
-                    break
-                time.sleep(
-                    random.uniform(2, 4)
-                )  # Wait for a random time before polling again
-
-            driver_.quit()
-
-            # content = BeautifulSoup(page_source, "html.parser")
-            if content:
-                return "text/html", content
-            else:
-                logger.warning("Selenium could not fetch content")
+                        logger.warning(f"Attempt {attempt + 1} failed: {e}")
+                    time.sleep(random.uniform(1, 3))
+                try:
+                    _clean_temp()
+                except Exception as e:
+                    print(e)
+                return "text/html", content if content else None
+                
+            except WebDriverException as e:
+                logger.error(f"Selenium error: {e}")
                 return None, None
+            finally:
+                if driver_instance:
+                    driver_instance.quit()
+                    
         elif 'scr' in driver.lower():
             settings = {
                 "USER_AGENT": user_agent(),
-                "DOWNLOAD_DELAY": 1,  # Prevent overloading the server
-                "COOKIES_ENABLED": True if cookies else False,
-                "LOG_LEVEL": "WARNING",  # Reduce log verbosity
+                "DOWNLOAD_DELAY": 1,
+                "COOKIES_ENABLED": bool(cookies),
+                "LOG_LEVEL": "WARNING",
+                "RETRY_TIMES": retry,
+                "DOWNLOAD_TIMEOUT": timeout,
             }
-            content=fetch_scrapy(url, 
-                                 parser=parser, 
-                                 cookies=cookies, 
-                                 headers=headers, 
-                                 settings=settings)
-            return parser, content 
-
-    except requests.RequestException as e:
-        logger.error(f"Error fetching URL '{url}': {e}")
+            content = fetch_scrapy(
+                url, 
+                parser=parser, 
+                cookies=cookies, 
+                headers=headers, 
+                settings=settings
+            )
+            return parser, content
+            
+    except Exception as e:
+        logger.error(f"Unexpected error in fetch_all: {e}")
         return None, None
-
-
+    
+    return None, None
+ 
 # # Function to change Tor IP address
 # def renew_tor_ip():
 #     with Controller.from_port(port=9051) as controller:
@@ -1050,6 +1346,8 @@ def find_img(url, driver="request", dir_save="images", rm_folder=False, verbose=
     if rm_folder:
         ips.rm_folder(dir_save)
     content_type, content = fetch_all(url, driver=driver)
+    if content_type is None:
+        content_type=""
     if "html" in content_type.lower():
         # Create the directory if it doesn't exist
         os.makedirs(dir_save, exist_ok=True)
@@ -1193,7 +1491,9 @@ def fetch_selenium(
     chrome_options = Options()
     chrome_options.add_argument("--headless")
     chrome_options.add_argument("--no-sandbox")
+    chrome_options.add_argument("--disable-gpu")
     chrome_options.add_argument("--disable-dev-shm-usage")
+    chrome_options.add_argument(f'--user-data-dir={os.path.expanduser("~/selenium_profile")}')
     chrome_options.add_argument(f"user-agent={user_agent()}")
     if proxy:
         chrome_options.add_argument(f"--proxy-server={proxy}")
@@ -1299,7 +1599,7 @@ def fetch(
             condition_ = [texts.empty, booster]
         else:
             condition_ = [not texts, booster]
-        if any(condition_):
+        if any(condition_): 
             print("trying to use 'fetcher2'...")
             texts = fetch_selenium(
                 url=url, where=where, what=what, extend=extend, **kws
@@ -1307,6 +1607,7 @@ def fetch(
         if texts:
             return texts
         else:
+            print("got nothing")
             return fetch(
                 url,
                 where=where,