abstract-webtools 0.1.6.145__py3-none-any.whl → 0.1.6.147__py3-none-any.whl

abstract_webtools/managers/cipherManager.py

@@ -1,21 +1,20 @@
 class CipherManager:
     @staticmethod
-    def  get_default_ciphers()-> list:
+    def get_default_ciphers() -> list:
         return [
-            "ECDHE-RSA-AES256-GCM-SHA384", "ECDHE-ECDSA-AES256-GCM-SHA384",
-            "ECDHE-RSA-AES256-SHA384", "ECDHE-ECDSA-AES256-SHA384",
-            "ECDHE-RSA-AES256-SHA", "ECDHE-ECDSA-AES256-SHA",
-            "ECDHE-RSA-AES128-GCM-SHA256", "ECDHE-RSA-AES128-SHA256",
-            "ECDHE-ECDSA-AES128-GCM-SHA256", "ECDHE-ECDSA-AES128-SHA256",
-            "AES256-SHA", "AES128-SHA"
+            "ECDHE-RSA-AES256-GCM-SHA384","ECDHE-ECDSA-AES256-GCM-SHA384",
+            "ECDHE-RSA-AES256-SHA384","ECDHE-ECDSA-AES256-SHA384",
+            "ECDHE-RSA-AES256-SHA","ECDHE-ECDSA-AES256-SHA",
+            "ECDHE-RSA-AES128-GCM-SHA256","ECDHE-RSA-AES128-SHA256",
+            "ECDHE-ECDSA-AES128-GCM-SHA256","ECDHE-ECDSA-AES128-SHA256",
+            "AES256-SHA","AES128-SHA"
         ]
+    def __init__(self, cipher_list=None):
+        self.cipher_list = cipher_list or self.get_default_ciphers()
+        if isinstance(self.cipher_list, str):
+            self.cipher_list = [c.strip() for c in self.cipher_list.split(',') if c.strip()]
+        self.ciphers_string = ','.join(self.cipher_list) if self.cipher_list else ''
 
-    def __init__(self,cipher_list=None):
-        if cipher_list == None:
-            cipher_list=self.get_default_ciphers()
-        self.cipher_list = cipher_list
-        self.create_list()
-        self.ciphers_string = self.add_string_list()
     def add_string_list(self):
         if len(self.cipher_list)==0:
             return ''

abstract_webtools/managers/crawlManager.py

@@ -119,14 +119,42 @@ if __name__ == "__main__":
     generator.run()
 
 class crawlManager:
-    def __init__(self, url=None, req_mgr=None, url_mgr=None, source_code=None, parse_type="html.parser"):
+    def __init__(self, url, req_mgr, url_mgr, source_code=None, parse_type="html.parser"):
+        self.url_mgr = url_mgr
+        self.req_mgr = req_mgr
         self.url = url
-        self.source_code = source_code
         self.parse_type = parse_type
-        self.url_mgr = url_mgr or urlManager(url=self.url)
-        self.req_mgr = req_mgr or requestManager(url_mgr=self.url_mgr)
-        self.get_new_source_and_url(url)
-        self.sime_map_mgr = SitemapGenerator(self.url_mgr.domain)
+        self.source_code = source_code or req_mgr.source_code
+        self.soup = BeautifulSoup(self.source_code or "", parse_type)
+        self.base_netloc = urlparse(self.url).netloc
+
+    def is_internal(self, link):
+        u = urlparse(link)
+        return (not u.netloc) or (u.netloc == self.base_netloc)
+
+    def links_on_page(self):
+        out = set()
+        for a in self.soup.find_all("a", href=True):
+            out.add(urljoin(self.url, a["href"]))
+        return out
+
+    def crawl(self, start=None, max_depth=2, _depth=0, visited=None, session=None):
+        start = start or self.url
+        visited = visited or set()
+        if _depth > max_depth or start in visited:
+            return visited
+        visited.add(start)
+
+        # fetch
+        r = self.req_mgr.session.get(start, timeout=30)
+        r.raise_for_status()
+        soup = BeautifulSoup(r.text, self.parse_type)
+
+        for a in soup.find_all("a", href=True):
+            link = urljoin(start, a["href"])
+            if self.is_internal(link) and link not in visited:
+                self.crawl(link, max_depth=max_depth, _depth=_depth+1, visited=visited)
+        return visited
     def get_new_source_and_url(self, url=None):
         """Fetches new source code and response for a given URL."""
         url = url
@@ -194,26 +222,7 @@ class crawlManager:
             return ('yearly', '0.3')
         return ('weekly', '1.0')
 
-    def crawl(self, url, max_depth=3, depth=1, visited=None):
-        """Recursively crawls the site up to max_depth and returns valid internal links."""
-        visited = visited or set()
-        if depth > max_depth or url in visited:
-            return []
-
-        visited.add(url)
-        try:
-            soup = get_soup(url)
-            links = []
-            for tag in soup.find_all('a', href=True):
-                link = urljoin(url, tag['href'])
-                if urlparse(link).netloc == urlparse(url).netloc and link not in visited:
-                    links.append(link)
-                    self.crawl(link, max_depth, depth + 1, visited)
-            return links
-        except Exception as e:
-            print(f"Error crawling {url}: {e}")
-            return []
-
+  
     def get_meta_info(self, url=None):
         """Fetches metadata, including title and meta tags, from the page."""
         url = url or self.url

abstract_webtools/managers/curlMgr.py

@@ -1,48 +1,26 @@
-import os
-import requests
-import os
-import subprocess
-import stat
-
-def get_site(website, destination_dir, filename):
-    # Ensure the directory exists
-    os.makedirs(destination_dir, exist_ok=True)
-
-    # Adjust directory permissions if needed (e.g. rwxr-xr-x -> 0o755)
-    os.chmod(destination_dir, 0o755)
-
-    # Construct the complete file path
-    destination_path = os.path.join(destination_dir, filename)
-
-    # Use curl to download the site
-    # The example user-agent is arbitrary; you can change it to your needs
-    os.system(
-        f'curl -L --output "{destination_path}" '
-        f'-H "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
-        f'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 '
-        f'Safari/537.36" -H "Accept: */*" "{website}"'
-    )
-
-def download_site(website, destination_dir, filename):
-    os.makedirs(destination_dir, exist_ok=True)
-    os.chmod(destination_dir, 0o755)  # set directory permissions if needed
-
-    destination_path = os.path.join(destination_dir, filename)
-
-    # GET the resource
-    response = requests.get(website, headers={
-        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-                      "AppleWebKit/537.36 (KHTML, like Gecko) "
-                      "Chrome/91.0.4472.124 Safari/537.36",
-        "Accept": "*/*"
-    }, allow_redirects=True)
-
-    # Raise an exception if the download fails
-    response.raise_for_status()
-
-    # Write content to file
+import os, subprocess, requests
+
+def curl_download(website, destination_path, user_agent=None):
+    os.makedirs(os.path.dirname(destination_path), exist_ok=True)
+    ua = user_agent or ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                        "AppleWebKit/537.36 (KHTML, like Gecko) "
+                        "Chrome/91.0.4472.124 Safari/537.36")
+    subprocess.run([
+        "curl","-L","--output", destination_path,
+        "-H", f"User-Agent: {ua}",
+        "-H", "Accept: */*",
+        website
+    ], check=True)
+
+def requests_download(website, destination_path, headers=None):
+    os.makedirs(os.path.dirname(destination_path), exist_ok=True)
+    hdr = {"User-Agent": ("Mozilla/5.0 ... Chrome/91.0 Safari/537.36"),
+           "Accept": "*/*"}
+    if headers: hdr.update(headers)
+    r = requests.get(website, headers=hdr, allow_redirects=True, timeout=30)
+    r.raise_for_status()
     with open(destination_path, "wb") as f:
-        f.write(response.content)
-website = 'https://www.pornhub.com'
-destination = '/home/computron/Documents/doge'
-get_site(website,destination,'doge')
+        f.write(r.content)
+
+if __name__ == "__main__":
+    pass  # no side effects

abstract_webtools/managers/meta_dump.py

@@ -0,0 +1,27 @@
+# meta_dump.py
+from abstract_webtools.managers.networkManager import NetworkManager
+from abstract_webtools.managers.userAgentManager import UserAgentManager
+from abstract_webtools.managers.soupManager.soupManager import soupManager
+import json, sys
+
+def dump_all_meta(url: str):
+    ua = UserAgentManager(browser="Chrome", operating_system="Windows")
+    net = NetworkManager(user_agent_manager=ua)
+
+    r = net.session.get(url, timeout=30)
+    r.raise_for_status()
+
+    sm = soupManager(url=url, source_code=r.text, req_mgr=net)
+    out = {
+        "url": url,
+        "title": sm.soup.title.string.strip() if sm.soup.title and sm.soup.title.string else None,
+        "meta": sm.all_meta(),
+        "citation": sm.citation_dict(),
+        "links": sm.all_links(),
+        "json_ld": sm.all_jsonld(),
+    }
+    print(json.dumps(out, indent=2, ensure_ascii=False))
+
+if __name__ == "__main__":
+    url = sys.argv[1]
+    dump_all_meta(url)

abstract_webtools/managers/networkManager.py

@@ -1,15 +1,50 @@
+from typing import Optional, List
+import requests
 from ..abstract_webtools import *
-from ..big_user_agent_list import *
+from .sslManager import SSLManager
+from .cipherManager import CipherManager
+
+class TLSAdapter(HTTPAdapter):
+    def __init__(self, ssl_manager: SSLManager=None):
+        ssl_manager = ssl_manager or SSLManager()
+        self.ssl_context = ssl_manager.ssl_context
+        super().__init__()
+    def init_poolmanager(self, *args, **kwargs):
+        kwargs['ssl_context'] = self.ssl_context
+        return super().init_poolmanager(*args, **kwargs)
+
 class NetworkManager:
-    def __init__(self, user_agent_manager=None,ssl_manager=None, tls_adapter=None,user_agent=None,proxies=None,cookies=None,ciphers=None, certification: Optional[str] = None, ssl_options: Optional[List[str]] = None):
-        if ssl_manager == None:
-            ssl_manager = SSLManager(ciphers=ciphers, ssl_options=ssl_options, certification=certification)
-        self.ssl_manager=ssl_manager
-        if tls_adapter == None:
-            tls_adapter=TLSAdapter(ssl_manager=ssl_manager,ciphers=ciphers, certification=certification, ssl_options=ssl_options)
-        self.tls_adapter=tls_adapter
-        self.ciphers=tls_adapter.ciphers
-        self.certification=tls_adapter.certification
-        self.ssl_options=tls_adapter.ssl_options
-        self.proxies=None or {}
-        self.cookies=cookies or "cb4c883efc59d0e990caf7508902591f4569e7bf-1617321078-0-150"
+    def __init__(self, user_agent_manager=None, ssl_manager=None, proxies=None, cookies=None,
+                 ciphers=None, certification: Optional[str]=None, ssl_options: Optional[List[str]]=None):
+        self.ua_mgr = user_agent_manager or UserAgentManager()
+        self.ssl_mgr = ssl_manager or SSLManager(
+            ciphers=ciphers or CipherManager().ciphers_string,
+            ssl_options=ssl_options,
+            certification=certification
+        )
+
+        self.session = requests.Session()
+        self.session.headers.update({
+            "User-Agent": self.ua_mgr.user_agent,
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.9",
+            "Connection": "keep-alive"
+        })
+        adapter = TLSAdapter(self.ssl_mgr)
+        self.session.mount("https://", adapter)
+        self.session.mount("http://", HTTPAdapter())
+
+        if proxies:
+            self.session.proxies = proxies
+        if cookies:
+            if isinstance(cookies, requests.cookies.RequestsCookieJar):
+                self.session.cookies = cookies
+            elif isinstance(cookies, dict):
+                jar = requests.cookies.RequestsCookieJar()
+                for k,v in cookies.items(): jar.set(k,v)
+                self.session.cookies = jar
+            # if string: up to you—parse or ignore
+
+        # retries (optional)
+        from requests.adapters import Retry
+        self.session.adapters['https://'].max_retries = Retry(total=5, backoff_factor=0.5, status_forcelist=[429,500,502,503,504])

abstract_webtools/managers/requestManager/requestManager.py

@@ -4,7 +4,7 @@ from ..cipherManager import *
 from ..sslManager import *
 from ..tlsAdapter import *
 from ..networkManager import *
-from ..seleniumManager import *
+from ..seleneumManager import *
 from ..urlManager import *
 logging.basicConfig(level=logging.INFO)
 

abstract_webtools/managers/seleneumManager.py

@@ -239,3 +239,4 @@ def get_driver(self, url):
     key = f"{url}#{time.time()}"
     self._sessions[key] = {"driver": driver, "profile": prof}
     return driver
+seleneumManager = seleniumManager

abstract_webtools/managers/seleniumManager.py

@@ -1,36 +1,114 @@
-import os
-from ..abstract_webtools import *
-from .urlManager import *
-from urllib.parse import urlparse
-from abstract_utilities import *
+import os, time, re, json, logging, urllib3, requests,tempfile, shutil, socket, atexit, errno
+from urllib.parse import urlparse, urljoin
+from bs4 import BeautifulSoup          # if you prefer, keep using your parser
 from selenium import webdriver
 from selenium.webdriver.chrome.options import Options
-import logging
-import urllib3
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from abstract_security import get_env_value
+from abstract_utilities import *
+from .urlManager import *               # your urlManager
 
-# Suppress urllib3 warnings and debug logs
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 logging.getLogger("urllib3").setLevel(logging.WARNING)
-
-# Suppress Selenium logs
 logging.getLogger("selenium").setLevel(logging.WARNING)
 
-import os
-from selenium import webdriver
-from selenium.webdriver.chrome.options import Options
-
-# Setup Chrome options
+# ---- Chrome options (keep yours; add safe fallbacks) ----
 chrome_options = Options()
-#chrome_options.binary_location = "/home/profiles/solcatcher/.cache/selenium/chrome/linux64/130.0.6723.58/chrome"
-chrome_options.add_argument("--headless")  # Run in headless mode
+_bin = get_env_value('CHROME_BINARY')
+if _bin:
+    chrome_options.binary_location = _bin
+chrome_options.add_argument("--headless=new")
 chrome_options.add_argument("--no-sandbox")
 chrome_options.add_argument("--disable-dev-shm-usage")
 chrome_options.add_argument("--disable-gpu")
 chrome_options.add_argument("--disable-software-rasterizer")
 chrome_options.add_argument("--disable-extensions")
 chrome_options.add_argument("--remote-debugging-port=9222")
+chrome_prefs = {"profile.managed_default_content_settings.images": 2}
+chrome_options.experimental_options["prefs"] = chrome_prefs
+
+MIN_HTML_BYTES = 2048  # tune: consider <2KB suspicious for real pages
+# --- NEW helpers: unique temp profile + free port + options builder ---
+
+def _free_port() -> int:
+    s = socket.socket()
+    s.bind(("127.0.0.1", 0))
+    port = s.getsockname()[1]
+    s.close()
+    return port
+
+def _make_profile_dir(base="/var/tmp/selenium-profiles") -> str:
+    os.makedirs(base, exist_ok=True)
+    return tempfile.mkdtemp(prefix="cw-", dir=base)
+
+def _make_chrome_options(binary_path: str | None = None,
+                         user_data_dir: str | None = None) -> tuple[Options, str]:
+    opts = Options()
+    if binary_path:
+        opts.binary_location = binary_path
+    opts.add_argument("--headless=new")
+    opts.add_argument("--no-sandbox")
+    opts.add_argument("--disable-dev-shm-usage")
+    opts.add_argument("--disable-gpu")
+    opts.add_argument("--disable-software-rasterizer")
+    opts.add_argument("--disable-extensions")
+
+    prof = user_data_dir or _make_profile_dir()
+    opts.add_argument(f"--user-data-dir={prof}")
+    opts.add_argument(f"--remote-debugging-port={_free_port()}")
+
+    prefs = {"profile.managed_default_content_settings.images": 2}
+    opts.add_experimental_option("prefs", prefs)
+    return opts, prof
+
 
+def _looks_like_html(text_or_bytes: bytes | str) -> bool:
+    if not text_or_bytes:
+        return False
+    s = text_or_bytes if isinstance(text_or_bytes, str) else text_or_bytes.decode("utf-8", "ignore")
+    if len(s) < MIN_HTML_BYTES:
+        return False
+    lowered = s.lower()
+    return ("<html" in lowered and "</html>" in lowered) or "<body" in lowered
 
+def _requests_fallback(url: str, headers: dict | None = None, timeout: float = 15.0):
+    """Plain requests fallback. Returns `requests.Response | None`."""
+    try:
+        sess = requests.Session()
+        sess.headers.update(headers or {"User-Agent": "Mozilla/5.0"})
+        # honor simple redirects and cert issues as needed
+        resp = sess.get(url, timeout=timeout, allow_redirects=True, verify=False)
+        return resp
+    except Exception as e:
+        logging.warning(f"requests fallback failed for {url}: {e}")
+        return None
+
+def _wait_until_ready(driver, timeout: float = 10.0):
+    """Waits for DOM readiness and presence of <body>."""
+    try:
+        WebDriverWait(driver, timeout).until(
+            lambda d: d.execute_script("return document.readyState") in ("interactive", "complete")
+        )
+    except Exception:
+        pass
+    try:
+        WebDriverWait(driver, timeout).until(EC.presence_of_element_located((By.TAG_NAME, "body")))
+    except Exception:
+        pass
+    # small settle delay for late JS injections
+    time.sleep(0.3)
+def normalize_url(url, base_url=None):
+    manager = seleniumManager(url)
+    base_url = manager.base_url
+    if url.startswith(base_url):
+        url = url[len(base_url):]
+    normalized_url = urljoin(base_url, url.split('#')[0])
+    if not normalized_url.startswith(base_url):
+        return None
+    return normalized_url
+# ---- Singleton driver manager (your class; small fixes) ----
 class SingletonMeta(type):
     _instances = {}
     def __call__(cls, *args, **kwargs):
@@ -41,79 +119,123 @@ class SingletonMeta(type):
 
 class seleniumManager(metaclass=SingletonMeta):
     def __init__(self, url):
-        if not hasattr(self, 'initialized'):  # Prevent reinitialization
-            self.initialized = True
-            parsed_url = urlparse(url)
-            self.domain = parsed_url.netloc
-            self.scheme = parsed_url.scheme
-            self.base_url= f"{self.scheme}{self.domain}"
-            self.site_dir = os.path.join(os.getcwd(), self.domain)
-            os.makedirs(self.site_dir, exist_ok=True)
-            self.drivers = {}
-            self.page_type = []
-    
+        if getattr(self, "initialized", False):
+            return
+        self.initialized = True
+
+        p = urlparse(url)
+        self.domain = p.netloc
+        self.scheme = p.scheme or "https"
+        self.base_url = f"{self.scheme}://{self.domain}"
+
+        self.site_dir = os.path.join("/var/tmp", "cw-sites", self.domain)
+        os.makedirs(self.site_dir, exist_ok=True)
+
+        self._sessions: dict[str, dict] = {}  # key -> {"driver": ..., "profile": ...}
+        atexit.register(lambda sm=self: sm.close_all())
+
     def get_url_to_path(self, url):
         url = eatAll(str(url), ['',' ','\n','\t','\\','/'])
-        parsed_url = urlparse(url)
-        if parsed_url.netloc == self.domain:
-            paths = parsed_url.path.split('/')
-            dir_path = self.site_dir
-            for path in paths[:-1]:
-                dir_path = os.path.join(dir_path, path)
-                os.makedirs(dir_path, exist_ok=True)
-            self.page_type.append(os.path.splitext(paths[-1])[-1] or 'html' if len(self.page_type) == 0 else self.page_type[-1])
-            
-            dir_path = os.path.join(dir_path, paths[-1])
-            return dir_path
-
-    def saved_url_check(self, url):
-        path = self.get_url_to_path(url)
-        return path
+        p = urlparse(url)
+        if p.netloc == self.domain:
+            parts = [x for x in p.path.split('/') if x]
+            d = self.site_dir
+            for seg in parts[:-1]:
+                d = os.path.join(d, seg)
+                os.makedirs(d, exist_ok=True)
+            last = parts[-1] if parts else "index.html"
+            ext = os.path.splitext(last)[-1] or ".html"
+            if not hasattr(self, "page_type"):
+                self.page_type = []
+            self.page_type.append(ext if not self.page_type else self.page_type[-1])
+            return os.path.join(d, last)
 
     def get_with_netloc(self, url):
-        parsed_url = urlparse(url)
-        if parsed_url.netloc == '':
-            url = f"{self.scheme}://{self.domain}/{url.strip()}"
+        p = urlparse(url)
+        if p.netloc == '':
+            url = f"{self.scheme}://{self.domain}/{url.strip().lstrip('/')}"
         return url
 
-    def get_driver(self, url):
-        if url and url not in self.drivers:
-            chrome_options = Options()
-            chrome_options.add_argument("--headless")
-            driver = webdriver.Chrome(options=chrome_options)
-            self.drivers[url] = driver
-            driver.get(url)
-        return self.drivers[url]
-def normalize_url(url, base_url=None):
-    """
-    Normalize and resolve relative URLs, ensuring proper domain and format.
-    """
-    # If URL starts with the base URL repeated, remove the extra part
-    manager = seleniumManager(url)
-    base_url = manager.base_url
-    if url.startswith(base_url):
-        url = url[len(base_url):]
+    def get_driver(self, url) -> tuple[str, webdriver.Chrome]:
+        bin_path = get_env_value('CHROME_BINARY')
+        opts, prof = _make_chrome_options(binary_path=bin_path, user_data_dir=None)
+        driver = webdriver.Chrome(options=opts)
+        key = f"{url}#{time.time()}"
+        self._sessions[key] = {"driver": driver, "profile": prof}
+        return key, driver
 
-    # Resolve the URL against the base URL
-    normalized_url = urljoin(base_url, url.split('#')[0])
+    def close_driver(self, key: str):
+        sess = self._sessions.pop(key, None)
+        if not sess: return
+        try:
+            try: sess["driver"].quit()
+            except Exception: pass
+        finally:
+            shutil.rmtree(sess.get("profile") or "", ignore_errors=True)
 
-    # Ensure only URLs belonging to the base domain are kept
-    if not normalized_url.startswith(base_url):
-        return None
+    def close_all(self):
+        for key in list(self._sessions.keys()):
+            self.close_driver(key)
 
-    return normalized_url
-# Function to get Selenium page source
-def get_selenium_source(url):
+
+
+# ---- Hardened page-source retrieval with fallback ----
+def get_selenium_source(url, max_retries: int = 2, request_fallback: bool = True, timeout: float = 12.0):
     url_mgr = urlManager(url)
-    if url_mgr.url:
-        url = str(url_mgr.url)
-        manager = seleniumManager(url)
-        driver = manager.get_driver(url)
-        try:
-            # Get page source
-            page_source = driver.page_source
-            return page_source
-        finally:
-            # Don't quit the driver unless you're done with all interactions
-            pass
+    if not url_mgr.url:
+        return None
+    url = str(url_mgr.url)
+
+    manager = seleniumManager(url)
+    key, driver = manager.get_driver(url)
+
+    last_exc = None
+    try:
+        for attempt in range(1, max_retries + 1):
+            try:
+                driver.get(url)
+                _wait_until_ready(driver, timeout=timeout)
+                html = driver.page_source or ""
+                if not _looks_like_html(html):
+                    html = driver.execute_script(
+                        "return document.documentElement ? document.documentElement.outerHTML : '';"
+                    ) or html
+                if _looks_like_html(html):
+                    return html
+                logging.warning(f"Selenium returned suspicious HTML (len={len(html)}) for {url} "
+                                f"[attempt {attempt}/{max_retries}]")
+            except Exception as e:
+                last_exc = e
+                logging.warning(f"Selenium attempt {attempt}/{max_retries} failed for {url}: {e}")
+            time.sleep(0.5 * attempt)
+
+        if request_fallback:
+            resp = _requests_fallback(url, headers={"User-Agent": "Mozilla/5.0"})
+            if resp is not None:
+                ctype = (resp.headers.get("content-type") or "").lower()
+                body = resp.text if hasattr(resp, "text") else (
+                    resp.content.decode("utf-8", "ignore") if hasattr(resp, "content") else ""
+                )
+                if "application/json" in ctype:
+                    try:
+                        return json.dumps(resp.json())
+                    except Exception:
+                        return body
+                return body if _looks_like_html(body) or body else None
+    finally:
+        # critical: release the user-data-dir to avoid “already in use”
+        manager.close_driver(key)
+
+    if last_exc:
+        logging.error(f"Unable to retrieve page for {url}: {last_exc}")
+    return None
 
+def get_driver(self, url):
+    # always new
+    bin_path = get_env_value('CHROME_BINARY')
+    opts, prof = _make_chrome_options(binary_path=bin_path, user_data_dir=None)
+    driver = webdriver.Chrome(options=opts)
+    # store so close_all() can clean up
+    key = f"{url}#{time.time()}"
+    self._sessions[key] = {"driver": driver, "profile": prof}
+    return driver