abstract-webtools 0.1.6.144__py3-none-any.whl → 0.1.6.145__py3-none-any.whl

abstract_webtools/managers/__init__.py

@@ -10,6 +10,7 @@ from .sslManager import *
 from .tlsAdapter import *
 from .urlManager import *
 from .userAgentManager import *
-from .seleniumManager import *
+from .seleneumManager import *
 from .videoDownloader import *
 from .middleManager import *
+seleniumManager = seleneumManager

abstract_webtools/managers/requestManager/requestManager.py

@@ -328,10 +328,11 @@ class requestManager:
 
     def make_request(self):
         """
-        Make a request and handle potential errors.
+        Make a request and handle potential errors, with retries.
         """
         if self.url_mgr.url is None:
             return None
+
         self.wait_between_requests()
         for _ in range(self.max_retries):
             try:
@@ -345,41 +346,52 @@ class requestManager:
                         elif self._response.status_code == 429:
                             logging.warning(f"Rate limited by {self.url_mgr.url}. Retrying...")
                             time.sleep(5)
+                    else:
+                        # String/bytes from Selenium path
+                        self.status_code = 200
+                        return self._response
             except requests.Timeout as e:
                 logging.error(f"Request to {self.url_mgr.url} timed out: {e}")
             except requests.ConnectionError:
                 logging.error(f"Connection error for URL {self.url_mgr.url}.")
             except requests.RequestException as e:
                 logging.error(f"Request exception for URL {self.url_mgr.url}: {e}")
-        try:
-            response = get_selenium_source(self.url_mgr.url)
-            if response:
-                self._response = response
-                self.status_code = 200  # Assume success
-                return self._response
-        except Exception as e:
-            logging.error(f"Failed to retrieve content from {self.url_mgr.url} after {self.max_retries} retries: {e}")
-            return None
+
+        logging.error(f"Failed to retrieve content from {self.url_mgr.url} after {self.max_retries} retries")
+        return None
 
     def try_request(self) -> requests.Response | str | bytes | None:
         """
-        Tries to make an HTTP request to the given URL using the provided session.
+        Tries Selenium first, then falls back to requests if Selenium fails.
         """
         if self.url_mgr.url is None:
             return None
+
+        # 1. Try Selenium
         try:
-            return get_selenium_source(self.url_mgr.url)  # or self.session.get(self.url_mgr.url, timeout=self.timeout, stream=self.stream)
+            return get_selenium_source(self.url_mgr.url)
+        except Exception as e:
+            logging.warning(f"Selenium failed for {self.url_mgr.url}, falling back to requests: {e}")
+
+        # 2. Fallback: requests
+        try:
+            resp = self.session.get(
+                self.url_mgr.url,
+                timeout=self.timeout or 10,
+                stream=self.stream
+            )
+            return resp
         except requests.RequestException as e:
-            logging.error(f"Request failed: {e}")
+            logging.error(f"Requests fallback also failed for {self.url_mgr.url}: {e}")
             return None
 
-    @property
-    def url(self):
-        return self.url_mgr.url
+        @property
+        def url(self):
+            return self.url_mgr.url
 
-    @url.setter
-    def url(self, new_url):
-        self._url = new_url
+        @url.setter
+        def url(self, new_url):
+            self._url = new_url
 class SafeRequestSingleton:
     _instance = None
     @staticmethod

abstract_webtools/managers/seleneumManager.py

@@ -1,34 +1,114 @@
-import os
-#from ..abstract_webtools import urlManager
-from .urlManager import *
-from urllib.parse import urlparse
-from abstract_utilities import *
+import os, time, re, json, logging, urllib3, requests,tempfile, shutil, socket, atexit, errno
+from urllib.parse import urlparse, urljoin
+from bs4 import BeautifulSoup          # if you prefer, keep using your parser
 from selenium import webdriver
 from selenium.webdriver.chrome.options import Options
-import logging
-import urllib3
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
 from abstract_security import get_env_value
-# Suppress urllib3 warnings and debug logs
+from abstract_utilities import *
+from .urlManager import *               # your urlManager
+
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 logging.getLogger("urllib3").setLevel(logging.WARNING)
-
-# Suppress Selenium logs
 logging.getLogger("selenium").setLevel(logging.WARNING)
 
-
-# Setup Chrome options
+# ---- Chrome options (keep yours; add safe fallbacks) ----
 chrome_options = Options()
-chrome_options.binary_location = get_env_value('CHROME_BINARY')
-chrome_options.add_argument("--headless")  # Run in headless mode
+_bin = get_env_value('CHROME_BINARY')
+if _bin:
+    chrome_options.binary_location = _bin
+chrome_options.add_argument("--headless=new")
 chrome_options.add_argument("--no-sandbox")
 chrome_options.add_argument("--disable-dev-shm-usage")
 chrome_options.add_argument("--disable-gpu")
 chrome_options.add_argument("--disable-software-rasterizer")
 chrome_options.add_argument("--disable-extensions")
 chrome_options.add_argument("--remote-debugging-port=9222")
+chrome_prefs = {"profile.managed_default_content_settings.images": 2}
+chrome_options.experimental_options["prefs"] = chrome_prefs
+
+MIN_HTML_BYTES = 2048  # tune: consider <2KB suspicious for real pages
+# --- NEW helpers: unique temp profile + free port + options builder ---
+
+def _free_port() -> int:
+    s = socket.socket()
+    s.bind(("127.0.0.1", 0))
+    port = s.getsockname()[1]
+    s.close()
+    return port
+
+def _make_profile_dir(base="/var/tmp/selenium-profiles") -> str:
+    os.makedirs(base, exist_ok=True)
+    return tempfile.mkdtemp(prefix="cw-", dir=base)
+
+def _make_chrome_options(binary_path: str | None = None,
+                         user_data_dir: str | None = None) -> tuple[Options, str]:
+    opts = Options()
+    if binary_path:
+        opts.binary_location = binary_path
+    opts.add_argument("--headless=new")
+    opts.add_argument("--no-sandbox")
+    opts.add_argument("--disable-dev-shm-usage")
+    opts.add_argument("--disable-gpu")
+    opts.add_argument("--disable-software-rasterizer")
+    opts.add_argument("--disable-extensions")
+
+    prof = user_data_dir or _make_profile_dir()
+    opts.add_argument(f"--user-data-dir={prof}")
+    opts.add_argument(f"--remote-debugging-port={_free_port()}")
+
+    prefs = {"profile.managed_default_content_settings.images": 2}
+    opts.add_experimental_option("prefs", prefs)
+    return opts, prof
+
 
+def _looks_like_html(text_or_bytes: bytes | str) -> bool:
+    if not text_or_bytes:
+        return False
+    s = text_or_bytes if isinstance(text_or_bytes, str) else text_or_bytes.decode("utf-8", "ignore")
+    if len(s) < MIN_HTML_BYTES:
+        return False
+    lowered = s.lower()
+    return ("<html" in lowered and "</html>" in lowered) or "<body" in lowered
 
+def _requests_fallback(url: str, headers: dict | None = None, timeout: float = 15.0):
+    """Plain requests fallback. Returns `requests.Response | None`."""
+    try:
+        sess = requests.Session()
+        sess.headers.update(headers or {"User-Agent": "Mozilla/5.0"})
+        # honor simple redirects and cert issues as needed
+        resp = sess.get(url, timeout=timeout, allow_redirects=True, verify=False)
+        return resp
+    except Exception as e:
+        logging.warning(f"requests fallback failed for {url}: {e}")
+        return None
 
+def _wait_until_ready(driver, timeout: float = 10.0):
+    """Waits for DOM readiness and presence of <body>."""
+    try:
+        WebDriverWait(driver, timeout).until(
+            lambda d: d.execute_script("return document.readyState") in ("interactive", "complete")
+        )
+    except Exception:
+        pass
+    try:
+        WebDriverWait(driver, timeout).until(EC.presence_of_element_located((By.TAG_NAME, "body")))
+    except Exception:
+        pass
+    # small settle delay for late JS injections
+    time.sleep(0.3)
+def normalize_url(url, base_url=None):
+    manager = seleniumManager(url)
+    base_url = manager.base_url
+    if url.startswith(base_url):
+        url = url[len(base_url):]
+    normalized_url = urljoin(base_url, url.split('#')[0])
+    if not normalized_url.startswith(base_url):
+        return None
+    return normalized_url
+# ---- Singleton driver manager (your class; small fixes) ----
 class SingletonMeta(type):
     _instances = {}
     def __call__(cls, *args, **kwargs):
@@ -39,78 +119,123 @@ class SingletonMeta(type):
 
 class seleniumManager(metaclass=SingletonMeta):
     def __init__(self, url):
-        if not hasattr(self, 'initialized'):  # Prevent reinitialization
-            self.initialized = True
-            parsed_url = urlparse(url)
-            self.domain = parsed_url.netloc
-            self.scheme = parsed_url.scheme
-            self.base_url= f"{self.scheme}{self.domain}"
-            self.site_dir = os.path.join(os.getcwd(), self.domain)
-            os.makedirs(self.site_dir, exist_ok=True)
-            self.drivers = {}
-            self.page_type = []
-    
+        if getattr(self, "initialized", False):
+            return
+        self.initialized = True
+
+        p = urlparse(url)
+        self.domain = p.netloc
+        self.scheme = p.scheme or "https"
+        self.base_url = f"{self.scheme}://{self.domain}"
+
+        self.site_dir = os.path.join("/var/tmp", "cw-sites", self.domain)
+        os.makedirs(self.site_dir, exist_ok=True)
+
+        self._sessions: dict[str, dict] = {}  # key -> {"driver": ..., "profile": ...}
+        atexit.register(lambda sm=self: sm.close_all())
+
     def get_url_to_path(self, url):
         url = eatAll(str(url), ['',' ','\n','\t','\\','/'])
-        parsed_url = urlparse(url)
-        if parsed_url.netloc == self.domain:
-            paths = parsed_url.path.split('/')
-            dir_path = self.site_dir
-            for path in paths[:-1]:
-                dir_path = os.path.join(dir_path, path)
-                os.makedirs(dir_path, exist_ok=True)
-            self.page_type.append(os.path.splitext(paths[-1])[-1] or 'html' if len(self.page_type) == 0 else self.page_type[-1])
-            
-            dir_path = os.path.join(dir_path, paths[-1])
-            return dir_path
-
-    def saved_url_check(self, url):
-        path = self.get_url_to_path(url)
-        return path
+        p = urlparse(url)
+        if p.netloc == self.domain:
+            parts = [x for x in p.path.split('/') if x]
+            d = self.site_dir
+            for seg in parts[:-1]:
+                d = os.path.join(d, seg)
+                os.makedirs(d, exist_ok=True)
+            last = parts[-1] if parts else "index.html"
+            ext = os.path.splitext(last)[-1] or ".html"
+            if not hasattr(self, "page_type"):
+                self.page_type = []
+            self.page_type.append(ext if not self.page_type else self.page_type[-1])
+            return os.path.join(d, last)
 
     def get_with_netloc(self, url):
-        parsed_url = urlparse(url)
-        if parsed_url.netloc == '':
-            url = f"{self.scheme}://{self.domain}/{url.strip()}"
+        p = urlparse(url)
+        if p.netloc == '':
+            url = f"{self.scheme}://{self.domain}/{url.strip().lstrip('/')}"
         return url
 
-    def get_driver(self, url):
-        if url and url not in self.drivers:
-#            chrome_options = Options()
-#            chrome_options.add_argument("--headless")
-            driver = webdriver.Chrome(options=chrome_options)
-            self.drivers[url] = driver
-            driver.get(url)
-        return self.drivers[url]
-def normalize_url(url, base_url=None):
-    """
-    Normalize and resolve relative URLs, ensuring proper domain and format.
-    """
-    # If URL starts with the base URL repeated, remove the extra part
-    manager = seleniumManager(url)
-    base_url = manager.base_url
-    if url.startswith(base_url):
-        url = url[len(base_url):]
+    def get_driver(self, url) -> tuple[str, webdriver.Chrome]:
+        bin_path = get_env_value('CHROME_BINARY')
+        opts, prof = _make_chrome_options(binary_path=bin_path, user_data_dir=None)
+        driver = webdriver.Chrome(options=opts)
+        key = f"{url}#{time.time()}"
+        self._sessions[key] = {"driver": driver, "profile": prof}
+        return key, driver
 
-    # Resolve the URL against the base URL
-    normalized_url = urljoin(base_url, url.split('#')[0])
+    def close_driver(self, key: str):
+        sess = self._sessions.pop(key, None)
+        if not sess: return
+        try:
+            try: sess["driver"].quit()
+            except Exception: pass
+        finally:
+            shutil.rmtree(sess.get("profile") or "", ignore_errors=True)
 
-    # Ensure only URLs belonging to the base domain are kept
-    if not normalized_url.startswith(base_url):
-        return None
+    def close_all(self):
+        for key in list(self._sessions.keys()):
+            self.close_driver(key)
 
-    return normalized_url
-# Function to get Selenium page source
-def get_selenium_source(url):
+
+
+# ---- Hardened page-source retrieval with fallback ----
+def get_selenium_source(url, max_retries: int = 2, request_fallback: bool = True, timeout: float = 12.0):
     url_mgr = urlManager(url)
-    if url_mgr.url:
-        url = str(url_mgr.url)
-        manager = seleniumManager(url)
-        driver = manager.get_driver(url)
-        try:
-            # Get page source
-            page_source = driver.page_source
-            return page_source
-        finally:
-            # Don't quit the driver unless you're done with all interactions
-            pass
+    if not url_mgr.url:
+        return None
+    url = str(url_mgr.url)
+
+    manager = seleniumManager(url)
+    key, driver = manager.get_driver(url)
+
+    last_exc = None
+    try:
+        for attempt in range(1, max_retries + 1):
+            try:
+                driver.get(url)
+                _wait_until_ready(driver, timeout=timeout)
+                html = driver.page_source or ""
+                if not _looks_like_html(html):
+                    html = driver.execute_script(
+                        "return document.documentElement ? document.documentElement.outerHTML : '';"
+                    ) or html
+                if _looks_like_html(html):
+                    return html
+                logging.warning(f"Selenium returned suspicious HTML (len={len(html)}) for {url} "
+                                f"[attempt {attempt}/{max_retries}]")
+            except Exception as e:
+                last_exc = e
+                logging.warning(f"Selenium attempt {attempt}/{max_retries} failed for {url}: {e}")
+            time.sleep(0.5 * attempt)
+
+        if request_fallback:
+            resp = _requests_fallback(url, headers={"User-Agent": "Mozilla/5.0"})
+            if resp is not None:
+                ctype = (resp.headers.get("content-type") or "").lower()
+                body = resp.text if hasattr(resp, "text") else (
+                    resp.content.decode("utf-8", "ignore") if hasattr(resp, "content") else ""
+                )
+                if "application/json" in ctype:
+                    try:
+                        return json.dumps(resp.json())
+                    except Exception:
+                        return body
+                return body if _looks_like_html(body) or body else None
+    finally:
+        # critical: release the user-data-dir to avoid “already in use”
+        manager.close_driver(key)
+
+    if last_exc:
+        logging.error(f"Unable to retrieve page for {url}: {last_exc}")
+    return None
+
+def get_driver(self, url):
+    # always new
+    bin_path = get_env_value('CHROME_BINARY')
+    opts, prof = _make_chrome_options(binary_path=bin_path, user_data_dir=None)
+    driver = webdriver.Chrome(options=opts)
+    # store so close_all() can clean up
+    key = f"{url}#{time.time()}"
+    self._sessions[key] = {"driver": driver, "profile": prof}
+    return driver

{abstract_webtools-0.1.6.144.dist-info → abstract_webtools-0.1.6.145.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: abstract_webtools
-Version: 0.1.6.144
+Version: 0.1.6.145
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff

{abstract_webtools-0.1.6.144.dist-info → abstract_webtools-0.1.6.145.dist-info}/RECORD

@@ -10,7 +10,7 @@ abstract_webtools/main.py,sha256=_I7pPXPkoLZOoYGLQDrSLGhGuQt6-PVyXEHZSmglk2g,132
 abstract_webtools/soup_gui.py,sha256=n95YAps1R6DpMwR4UbthSqQby0C5WHUa9tsW-f2qpLg,5184
 abstract_webtools/url_grabber.py,sha256=pnCCev7ZIuM-6cAGTLmK5HfzZg_AX-fLcRpB6ZE70B8,10441
 abstract_webtools/url_grabber_new.py,sha256=xb23qo4anOY0Ax3CAfaHJ8s5VEz61Sinh-XpEDFW7Is,3621
-abstract_webtools/managers/__init__.py,sha256=d7Q6_McRuKOHmKuna19s0l1wMgtM1JgUX8rHaSqJIcE,436
+abstract_webtools/managers/__init__.py,sha256=RXQAK5z9nYlocM91P2OC4jR352-MiqT5bAi4xZl7_FU,470
 abstract_webtools/managers/allss\.py,sha256=IBhlyRQHfK-BtwUnSEbIPqlI1MtZ8-XsdaHv0b91HQ0,269
 abstract_webtools/managers/cipherManager.py,sha256=NHQGdR11eNSm-1H-GezD5dyQgsPTJwY5kczt8Sher2s,1621
 abstract_webtools/managers/crawlManager.py,sha256=62Ej6AQC6-qXX_EWOmcJ2szNvEjmebFGugMz65HF1qI,12983
@@ -21,7 +21,7 @@ abstract_webtools/managers/dynamicRateLimiter.py,sha256=ycn5VQEPnmxjNMew4IVh-t5t
 abstract_webtools/managers/get_test.py,sha256=nISrhUGdyvRv18wTGoifGhizBFoHeK0N3FymMASloFw,825
 abstract_webtools/managers/mySocketClient.py,sha256=-j1Q8Ds9RCSbjZdx3ZF9mVpgwxaO0BBssanUcpYVQoY,2045
 abstract_webtools/managers/networkManager.py,sha256=Op2QDXrP-gmm0tCToe-Ryt9xuOtMppcN2KLKP1WZiu0,952
-abstract_webtools/managers/seleneumManager.py,sha256=1toMSoIPZmKwU88FMDTJl0DL398Zg_7uH-O1QqJpZC4,4184
+abstract_webtools/managers/seleneumManager.py,sha256=wyo4SpocgRz3W50b33GW3po32_uxYwmdE1TFZ_0k07s,9539
 abstract_webtools/managers/seleniumManager.py,sha256=RRpA1_oOnZuzzQ4S6VX7tDFcI31E_mOou2CZOOZH6yI,4274
 abstract_webtools/managers/sslManager.py,sha256=I9YUqJo8_KwLOwfBTAoSfzKSfR4Vtjw1HQXsXRnCV-g,641
 abstract_webtools/managers/tlsAdapter.py,sha256=XZSMZz9EUOhv-h3_Waf6mjV1dA3oN_M_oWuoo4VZ_HE,1454
@@ -39,14 +39,14 @@ abstract_webtools/managers/middleManager/src/UnifiedWebManager.py,sha256=qYCvfjU
 abstract_webtools/managers/middleManager/src/__init__.py,sha256=YaSAh7AG1EvFWFZBIe4pGvzmfr60rpR9ZDWoQKqAMd0,61
 abstract_webtools/managers/middleManager/src/legacy_tools.py,sha256=2cCnRaq8UO7HdtffNtAOsZFJm_mpZbpvBuX0pIIWGaM,125
 abstract_webtools/managers/requestManager/__init__.py,sha256=z2qGtweEoO_OKr959LGxVXEMu1hu7PIkmh89BEh5TI8,30
-abstract_webtools/managers/requestManager/requestManager.py,sha256=26BdfGrkWq2ouDaf0P8HTVK46PtPZJHUO46lIZgd8D8,19768
+abstract_webtools/managers/requestManager/requestManager.py,sha256=0d1Z5dFIjOg8KyJakzOilJiiq6SR3iKUr5vfnssWDu8,20048
 abstract_webtools/managers/soupManager/__init__.py,sha256=mqfXfqM9sWlYpOkoXUqtBoVvk2KQx1862NnmRVJwGtY,27
 abstract_webtools/managers/soupManager/asoueces.py,sha256=OaXqolZl0dI7b09NYwJ3Wnhuxf89ahZ1GjsOqy0GXfk,3506
 abstract_webtools/managers/soupManager/soupManager.py,sha256=75gwqVXIRwgVqzATBC-DiJF2AT_AdE6FSBWy3DbW5ZA,17393
 abstract_webtools/managers/urlManager/__init__.py,sha256=gaJCHeK91Z-eYsBnxgdhbIUten1-gbx-zqx70R6ag-Y,26
 abstract_webtools/managers/urlManager/urlManager (Copy).py,sha256=vCFuLADmv3h7icaaoAsImGqb_49VizPY_ZvMl-C7PYk,7756
 abstract_webtools/managers/urlManager/urlManager.py,sha256=vY4KQXtcrlC2YtlultxQpVe581l5kAuT5VGA0WrI16g,8945
-abstract_webtools-0.1.6.144.dist-info/METADATA,sha256=3pP4vVIzPwj649lj6QTC2wxt7yxsx4YRwl-iRkixR9M,7289
-abstract_webtools-0.1.6.144.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-abstract_webtools-0.1.6.144.dist-info/top_level.txt,sha256=2DMJ7RmjTcjCsa-uwAV0K6eXXlIIkFDEjBLg_uyCmCI,18
-abstract_webtools-0.1.6.144.dist-info/RECORD,,
+abstract_webtools-0.1.6.145.dist-info/METADATA,sha256=7eU_thbiawnNyvNUcQOBHclY44_tH3DikGbdnllhtXE,7289
+abstract_webtools-0.1.6.145.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+abstract_webtools-0.1.6.145.dist-info/top_level.txt,sha256=2DMJ7RmjTcjCsa-uwAV0K6eXXlIIkFDEjBLg_uyCmCI,18
+abstract_webtools-0.1.6.145.dist-info/RECORD,,