abstract-webtools 0.1.6.146__py3-none-any.whl → 0.1.6.147__py3-none-any.whl

abstract_webtools/managers/requestManager/requestManager.py

@@ -4,7 +4,7 @@ from ..cipherManager import *
 from ..sslManager import *
 from ..tlsAdapter import *
 from ..networkManager import *
-from ..seleniumManager import *
+from ..seleneumManager import *
 from ..urlManager import *
 logging.basicConfig(level=logging.INFO)
 

abstract_webtools/managers/seleneumManager.py

@@ -239,3 +239,4 @@ def get_driver(self, url):
     key = f"{url}#{time.time()}"
     self._sessions[key] = {"driver": driver, "profile": prof}
     return driver
+seleneumManager = seleniumManager

abstract_webtools/managers/seleniumManager.py

@@ -0,0 +1,241 @@
+import os, time, re, json, logging, urllib3, requests,tempfile, shutil, socket, atexit, errno
+from urllib.parse import urlparse, urljoin
+from bs4 import BeautifulSoup          # if you prefer, keep using your parser
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from abstract_security import get_env_value
+from abstract_utilities import *
+from .urlManager import *               # your urlManager
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+logging.getLogger("urllib3").setLevel(logging.WARNING)
+logging.getLogger("selenium").setLevel(logging.WARNING)
+
+# ---- Chrome options (keep yours; add safe fallbacks) ----
+chrome_options = Options()
+_bin = get_env_value('CHROME_BINARY')
+if _bin:
+    chrome_options.binary_location = _bin
+chrome_options.add_argument("--headless=new")
+chrome_options.add_argument("--no-sandbox")
+chrome_options.add_argument("--disable-dev-shm-usage")
+chrome_options.add_argument("--disable-gpu")
+chrome_options.add_argument("--disable-software-rasterizer")
+chrome_options.add_argument("--disable-extensions")
+chrome_options.add_argument("--remote-debugging-port=9222")
+chrome_prefs = {"profile.managed_default_content_settings.images": 2}
+chrome_options.experimental_options["prefs"] = chrome_prefs
+
+MIN_HTML_BYTES = 2048  # tune: consider <2KB suspicious for real pages
+# --- NEW helpers: unique temp profile + free port + options builder ---
+
+def _free_port() -> int:
+    s = socket.socket()
+    s.bind(("127.0.0.1", 0))
+    port = s.getsockname()[1]
+    s.close()
+    return port
+
+def _make_profile_dir(base="/var/tmp/selenium-profiles") -> str:
+    os.makedirs(base, exist_ok=True)
+    return tempfile.mkdtemp(prefix="cw-", dir=base)
+
+def _make_chrome_options(binary_path: str | None = None,
+                         user_data_dir: str | None = None) -> tuple[Options, str]:
+    opts = Options()
+    if binary_path:
+        opts.binary_location = binary_path
+    opts.add_argument("--headless=new")
+    opts.add_argument("--no-sandbox")
+    opts.add_argument("--disable-dev-shm-usage")
+    opts.add_argument("--disable-gpu")
+    opts.add_argument("--disable-software-rasterizer")
+    opts.add_argument("--disable-extensions")
+
+    prof = user_data_dir or _make_profile_dir()
+    opts.add_argument(f"--user-data-dir={prof}")
+    opts.add_argument(f"--remote-debugging-port={_free_port()}")
+
+    prefs = {"profile.managed_default_content_settings.images": 2}
+    opts.add_experimental_option("prefs", prefs)
+    return opts, prof
+
+
+def _looks_like_html(text_or_bytes: bytes | str) -> bool:
+    if not text_or_bytes:
+        return False
+    s = text_or_bytes if isinstance(text_or_bytes, str) else text_or_bytes.decode("utf-8", "ignore")
+    if len(s) < MIN_HTML_BYTES:
+        return False
+    lowered = s.lower()
+    return ("<html" in lowered and "</html>" in lowered) or "<body" in lowered
+
+def _requests_fallback(url: str, headers: dict | None = None, timeout: float = 15.0):
+    """Plain requests fallback. Returns `requests.Response | None`."""
+    try:
+        sess = requests.Session()
+        sess.headers.update(headers or {"User-Agent": "Mozilla/5.0"})
+        # honor simple redirects and cert issues as needed
+        resp = sess.get(url, timeout=timeout, allow_redirects=True, verify=False)
+        return resp
+    except Exception as e:
+        logging.warning(f"requests fallback failed for {url}: {e}")
+        return None
+
+def _wait_until_ready(driver, timeout: float = 10.0):
+    """Waits for DOM readiness and presence of <body>."""
+    try:
+        WebDriverWait(driver, timeout).until(
+            lambda d: d.execute_script("return document.readyState") in ("interactive", "complete")
+        )
+    except Exception:
+        pass
+    try:
+        WebDriverWait(driver, timeout).until(EC.presence_of_element_located((By.TAG_NAME, "body")))
+    except Exception:
+        pass
+    # small settle delay for late JS injections
+    time.sleep(0.3)
+def normalize_url(url, base_url=None):
+    manager = seleniumManager(url)
+    base_url = manager.base_url
+    if url.startswith(base_url):
+        url = url[len(base_url):]
+    normalized_url = urljoin(base_url, url.split('#')[0])
+    if not normalized_url.startswith(base_url):
+        return None
+    return normalized_url
+# ---- Singleton driver manager (your class; small fixes) ----
+class SingletonMeta(type):
+    _instances = {}
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            instance = super().__call__(*args, **kwargs)
+            cls._instances[cls] = instance
+        return cls._instances[cls]
+
+class seleniumManager(metaclass=SingletonMeta):
+    def __init__(self, url):
+        if getattr(self, "initialized", False):
+            return
+        self.initialized = True
+
+        p = urlparse(url)
+        self.domain = p.netloc
+        self.scheme = p.scheme or "https"
+        self.base_url = f"{self.scheme}://{self.domain}"
+
+        self.site_dir = os.path.join("/var/tmp", "cw-sites", self.domain)
+        os.makedirs(self.site_dir, exist_ok=True)
+
+        self._sessions: dict[str, dict] = {}  # key -> {"driver": ..., "profile": ...}
+        atexit.register(lambda sm=self: sm.close_all())
+
+    def get_url_to_path(self, url):
+        url = eatAll(str(url), ['',' ','\n','\t','\\','/'])
+        p = urlparse(url)
+        if p.netloc == self.domain:
+            parts = [x for x in p.path.split('/') if x]
+            d = self.site_dir
+            for seg in parts[:-1]:
+                d = os.path.join(d, seg)
+                os.makedirs(d, exist_ok=True)
+            last = parts[-1] if parts else "index.html"
+            ext = os.path.splitext(last)[-1] or ".html"
+            if not hasattr(self, "page_type"):
+                self.page_type = []
+            self.page_type.append(ext if not self.page_type else self.page_type[-1])
+            return os.path.join(d, last)
+
+    def get_with_netloc(self, url):
+        p = urlparse(url)
+        if p.netloc == '':
+            url = f"{self.scheme}://{self.domain}/{url.strip().lstrip('/')}"
+        return url
+
+    def get_driver(self, url) -> tuple[str, webdriver.Chrome]:
+        bin_path = get_env_value('CHROME_BINARY')
+        opts, prof = _make_chrome_options(binary_path=bin_path, user_data_dir=None)
+        driver = webdriver.Chrome(options=opts)
+        key = f"{url}#{time.time()}"
+        self._sessions[key] = {"driver": driver, "profile": prof}
+        return key, driver
+
+    def close_driver(self, key: str):
+        sess = self._sessions.pop(key, None)
+        if not sess: return
+        try:
+            try: sess["driver"].quit()
+            except Exception: pass
+        finally:
+            shutil.rmtree(sess.get("profile") or "", ignore_errors=True)
+
+    def close_all(self):
+        for key in list(self._sessions.keys()):
+            self.close_driver(key)
+
+
+
+# ---- Hardened page-source retrieval with fallback ----
+def get_selenium_source(url, max_retries: int = 2, request_fallback: bool = True, timeout: float = 12.0):
+    url_mgr = urlManager(url)
+    if not url_mgr.url:
+        return None
+    url = str(url_mgr.url)
+
+    manager = seleniumManager(url)
+    key, driver = manager.get_driver(url)
+
+    last_exc = None
+    try:
+        for attempt in range(1, max_retries + 1):
+            try:
+                driver.get(url)
+                _wait_until_ready(driver, timeout=timeout)
+                html = driver.page_source or ""
+                if not _looks_like_html(html):
+                    html = driver.execute_script(
+                        "return document.documentElement ? document.documentElement.outerHTML : '';"
+                    ) or html
+                if _looks_like_html(html):
+                    return html
+                logging.warning(f"Selenium returned suspicious HTML (len={len(html)}) for {url} "
+                                f"[attempt {attempt}/{max_retries}]")
+            except Exception as e:
+                last_exc = e
+                logging.warning(f"Selenium attempt {attempt}/{max_retries} failed for {url}: {e}")
+            time.sleep(0.5 * attempt)
+
+        if request_fallback:
+            resp = _requests_fallback(url, headers={"User-Agent": "Mozilla/5.0"})
+            if resp is not None:
+                ctype = (resp.headers.get("content-type") or "").lower()
+                body = resp.text if hasattr(resp, "text") else (
+                    resp.content.decode("utf-8", "ignore") if hasattr(resp, "content") else ""
+                )
+                if "application/json" in ctype:
+                    try:
+                        return json.dumps(resp.json())
+                    except Exception:
+                        return body
+                return body if _looks_like_html(body) or body else None
+    finally:
+        # critical: release the user-data-dir to avoid “already in use”
+        manager.close_driver(key)
+
+    if last_exc:
+        logging.error(f"Unable to retrieve page for {url}: {last_exc}")
+    return None
+
+def get_driver(self, url):
+    # always new
+    bin_path = get_env_value('CHROME_BINARY')
+    opts, prof = _make_chrome_options(binary_path=bin_path, user_data_dir=None)
+    driver = webdriver.Chrome(options=opts)
+    # store so close_all() can clean up
+    key = f"{url}#{time.time()}"
+    self._sessions[key] = {"driver": driver, "profile": prof}
+    return driver

abstract_webtools/url_grabber.py

@@ -1,7 +1,11 @@
 from abstract_gui import make_component,sg
 import inspect
 import re
+from . import UserAgentManager,UrlManager,SafeRequest,SoupManager,LinkManager,CipherManager,requests,ssl,BeautifulSoup,HTTPAdapter,PoolManager,ssl_
+<<<<<<< HEAD
+=======
 from .managers import *
+>>>>>>> ba4baf2 (Deploy version 0.1.6.147 at 2025-09-07 09:40:38 UTC)
 window = None
 
 def get_attrs(values):
@@ -25,7 +29,11 @@ def get_attrs(values):
   else:
     tags_js['attribute']=tags_js['attribute'][0]
   return tags_js
+<<<<<<< HEAD
+def get_user_agent_manager(user_agent=None):
+=======
 def get_user_agent_mgr(user_agent=None):
+>>>>>>> ba4baf2 (Deploy version 0.1.6.147 at 2025-09-07 09:40:38 UTC)
   return UserAgentManager(user_agent=user_agent)
 def get_cipher_list():
   return CipherManager().get_default_ciphers()
@@ -128,17 +136,43 @@ def process_url(window,values):
     if warn_url=='' or warn_url == None:
       update_status(window,warn,warn_url,response_code,valid)
       return False
+    temp_url=UrlManager(url=warn_url).url
+    if temp_url:
+      valid='valid'
+      response_code = SafeRequest(url=temp_mgr).response.status_code
+<<<<<<< HEAD
+=======
     temp_url=urlManager(url=warn_url).url
     if temp_url:
       valid='valid'
       response_code = requestManager(url=temp_mgr).response.status_code
+>>>>>>> ba4baf2 (Deploy version 0.1.6.147 at 2025-09-07 09:40:38 UTC)
       warn = 'success'
       warn_url = temp_mgr
       update_status(window,warn,warn_url,response_code,valid)
       return temp_mgr
     update_status(window,warn,warn_url,response_code,valid)
     return False
-def update_url(url_mgr,request_mgr,soup_mgr,link_mgr,values,cipher_list=get_cipher_list(),user_agent=get_user_agents()[0]):
+def update_url(url_manager,request_manager,soup_manager,link_manager,values,cipher_list=get_cipher_list(),user_agent=get_user_agents()[0]):
+      ciphers = CipherManager(cipher_list=cipher_list).ciphers_string
+      request_manager = SafeRequest(url_manager=url_manager,ciphers=ciphers,user_agent=get_user_agents()[0])
+      if request_manager.source_code:
+        soup_manager= SoupManager(url_manager=url_manager,request_manager=request_manager)
+        link_manager= LinkManager(url_manager=url_manager,request_manager=request_manager,soup_manager=soup_manager)
+        window['-URL-'].update(value=url_manager.url)
+        window['-CIPHERS_OUTPUT-'].update(value=request_manager.ciphers)
+        return update_source_code(url_manager,request_manager,soup_manager,link_manager,values)
+      else:
+        return url_manager,request_manager,soup_manager,link_manager
+def update_source_code(url_manager,request_manager,soup_manager,link_manager,values):
+    parse_type = values['-parse_type-']
+    if parse_type != soup_manager.parse_type:
+      soup_manager.update_parse_type(parse_type=parse_type)
+    all_tags=soup_manager.get_all_tags_and_attribute_names()
+<<<<<<< HEAD
+    window['-SOURCECODE-'].update(value=soup_manager.soup)
+=======
+    window['-SOURCECODE-'].update(value=soup_manager.soupdef update_url(url_mgr,request_mgr,soup_mgr,link_mgr,values,cipher_list=get_cipher_list(),user_agent=get_user_agents()[0]):
       ciphers = CipherManager(cipher_list=cipher_list).ciphers_string
       request_mgr = requestManager(url_mgr=url_mgr,ciphers=ciphers,user_agent=get_user_agents()[0])
       if request_mgr.source_code:
@@ -155,12 +189,23 @@ def update_source_code(url_mgr,request_mgr,soup_mgr,link_mgr,values):
       soup_mgr.update_parse_type(parse_type=parse_type)
     all_tags=soup_mgr.get_all_tags_and_attribute_names()
     window['-SOURCECODE-'].update(value=soup_mgr.soup)
+>>>>>>> ba4baf2 (Deploy version 0.1.6.147 at 2025-09-07 09:40:38 UTC)
     if values['-SOUP_TAG-'] != all_tags['tags']:
       window['-SOUP_TAG-'].update(values=all_tags['tags'],value=all_tags['tags'][0])
     if values['-SOUP_ATTRIBUTE-'] != all_tags['attributes']:
       window['-SOUP_ATTRIBUTE-'].update(values=all_tags['attributes'],value=all_tags['attributes'][0])
       window['-SOUP_ATTRIBUTE_1-'].update(values=all_tags['attributes'],value=all_tags['attributes'][0])
       window['-SOUP_ATTRIBUTE_2-'].update(values=all_tags['attributes'],value=all_tags['attributes'][0])
+      return url_manager,request_manager,soup_manager,link_manager
+def url_grabber_while(window,initial_url="www.example.com"):
+    return_data=None
+    url_grab = False
+    url_manager=UrlManager(url=initial_url)
+    request_manager = SafeRequest(url_manager=url_manager)
+    soup_manager= SoupManager(url_manager=url_manager,request_manager=request_manager)
+    link_manager= LinkManager(url_manager=url_manager,request_manager=request_manager,soup_manager=soup_manager)
+<<<<<<< HEAD
+=======
       return url_mgr,request_mgr,soup_mgr,link_mgr
 def url_grabber_while(window,initial_url="www.example.com"):
     return_data=None
@@ -169,12 +214,25 @@ def url_grabber_while(window,initial_url="www.example.com"):
     request_mgr = requestManager(url_mgr=url_mgr)
     soup_mgr= SoupManager(url_mgr=url_mgr,request_mgr=request_mgr)
     link_mgr= LinkManager(url_mgr=url_mgr,request_mgr=request_mgr,soup_mgr=soup_mgr)
+>>>>>>> ba4baf2 (Deploy version 0.1.6.147 at 2025-09-07 09:40:38 UTC)
     while True:
         event, values = window.read()
         if event == sg.WINDOW_CLOSED:
             break
         if event=='-GRAB_URL-' or not url_grab:
           url=values['-URL-']
+          if UrlManager(url=url).url:
+            if url != url_manager.url or url == initial_url:
+              url_manager = UrlManager(url=url)
+              
+              url_manager,request_manager,soup_manager,link_manager=update_url(url_manager=url_manager,request_manager=request_manager,soup_manager=soup_manager,link_manager=link_manager,values=values)
+              window['-URL-'].update(value=url_manager.url)
+              url_grab=True
+        if event == 'get soup':
+            tags_js = get_attrs(values)
+            all_desired=soup_manager.find_tags_by_attributes(tag=tags_js['tag'], attr=tags_js['attribute'],attr_values=tags_js['input'])
+<<<<<<< HEAD
+=======
           if urlManager(url=url).url:
             if url != url_mgr.url or url == initial_url:
               url_mgr = urlManager(url=url)
@@ -185,21 +243,35 @@ def url_grabber_while(window,initial_url="www.example.com"):
         if event == 'get soup':
             tags_js = get_attrs(values)
             all_desired=soup_mgr.find_tags_by_attributes(tag=tags_js['tag'], attr=tags_js['attribute'],attr_values=tags_js['input'])
+>>>>>>> ba4baf2 (Deploy version 0.1.6.147 at 2025-09-07 09:40:38 UTC)
             window['-FIND_ALL_OUTPUT-'].update(value=all_desired)
         if event == '-CUSTOMUA-':
             window['-SOURCECODE-'].update(disabled=values['-CUSTOMUA-'])
             if not values['-CUSTOMUA-']:
+                window['-USERAGENT-'].update(value=user_agent_manager.user_agent_header)
+<<<<<<< HEAD
+=======
                 window['-USERAGENT-'].update(value=user_agent_mgr.user_agent_header)
+>>>>>>> ba4baf2 (Deploy version 0.1.6.147 at 2025-09-07 09:40:38 UTC)
                 window['-USERAGENT-'].update(disabled=True)
             else:
                 window['-USERAGENT-'].update(disabled=False)
         if event=='Get All Text':
+            window['-FIND_ALL_OUTPUT-'].update(value=soup_manager.extract_text_sections())
+        if event == 'Action':
+            parse_type = values['-parse_type-']
+            if parse_type != soup_manager.parse_type:
+              soup_manager.update_parse_type(parse_type=parse_type)
+            window['-SOURCECODE-'].update(value=soup_manager.soup)
+<<<<<<< HEAD
+=======
             window['-FIND_ALL_OUTPUT-'].update(value=soup_mgr.extract_text_sections())
         if event == 'Action':
             parse_type = values['-parse_type-']
             if parse_type != soup_mgr.parse_type:
               soup_mgr.update_parse_type(parse_type=parse_type)
             window['-SOURCECODE-'].update(value=soup_mgr.soup)
+>>>>>>> ba4baf2 (Deploy version 0.1.6.147 at 2025-09-07 09:40:38 UTC)
         elif event == 'Send Soup':
           return_data = values['-FIND_ALL_OUTPUT-']
           break