abstract-webtools 0.1.6.39__py3-none-any.whl → 0.1.6.40__py3-none-any.whl

abstract_webtools/managers/seleniumManager.py

@@ -1,35 +1,29 @@
 import os
-from ..abstract_webtools import *
-from .urlManager import *
 from urllib.parse import urlparse
-from abstract_utilities import *
 from selenium import webdriver
 from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.chrome.service import Service
+from webdriver_manager.chrome import ChromeDriverManager  # For automatic ChromeDriver installation
 import logging
 import urllib3
+from ..abstract_webtools import *  # Assuming this is a valid import
+from .urlManager import *
 
 # Suppress urllib3 warnings and debug logs
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 logging.getLogger("urllib3").setLevel(logging.WARNING)
-
-# Suppress Selenium logs
 logging.getLogger("selenium").setLevel(logging.WARNING)
 
-import os
-from selenium import webdriver
-from selenium.webdriver.chrome.options import Options
-
-# Setup Chrome options
-chrome_options = Options()
-chrome_options.binary_location = "/home/profiles/solcatcher/.cache/selenium/chrome/linux64/130.0.6723.58/chrome"
-chrome_options.add_argument("--headless")  # Run in headless mode
-chrome_options.add_argument("--no-sandbox")
-chrome_options.add_argument("--disable-dev-shm-usage")
-chrome_options.add_argument("--disable-gpu")
-chrome_options.add_argument("--disable-software-rasterizer")
-chrome_options.add_argument("--disable-extensions")
-chrome_options.add_argument("--remote-debugging-port=9222")
-
+# Default Chrome options (can be overridden)
+DEFAULT_CHROME_OPTIONS = [
+    "--headless",  # Run in headless mode
+    "--no-sandbox",
+    "--disable-dev-shm-usage",  # Avoid memory issues on servers
+    "--disable-gpu",
+    "--disable-software-rasterizer",
+    "--disable-extensions",
+    "--remote-debugging-port=9222"
+]
 
 class SingletonMeta(type):
     _instances = {}
@@ -39,21 +33,21 @@ class SingletonMeta(type):
             cls._instances[cls] = instance
         return cls._instances[cls]
 
-class seleniumManager(metaclass=SingletonMeta):
+class SeleniumManager(metaclass=SingletonMeta):
     def __init__(self, url):
         if not hasattr(self, 'initialized'):  # Prevent reinitialization
             self.initialized = True
             parsed_url = urlparse(url)
             self.domain = parsed_url.netloc
-            self.scheme = parsed_url.scheme
-            self.base_url= f"{self.scheme}{self.domain}"
+            self.scheme = parsed_url.scheme or "https"  # Default to https if scheme is missing
+            self.base_url = f"{self.scheme}://{self.domain}"
             self.site_dir = os.path.join(os.getcwd(), self.domain)
             os.makedirs(self.site_dir, exist_ok=True)
             self.drivers = {}
             self.page_type = []
-    
+
     def get_url_to_path(self, url):
-        url = eatAll(str(url), ['',' ','\n','\t','\\','/'])
+        url = eatAll(str(url), ['', ' ', '\n', '\t', '\\', '/'])  # Assuming eatAll is defined elsewhere
         parsed_url = urlparse(url)
         if parsed_url.netloc == self.domain:
             paths = parsed_url.path.split('/')
@@ -61,59 +55,74 @@ class seleniumManager(metaclass=SingletonMeta):
             for path in paths[:-1]:
                 dir_path = os.path.join(dir_path, path)
                 os.makedirs(dir_path, exist_ok=True)
-            self.page_type.append(os.path.splitext(paths[-1])[-1] or 'html' if len(self.page_type) == 0 else self.page_type[-1])
-            
+            self.page_type.append(os.path.splitext(paths[-1])[-1] or 'html' if not self.page_type else self.page_type[-1])
             dir_path = os.path.join(dir_path, paths[-1])
             return dir_path
 
     def saved_url_check(self, url):
-        path = self.get_url_to_path(url)
-        return path
+        return self.get_url_to_path(url)
 
     def get_with_netloc(self, url):
         parsed_url = urlparse(url)
-        if parsed_url.netloc == '':
-            url = f"{self.scheme}://{self.domain}/{url.strip()}"
+        if not parsed_url.netloc:
+            url = f"{self.scheme}://{self.domain}/{url.strip('/')}"
         return url
 
     def get_driver(self, url):
         if url and url not in self.drivers:
+            # Set up Chrome options
             chrome_options = Options()
-            chrome_options.add_argument("--headless")
-            driver = webdriver.Chrome(options=chrome_options)
+            for option in DEFAULT_CHROME_OPTIONS:
+                chrome_options.add_argument(option)
+            
+            # Specify Chrome binary location if needed (optional, comment out if not applicable)
+            # chrome_options.binary_location = "/home/profiles/solcatcher/.cache/selenium/chrome/linux64/130.0.6723.58/chrome"
+
+            # Automatically install and use ChromeDriver
+            service = Service(ChromeDriverManager().install())
+            driver = webdriver.Chrome(service=service, options=chrome_options)
+            
             self.drivers[url] = driver
             driver.get(url)
         return self.drivers[url]
+
+    def quit_driver(self, url):
+        """Clean up a specific driver instance."""
+        if url in self.drivers:
+            self.drivers[url].quit()
+            del self.drivers[url]
+
+    def quit_all_drivers(self):
+        """Clean up all driver instances."""
+        for driver in self.drivers.values():
+            driver.quit()
+        self.drivers.clear()
+
 def normalize_url(url, base_url=None):
-    """
-    Normalize and resolve relative URLs, ensuring proper domain and format.
-    """
-    # If URL starts with the base URL repeated, remove the extra part
-    manager = seleniumManager(url)
-    base_url = manager.base_url
+    """Normalize and resolve relative URLs."""
+    manager = SeleniumManager(url)
+    base_url = manager.base_url if base_url is None else base_url
     if url.startswith(base_url):
         url = url[len(base_url):]
-
-    # Resolve the URL against the base URL
     normalized_url = urljoin(base_url, url.split('#')[0])
-
-    # Ensure only URLs belonging to the base domain are kept
     if not normalized_url.startswith(base_url):
         return None
-
     return normalized_url
-# Function to get Selenium page source
+
 def get_selenium_source(url):
-    url_mgr = urlManager(url)
+    """Fetch page source using Selenium."""
+    url_mgr = urlManager(url)  # Assuming urlManager is defined elsewhere
     if url_mgr.url:
         url = str(url_mgr.url)
-        manager = seleniumManager(url)
+        manager = SeleniumManager(url)
         driver = manager.get_driver(url)
         try:
-            # Get page source
-            page_source = driver.page_source
-            return page_source
-        finally:
-            # Don't quit the driver unless you're done with all interactions
-            pass
+            return driver.page_source
+        except Exception as e:
+            logging.error(f"Error fetching page source for {url}: {e}")
+            return None
+        # Note: Driver is not quit here to maintain Singleton behavior
 
+# Ensure cleanup on program exit (optional)
+import atexit
+atexit.register(lambda: SeleniumManager(url="").quit_all_drivers())  # Cleanup all drivers on exit

{abstract_webtools-0.1.6.39.dist-info → abstract_webtools-0.1.6.40.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: abstract_webtools
-Version: 0.1.6.39
+Version: 0.1.6.40
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff

{abstract_webtools-0.1.6.39.dist-info → abstract_webtools-0.1.6.40.dist-info}/RECORD

@@ -17,7 +17,7 @@ abstract_webtools/managers/linkManager.py,sha256=m6y9s8jknrTX8RtOAFKeHd4yd23G7Rg
 abstract_webtools/managers/mySocketClient.py,sha256=-j1Q8Ds9RCSbjZdx3ZF9mVpgwxaO0BBssanUcpYVQoY,2045
 abstract_webtools/managers/networkManager.py,sha256=Op2QDXrP-gmm0tCToe-Ryt9xuOtMppcN2KLKP1WZiu0,952
 abstract_webtools/managers/requestManager.py,sha256=zXD31WAYghV1OjnTQzRQnQGqZz6_J4mjHTdNLnBop_0,17343
-abstract_webtools/managers/seleniumManager.py,sha256=qSY8gH3N5YJIMwE_Alj9HNQRip_PziIo4_T9AZE_FQo,4273
+abstract_webtools/managers/seleniumManager.py,sha256=B7X6nTfxs1eHFDo7LKB1N5LhDytZQzHPgJjna2c2j6E,5017
 abstract_webtools/managers/soupManager.py,sha256=-_mRCWlyzfKlF64UU53WXBmCvJ98jQ4GyHh8S8Pw3xs,17198
 abstract_webtools/managers/sslManager.py,sha256=C-QgQw9CW84uOE5kx2MPjC3RsLbE2JQqdwdTs0H4ecc,1370
 abstract_webtools/managers/tlsAdapter.py,sha256=XZSMZz9EUOhv-h3_Waf6mjV1dA3oN_M_oWuoo4VZ_HE,1454
@@ -35,8 +35,8 @@ abstract_webtools/managers/soupManager/soupManager.py,sha256=U3_o189-OWoBRaSCe2s
 abstract_webtools/managers/urlManager/__init__.py,sha256=gaJCHeK91Z-eYsBnxgdhbIUten1-gbx-zqx70R6ag-Y,26
 abstract_webtools/managers/urlManager/urlManager.py,sha256=vCFuLADmv3h7icaaoAsImGqb_49VizPY_ZvMl-C7PYk,7756
 abstract_webtools/managers/videos/Heather brooke swallo from condom.mp4,sha256=h-bKFLAHt7pGLGu4EcMvSSox7BPRK0Nga3u813iMVKQ,8335544
-abstract_webtools-0.1.6.39.dist-info/LICENSE,sha256=g3WEJFiVS27HyCGRTwKSsMLyciMaGFdWcZGOe1QalZk,3877
-abstract_webtools-0.1.6.39.dist-info/METADATA,sha256=8sK_2myKFHmzx2eW1vsPGUvlyGWK19DX_tCI2PTP6gc,16051
-abstract_webtools-0.1.6.39.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
-abstract_webtools-0.1.6.39.dist-info/top_level.txt,sha256=2DMJ7RmjTcjCsa-uwAV0K6eXXlIIkFDEjBLg_uyCmCI,18
-abstract_webtools-0.1.6.39.dist-info/RECORD,,
+abstract_webtools-0.1.6.40.dist-info/LICENSE,sha256=g3WEJFiVS27HyCGRTwKSsMLyciMaGFdWcZGOe1QalZk,3877
+abstract_webtools-0.1.6.40.dist-info/METADATA,sha256=kKCrv_8-h4JuBeI1YGfB-sWkaQp-H5LS5IyPfw1ooUs,16051
+abstract_webtools-0.1.6.40.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
+abstract_webtools-0.1.6.40.dist-info/top_level.txt,sha256=2DMJ7RmjTcjCsa-uwAV0K6eXXlIIkFDEjBLg_uyCmCI,18
+abstract_webtools-0.1.6.40.dist-info/RECORD,,