abstract-webtools 0.1.6.40__tar.gz → 0.1.6.41__tar.gz

{abstract_webtools-0.1.6.40/src/abstract_webtools.egg-info → abstract_webtools-0.1.6.41}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: abstract_webtools
-Version: 0.1.6.40
+Version: 0.1.6.41
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff

{abstract_webtools-0.1.6.40 → abstract_webtools-0.1.6.41}/setup.py

@@ -4,7 +4,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 setuptools.setup(
     name='abstract_webtools',
-    version='0.1.6.40',
+    version='0.1.6.41',
     author='putkoff',
     author_email='partners@abstractendeavors.com',
     description='Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.',

abstract_webtools-0.1.6.41/src/abstract_webtools/managers/seleniumManager.py

@@ -0,0 +1,119 @@
+import os
+from ..abstract_webtools import *
+from .urlManager import *
+from urllib.parse import urlparse
+from abstract_utilities import *
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+import logging
+import urllib3
+
+# Suppress urllib3 warnings and debug logs
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+logging.getLogger("urllib3").setLevel(logging.WARNING)
+
+# Suppress Selenium logs
+logging.getLogger("selenium").setLevel(logging.WARNING)
+
+import os
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+
+# Setup Chrome options
+chrome_options = Options()
+chrome_options.binary_location = "/home/profiles/solcatcher/.cache/selenium/chrome/linux64/130.0.6723.58/chrome"
+chrome_options.add_argument("--headless")  # Run in headless mode
+chrome_options.add_argument("--no-sandbox")
+chrome_options.add_argument("--disable-dev-shm-usage")
+chrome_options.add_argument("--disable-gpu")
+chrome_options.add_argument("--disable-software-rasterizer")
+chrome_options.add_argument("--disable-extensions")
+chrome_options.add_argument("--remote-debugging-port=9222")
+
+
+class SingletonMeta(type):
+    _instances = {}
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            instance = super().__call__(*args, **kwargs)
+            cls._instances[cls] = instance
+        return cls._instances[cls]
+
+class seleniumManager(metaclass=SingletonMeta):
+    def __init__(self, url):
+        if not hasattr(self, 'initialized'):  # Prevent reinitialization
+            self.initialized = True
+            parsed_url = urlparse(url)
+            self.domain = parsed_url.netloc
+            self.scheme = parsed_url.scheme
+            self.base_url= f"{self.scheme}{self.domain}"
+            self.site_dir = os.path.join(os.getcwd(), self.domain)
+            os.makedirs(self.site_dir, exist_ok=True)
+            self.drivers = {}
+            self.page_type = []
+    
+    def get_url_to_path(self, url):
+        url = eatAll(str(url), ['',' ','\n','\t','\\','/'])
+        parsed_url = urlparse(url)
+        if parsed_url.netloc == self.domain:
+            paths = parsed_url.path.split('/')
+            dir_path = self.site_dir
+            for path in paths[:-1]:
+                dir_path = os.path.join(dir_path, path)
+                os.makedirs(dir_path, exist_ok=True)
+            self.page_type.append(os.path.splitext(paths[-1])[-1] or 'html' if len(self.page_type) == 0 else self.page_type[-1])
+            
+            dir_path = os.path.join(dir_path, paths[-1])
+            return dir_path
+
+    def saved_url_check(self, url):
+        path = self.get_url_to_path(url)
+        return path
+
+    def get_with_netloc(self, url):
+        parsed_url = urlparse(url)
+        if parsed_url.netloc == '':
+            url = f"{self.scheme}://{self.domain}/{url.strip()}"
+        return url
+
+    def get_driver(self, url):
+        if url and url not in self.drivers:
+            chrome_options = Options()
+            chrome_options.add_argument("--headless")
+            driver = webdriver.Chrome(options=chrome_options)
+            self.drivers[url] = driver
+            driver.get(url)
+        return self.drivers[url]
+def normalize_url(url, base_url=None):
+    """
+    Normalize and resolve relative URLs, ensuring proper domain and format.
+    """
+    # If URL starts with the base URL repeated, remove the extra part
+    manager = seleniumManager(url)
+    base_url = manager.base_url
+    if url.startswith(base_url):
+        url = url[len(base_url):]
+
+    # Resolve the URL against the base URL
+    normalized_url = urljoin(base_url, url.split('#')[0])
+
+    # Ensure only URLs belonging to the base domain are kept
+    if not normalized_url.startswith(base_url):
+        return None
+
+    return normalized_url
+# Function to get Selenium page source
+def get_selenium_source(url):
+    url_mgr = urlManager(url)
+    if url_mgr.url:
+        url = str(url_mgr.url)
+        manager = seleniumManager(url)
+        driver = manager.get_driver(url)
+        try:
+            # Get page source
+            page_source = driver.page_source
+            return page_source
+        finally:
+            # Don't quit the driver unless you're done with all interactions
+            pass
+

{abstract_webtools-0.1.6.40 → abstract_webtools-0.1.6.41/src/abstract_webtools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: abstract_webtools
-Version: 0.1.6.40
+Version: 0.1.6.41
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff

abstract_webtools-0.1.6.40/src/abstract_webtools/managers/seleniumManager.py

@@ -1,128 +0,0 @@
-import os
-from urllib.parse import urlparse
-from selenium import webdriver
-from selenium.webdriver.chrome.options import Options
-from selenium.webdriver.chrome.service import Service
-from webdriver_manager.chrome import ChromeDriverManager  # For automatic ChromeDriver installation
-import logging
-import urllib3
-from ..abstract_webtools import *  # Assuming this is a valid import
-from .urlManager import *
-
-# Suppress urllib3 warnings and debug logs
-urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-logging.getLogger("urllib3").setLevel(logging.WARNING)
-logging.getLogger("selenium").setLevel(logging.WARNING)
-
-# Default Chrome options (can be overridden)
-DEFAULT_CHROME_OPTIONS = [
-    "--headless",  # Run in headless mode
-    "--no-sandbox",
-    "--disable-dev-shm-usage",  # Avoid memory issues on servers
-    "--disable-gpu",
-    "--disable-software-rasterizer",
-    "--disable-extensions",
-    "--remote-debugging-port=9222"
-]
-
-class SingletonMeta(type):
-    _instances = {}
-    def __call__(cls, *args, **kwargs):
-        if cls not in cls._instances:
-            instance = super().__call__(*args, **kwargs)
-            cls._instances[cls] = instance
-        return cls._instances[cls]
-
-class SeleniumManager(metaclass=SingletonMeta):
-    def __init__(self, url):
-        if not hasattr(self, 'initialized'):  # Prevent reinitialization
-            self.initialized = True
-            parsed_url = urlparse(url)
-            self.domain = parsed_url.netloc
-            self.scheme = parsed_url.scheme or "https"  # Default to https if scheme is missing
-            self.base_url = f"{self.scheme}://{self.domain}"
-            self.site_dir = os.path.join(os.getcwd(), self.domain)
-            os.makedirs(self.site_dir, exist_ok=True)
-            self.drivers = {}
-            self.page_type = []
-
-    def get_url_to_path(self, url):
-        url = eatAll(str(url), ['', ' ', '\n', '\t', '\\', '/'])  # Assuming eatAll is defined elsewhere
-        parsed_url = urlparse(url)
-        if parsed_url.netloc == self.domain:
-            paths = parsed_url.path.split('/')
-            dir_path = self.site_dir
-            for path in paths[:-1]:
-                dir_path = os.path.join(dir_path, path)
-                os.makedirs(dir_path, exist_ok=True)
-            self.page_type.append(os.path.splitext(paths[-1])[-1] or 'html' if not self.page_type else self.page_type[-1])
-            dir_path = os.path.join(dir_path, paths[-1])
-            return dir_path
-
-    def saved_url_check(self, url):
-        return self.get_url_to_path(url)
-
-    def get_with_netloc(self, url):
-        parsed_url = urlparse(url)
-        if not parsed_url.netloc:
-            url = f"{self.scheme}://{self.domain}/{url.strip('/')}"
-        return url
-
-    def get_driver(self, url):
-        if url and url not in self.drivers:
-            # Set up Chrome options
-            chrome_options = Options()
-            for option in DEFAULT_CHROME_OPTIONS:
-                chrome_options.add_argument(option)
-            
-            # Specify Chrome binary location if needed (optional, comment out if not applicable)
-            # chrome_options.binary_location = "/home/profiles/solcatcher/.cache/selenium/chrome/linux64/130.0.6723.58/chrome"
-
-            # Automatically install and use ChromeDriver
-            service = Service(ChromeDriverManager().install())
-            driver = webdriver.Chrome(service=service, options=chrome_options)
-            
-            self.drivers[url] = driver
-            driver.get(url)
-        return self.drivers[url]
-
-    def quit_driver(self, url):
-        """Clean up a specific driver instance."""
-        if url in self.drivers:
-            self.drivers[url].quit()
-            del self.drivers[url]
-
-    def quit_all_drivers(self):
-        """Clean up all driver instances."""
-        for driver in self.drivers.values():
-            driver.quit()
-        self.drivers.clear()
-
-def normalize_url(url, base_url=None):
-    """Normalize and resolve relative URLs."""
-    manager = SeleniumManager(url)
-    base_url = manager.base_url if base_url is None else base_url
-    if url.startswith(base_url):
-        url = url[len(base_url):]
-    normalized_url = urljoin(base_url, url.split('#')[0])
-    if not normalized_url.startswith(base_url):
-        return None
-    return normalized_url
-
-def get_selenium_source(url):
-    """Fetch page source using Selenium."""
-    url_mgr = urlManager(url)  # Assuming urlManager is defined elsewhere
-    if url_mgr.url:
-        url = str(url_mgr.url)
-        manager = SeleniumManager(url)
-        driver = manager.get_driver(url)
-        try:
-            return driver.page_source
-        except Exception as e:
-            logging.error(f"Error fetching page source for {url}: {e}")
-            return None
-        # Note: Driver is not quit here to maintain Singleton behavior
-
-# Ensure cleanup on program exit (optional)
-import atexit
-atexit.register(lambda: SeleniumManager(url="").quit_all_drivers())  # Cleanup all drivers on exit