abstract-webtools 0.1.6.142__py3-none-any.whl → 0.1.6.143__py3-none-any.whl

abstract_webtools/managers/seleneumManager.py

@@ -0,0 +1,116 @@
+import os
+#from ..abstract_webtools import urlManager
+from .urlManager import *
+from urllib.parse import urlparse
+from abstract_utilities import *
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+import logging
+import urllib3
+from abstract_security import get_env_value
+# Suppress urllib3 warnings and debug logs
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+logging.getLogger("urllib3").setLevel(logging.WARNING)
+
+# Suppress Selenium logs
+logging.getLogger("selenium").setLevel(logging.WARNING)
+
+
+# Setup Chrome options
+chrome_options = Options()
+chrome_options.binary_location = get_env_value('CHROME_BINARY')
+chrome_options.add_argument("--headless")  # Run in headless mode
+chrome_options.add_argument("--no-sandbox")
+chrome_options.add_argument("--disable-dev-shm-usage")
+chrome_options.add_argument("--disable-gpu")
+chrome_options.add_argument("--disable-software-rasterizer")
+chrome_options.add_argument("--disable-extensions")
+chrome_options.add_argument("--remote-debugging-port=9222")
+
+
+
+class SingletonMeta(type):
+    _instances = {}
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            instance = super().__call__(*args, **kwargs)
+            cls._instances[cls] = instance
+        return cls._instances[cls]
+
+class seleniumManager(metaclass=SingletonMeta):
+    def __init__(self, url):
+        if not hasattr(self, 'initialized'):  # Prevent reinitialization
+            self.initialized = True
+            parsed_url = urlparse(url)
+            self.domain = parsed_url.netloc
+            self.scheme = parsed_url.scheme
+            self.base_url= f"{self.scheme}{self.domain}"
+            self.site_dir = os.path.join(os.getcwd(), self.domain)
+            os.makedirs(self.site_dir, exist_ok=True)
+            self.drivers = {}
+            self.page_type = []
+    
+    def get_url_to_path(self, url):
+        url = eatAll(str(url), ['',' ','\n','\t','\\','/'])
+        parsed_url = urlparse(url)
+        if parsed_url.netloc == self.domain:
+            paths = parsed_url.path.split('/')
+            dir_path = self.site_dir
+            for path in paths[:-1]:
+                dir_path = os.path.join(dir_path, path)
+                os.makedirs(dir_path, exist_ok=True)
+            self.page_type.append(os.path.splitext(paths[-1])[-1] or 'html' if len(self.page_type) == 0 else self.page_type[-1])
+            
+            dir_path = os.path.join(dir_path, paths[-1])
+            return dir_path
+
+    def saved_url_check(self, url):
+        path = self.get_url_to_path(url)
+        return path
+
+    def get_with_netloc(self, url):
+        parsed_url = urlparse(url)
+        if parsed_url.netloc == '':
+            url = f"{self.scheme}://{self.domain}/{url.strip()}"
+        return url
+
+    def get_driver(self, url):
+        if url and url not in self.drivers:
+#            chrome_options = Options()
+#            chrome_options.add_argument("--headless")
+            driver = webdriver.Chrome(options=chrome_options)
+            self.drivers[url] = driver
+            driver.get(url)
+        return self.drivers[url]
+def normalize_url(url, base_url=None):
+    """
+    Normalize and resolve relative URLs, ensuring proper domain and format.
+    """
+    # If URL starts with the base URL repeated, remove the extra part
+    manager = seleniumManager(url)
+    base_url = manager.base_url
+    if url.startswith(base_url):
+        url = url[len(base_url):]
+
+    # Resolve the URL against the base URL
+    normalized_url = urljoin(base_url, url.split('#')[0])
+
+    # Ensure only URLs belonging to the base domain are kept
+    if not normalized_url.startswith(base_url):
+        return None
+
+    return normalized_url
+# Function to get Selenium page source
+def get_selenium_source(url):
+    url_mgr = urlManager(url)
+    if url_mgr.url:
+        url = str(url_mgr.url)
+        manager = seleniumManager(url)
+        driver = manager.get_driver(url)
+        try:
+            # Get page source
+            page_source = driver.page_source
+            return page_source
+        finally:
+            # Don't quit the driver unless you're done with all interactions
+            pass

abstract_webtools/managers/sslManager.py

@@ -10,12 +10,3 @@ class SSLManager:
     def get_context(self):
         return ssl_.create_urllib3_context(ciphers=self.ciphers, cert_reqs=self.certification, options=self.ssl_options)
 
-class SSLManagerSingleton:
-    _instance = None
-    @staticmethod
-    def get_instance(ciphers=None, ssl_options_list=None, certification=None):
-        if SSLManagerSingleton._instance is None:
-            SSLManagerSingleton._instance = SSLManager(ciphers=ciphers, ssl_options_list=ssl_options_list, certification=certification)
-        elif SSLManagerSingleton._instance.cipher_manager.ciphers_string != ciphers or SSLManagerSingleton._instance.ssl_options_list !=ssl_options_list or SSLManagerSingleton._instance.certification !=certification:
-            SSLManagerSingleton._instance = SSLManager(ciphers=ciphers, ssl_options_list=ssl_options_list, certification=certification)
-        return SSLManagerSingleton._instance

{abstract_webtools-0.1.6.142.dist-info → abstract_webtools-0.1.6.143.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: abstract_webtools
-Version: 0.1.6.142
+Version: 0.1.6.143
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff

{abstract_webtools-0.1.6.142.dist-info → abstract_webtools-0.1.6.143.dist-info}/RECORD

@@ -21,8 +21,9 @@ abstract_webtools/managers/dynamicRateLimiter.py,sha256=ycn5VQEPnmxjNMew4IVh-t5t
 abstract_webtools/managers/get_test.py,sha256=nISrhUGdyvRv18wTGoifGhizBFoHeK0N3FymMASloFw,825
 abstract_webtools/managers/mySocketClient.py,sha256=-j1Q8Ds9RCSbjZdx3ZF9mVpgwxaO0BBssanUcpYVQoY,2045
 abstract_webtools/managers/networkManager.py,sha256=Op2QDXrP-gmm0tCToe-Ryt9xuOtMppcN2KLKP1WZiu0,952
+abstract_webtools/managers/seleneumManager.py,sha256=1toMSoIPZmKwU88FMDTJl0DL398Zg_7uH-O1QqJpZC4,4184
 abstract_webtools/managers/seleniumManager.py,sha256=RRpA1_oOnZuzzQ4S6VX7tDFcI31E_mOou2CZOOZH6yI,4274
-abstract_webtools/managers/sslManager.py,sha256=C-QgQw9CW84uOE5kx2MPjC3RsLbE2JQqdwdTs0H4ecc,1370
+abstract_webtools/managers/sslManager.py,sha256=I9YUqJo8_KwLOwfBTAoSfzKSfR4Vtjw1HQXsXRnCV-g,641
 abstract_webtools/managers/tlsAdapter.py,sha256=XZSMZz9EUOhv-h3_Waf6mjV1dA3oN_M_oWuoo4VZ_HE,1454
 abstract_webtools/managers/userAgentManager.py,sha256=cUaOlcCTzftVBCp9ZHwMXR9IB1wAE-03YSVwUBaIFLM,2514
 abstract_webtools/managers/videoDownloader.py,sha256=mKXhKYNnJwPaiqAsHvFTBGLdXFgR3wdV0G1OIimiKbE,15424
@@ -45,7 +46,7 @@ abstract_webtools/managers/soupManager/soupManager.py,sha256=75gwqVXIRwgVqzATBC-
 abstract_webtools/managers/urlManager/__init__.py,sha256=gaJCHeK91Z-eYsBnxgdhbIUten1-gbx-zqx70R6ag-Y,26
 abstract_webtools/managers/urlManager/urlManager (Copy).py,sha256=vCFuLADmv3h7icaaoAsImGqb_49VizPY_ZvMl-C7PYk,7756
 abstract_webtools/managers/urlManager/urlManager.py,sha256=vY4KQXtcrlC2YtlultxQpVe581l5kAuT5VGA0WrI16g,8945
-abstract_webtools-0.1.6.142.dist-info/METADATA,sha256=FXWOBxUiBcDXee0Dbf_JGQFsIZfI6caniOuWkEEB9P0,7289
-abstract_webtools-0.1.6.142.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-abstract_webtools-0.1.6.142.dist-info/top_level.txt,sha256=2DMJ7RmjTcjCsa-uwAV0K6eXXlIIkFDEjBLg_uyCmCI,18
-abstract_webtools-0.1.6.142.dist-info/RECORD,,
+abstract_webtools-0.1.6.143.dist-info/METADATA,sha256=E_BIW_Q1IeDMeemA0jKQF4nlGmwnPvIJTCVbyFplnTA,7289
+abstract_webtools-0.1.6.143.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+abstract_webtools-0.1.6.143.dist-info/top_level.txt,sha256=2DMJ7RmjTcjCsa-uwAV0K6eXXlIIkFDEjBLg_uyCmCI,18
+abstract_webtools-0.1.6.143.dist-info/RECORD,,