abstract-webtools 0.1.5.90__tar.gz → 0.1.5.92__tar.gz

{abstract_webtools-0.1.5.90/src/abstract_webtools.egg-info → abstract_webtools-0.1.5.92}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: abstract_webtools
-Version: 0.1.5.90
+Version: 0.1.5.92
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff

{abstract_webtools-0.1.5.90 → abstract_webtools-0.1.5.92}/setup.py

@@ -4,7 +4,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 setuptools.setup(
     name='abstract_webtools',
-    version='0.1.5.90',
+    version='0.1.5.92',
     author='putkoff',
     author_email='partners@abstractendeavors.com',
     description='Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.',

abstract_webtools-0.1.5.92/src/abstract_webtools/managers/domainManager.py

@@ -0,0 +1,100 @@
+from abstract_webtools import *
+import os
+from abstract_webtools import *
+#from .urlManager import *
+from urllib.parse import urlparse
+from abstract_utilities import *
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+import logging
+import urllib3
+
+# Suppress urllib3 warnings and debug logs
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+logging.getLogger("urllib3").setLevel(logging.WARNING)
+
+# Suppress Selenium logs
+logging.getLogger("selenium").setLevel(logging.WARNING)
+
+import os
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+
+# Setup Chrome options
+chrome_options = Options()
+chrome_options.binary_location = "/home/profiles/solcatcher/.cache/selenium/chrome/linux64/130.0.6723.58/chrome"
+chrome_options.add_argument("--headless")  # Run in headless mode
+chrome_options.add_argument("--no-sandbox")
+chrome_options.add_argument("--disable-dev-shm-usage")
+chrome_options.add_argument("--disable-gpu")
+chrome_options.add_argument("--disable-software-rasterizer")
+chrome_options.add_argument("--disable-extensions")
+chrome_options.add_argument("--remote-debugging-port=9222")
+
+
+class domainManager(metaclass=SingletonMeta):
+    def __init__(self, url):
+        if not hasattr(self, 'initialized'):  # Prevent reinitialization
+            self.initialized = True
+            parsed_url = urlparse(url)
+            self.domain = parsed_url.netloc
+            self.scheme = parsed_url.scheme
+            self.site_dir = os.path.join(os.getcwd(), self.domain)
+            os.makedirs(self.site_dir, exist_ok=True)
+            self.drivers = {}
+            self.page_type = []
+    def get_url_to_path(self, url):
+        url = eatAll(str(url),['',' ','\n','\t','\\','/'])
+        parsed_url = urlparse(url)
+        if 'data:image' in url:
+            input(url)
+        if parsed_url.netloc == self.domain:
+            paths = parsed_url.path.split('/')
+            dir_path =self.site_dir
+            for path in paths[:-1]:
+                dir_path = os.path.join(dir_path, path)
+                os.makedirs(dir_path, exist_ok=True)
+        #if 'svg' in url:
+        #$    input(url)
+         #   dir_path = get_image_name('contents',directory=dir_path,ext='png',url=item_url)
+
+
+            self.page_type.append(os.path.splitext(paths[-1])[-1] or 'html' if len(self.page_type) == 0 else self.page_type[-1])
+            
+            dir_path = os.path.join(dir_path, paths[-1])
+            return dir_path
+
+    def saved_url_check(self, url):
+  
+        path = self.get_url_to_path(url)
+        return path
+
+    def get_with_netloc(self, url):
+        parsed_url = urlparse(url)
+        if parsed_url.netloc == '':
+            url = f"{self.scheme}://{self.domain}/{url.strip()}"
+        return url
+
+    def get_driver(self, url):
+        if url and url not in self.drivers:
+            chrome_options = Options()
+            chrome_options.add_argument("--headless")
+            driver = webdriver.Chrome(options=chrome_options)
+            self.drivers[url] = driver
+            driver.get(url)
+        return self.drivers[url]
+def get_selenium_source(url):
+    url_mgr = urlManager(url)
+    if url_mgr.url:
+        url = str(url_mgr.url)
+        manager = domainManager(url)
+        driver = manager.get_driver(url)
+        try:
+            # Get page source
+            page_source = driver.page_source
+            return page_source
+        finally:
+            # Don't quit the driver unless you're done with all interactions
+            pass
+driver = get_selenium_source('http://solpump.io/')
+input(driver)

{abstract_webtools-0.1.5.90 → abstract_webtools-0.1.5.92}/src/abstract_webtools/managers/requestManager.py

@@ -1,11 +1,12 @@
 from ..abstract_webtools import *
-from .urlManager import *
+
 from .userAgentManager import *
 from .cipherManager import *
 from .sslManager import *
 from .tlsAdapter import *
 from .networkManager import *
 from .seleniumManager import *
+from .urlManager import *
 class requestManager:
     """
     SafeRequest is a class for making HTTP requests with error handling and retries.
@@ -340,12 +341,10 @@ class SafeRequestSingleton:
         elif SafeRequestSingleton._instance.url != url or SafeRequestSingleton._instance.headers != headers or SafeRequestSingleton._instance.max_retries != max_retries or SafeRequestSingleton._instance.request_wait_limit != request_wait_limit:
             SafeRequestSingleton._instance = SafeRequest(url,url_mgr=urlManagerSingleton,headers=headers,max_retries=max_retries,last_request_time=last_request_time,request_wait_limit=request_wait_limit)
         return SafeRequestSingleton._instance
-def get_req_mgr(req_mgr=None,url=None,url_mgr=None,source_code=None):
-    if req_mgr:
-        url_mgr = req_mgr.url_mgr
+def get_req_mgr(url=None,url_mgr=None,source_code=None):
     url = get_url(url=url,url_mgr=url_mgr)
-    url_mgr = get_url_mgr(url=url,url_mgr=url_mgr)
-    req_mgr = req_mgr or requestManager(url_mgr=url_mgr,url=url,source_code=source_code)
+    url_mgr = get_url_mgr(url=url,url_mgr=url_mgr )
+    req_mgr = requestManager(url_mgr=url_mgr,url=url,source_code=source_code)
     return req_mgr
 def get_source(url=None,url_mgr=None,source_code=None):
     # Placeholder for actual implementation.

{abstract_webtools-0.1.5.90 → abstract_webtools-0.1.5.92}/src/abstract_webtools/managers/seleniumManager.py

@@ -39,7 +39,7 @@ class SingletonMeta(type):
             cls._instances[cls] = instance
         return cls._instances[cls]
 
-class domainManager(metaclass=SingletonMeta):
+class seleneumManager(metaclass=SingletonMeta):
     def __init__(self, url):
         if not hasattr(self, 'initialized'):  # Prevent reinitialization
             self.initialized = True
@@ -77,6 +77,8 @@ class domainManager(metaclass=SingletonMeta):
 
     def get_driver(self, url):
         if url and url not in self.drivers:
+            chrome_options = Options()
+            chrome_options.add_argument("--headless")
             driver = webdriver.Chrome(options=chrome_options)
             self.drivers[url] = driver
             driver.get(url)
@@ -96,3 +98,4 @@ def get_selenium_source(url):
         finally:
             # Don't quit the driver unless you're done with all interactions
             pass
+

{abstract_webtools-0.1.5.90 → abstract_webtools-0.1.5.92/src/abstract_webtools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: abstract_webtools
-Version: 0.1.5.90
+Version: 0.1.5.92
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff

abstract_webtools-0.1.5.90/src/abstract_webtools/managers/domainManager.py

@@ -1,48 +0,0 @@
-from ..abstract_webtools import *
-class domainManager(metaclass=SingletonMeta):
-    def __init__(self, url):
-        if not hasattr(self, 'initialized'):  # Prevent reinitialization
-            self.initialized = True
-            parsed_url = urlparse(url)
-            self.domain = parsed_url.netloc
-            self.scheme = parsed_url.scheme
-    def get_url_to_path(self, url):
-        url = eatAll(str(url),['',' ','\n','\t','\\','/'])
-        parsed_url = urlparse(url)
-        if 'data:image' in url:
-            input(url)
-        if parsed_url.netloc == self.domain:
-            paths = parsed_url.path.split('/')
-            dir_path =self.site_dir
-            for path in paths[:-1]:
-                dir_path = os.path.join(dir_path, path)
-                os.makedirs(dir_path, exist_ok=True)
-        #if 'svg' in url:
-        #$    input(url)
-         #   dir_path = get_image_name('contents',directory=dir_path,ext='png',url=item_url)
-
-
-            self.page_type.append(os.path.splitext(paths[-1])[-1] or 'html' if len(self.page_type) == 0 else self.page_type[-1])
-            
-            dir_path = os.path.join(dir_path, paths[-1])
-            return dir_path
-
-    def saved_url_check(self, url):
-  
-        path = self.get_url_to_path(url)
-        return path
-
-    def get_with_netloc(self, url):
-        parsed_url = urlparse(url)
-        if parsed_url.netloc == '':
-            url = f"{self.scheme}://{self.domain}/{url.strip()}"
-        return url
-
-    def get_driver(self, url):
-        if url and url not in self.drivers:
-            chrome_options = Options()
-            chrome_options.add_argument("--headless")
-            driver = webdriver.Chrome(options=chrome_options)
-            self.drivers[url] = driver
-            driver.get(url)
-        return self.drivers[url]