abstract-webtools 0.1.4.1__py3-none-any.whl → 0.1.4.12__py3-none-any.whl

abstract_webtools/abstract_crawler.py

@@ -1,4 +1,5 @@
-from .abstract_webtools import *
+from abstract_webtools import *
+response = SafeRequest()
 def discover_classes_with_links(base_url):
     """
     Discovers classes in the HTML content of the provided URL 
@@ -10,9 +11,9 @@ def discover_classes_with_links(base_url):
     Returns:
         set: A set of unique class names.
     """
-    response = SafeRequest().get_source_code(url=base_url)
+    
     if response:
-        soup = BeautifulSoup(response.content, 'html.parser')
+        soup = BeautifulSoup(response, 'html.parser')
     
         unique_classes = set()
 
@@ -228,4 +229,5 @@ def generate_sitemap(domain):
 
 if __name__ == '__main__':
     domain = "https://uuvo.com"
+    request_manager = SafeRequest(url=domain)
     generate_sitemap(domain)

abstract_webtools/abstract_webtools.py

@@ -69,6 +69,7 @@ Version: 1.0
 import ssl
 import requests
 from requests.adapters import HTTPAdapter
+from typing import Optional, List
 from requests.packages.urllib3.poolmanager import PoolManager
 from requests.packages.urllib3.util import ssl_
 from urllib.parse import urlparse
@@ -196,7 +197,6 @@ class DynamicRateLimiter:
         print(f"Adjusted tokens to: {self.tokens} and high_limit to: {self.current_limit}")
 class DynamicRateLimiterManagerSingleton:
     _instance = None
-    
     @staticmethod
     def get_instance():
         if DynamicRateLimiterManagerSingleton._instance is None:
@@ -205,7 +205,7 @@ class DynamicRateLimiterManagerSingleton:
 
 class CipherManager:
     @staticmethod
-    def get_default_ciphers() -> list:
+    def  get_default_ciphers()-> list:
         return [
             "ECDHE-RSA-AES256-GCM-SHA384", "ECDHE-ECDSA-AES256-GCM-SHA384",
             "ECDHE-RSA-AES256-SHA384", "ECDHE-ECDSA-AES256-SHA384",
@@ -215,19 +215,32 @@ class CipherManager:
             "AES256-SHA", "AES128-SHA"
         ]
 
+    def __init__(self,cipher_list=None):
+        if cipher_list == None:
+            cipher_list=self.get_default_ciphers()
+        self.cipher_list = cipher_list
+        self.create_list()
+        self.ciphers_string = self.add_string_list()
+    def add_string_list(self):
+        if len(self.cipher_list)==0:
+            return ''
+        return','.join(self.cipher_list)
+    def create_list(self):
+        if self.cipher_list == None:
+            self.cipher_list= []
+        elif isinstance(self.cipher_list, str):
+            self.cipher_list=self.cipher_list.split(',')
+        if isinstance(self.cipher_list, str):
+            self.cipher_list=[self.cipher_list]
+class CipherManagerSingleton:
+    _instance = None
     @staticmethod
-    def create_list(lst=None):
-        if not lst:
-            return []
-        elif isinstance(lst, str):
-            return lst.split(',')
-        return lst
-
-    @staticmethod
-    def add_string_list(cipher_list=[], delim=','):
-        return delim.join(cipher_list)
-
-
+    def get_instance(cipher_list=None):
+        if CipherManagerSingleton._instance is None:
+            CipherManagerSingleton._instance = CipherManager(cipher_list=cipher_list)
+        elif CipherManagerSingleton._instance.cipher_list != cipher_list:
+            CipherManagerSingleton._instance = CipherManager(cipher_list=cipher_list)
+        return CipherManagerSingleton._instance
 class SSLManager:
     @staticmethod
     def get_default_certification():
@@ -320,40 +333,70 @@ class SSLManager:
             "OP_ALL"
             ]
 
-    @staticmethod
-    def create_list(lst=None):
-        return CipherManager.create_list(lst)
-
-    @staticmethod
-    def combine_ssl_options(ssl_options_values=[]):
-        combined_options = 0
-        for option in ssl_options_values:
-            combined_options |= option
-        return combined_options
-
-    @staticmethod
-    def get_options_values(ssl_options_list=[]):
-        return [getattr(ssl, option_name) for option_name in ssl_options_list]
-
     @staticmethod
     def get_context(ciphers=None, options=None, cert_reqs=None):
+        
         return ssl_.create_urllib3_context(ciphers=ciphers, cert_reqs=cert_reqs, options=options)
 
     def __init__(self, ciphers=None, ssl_options_list=None, certification=None):
-        self.ssl_options_list = self.create_list(ssl_options_list or self.get_default_tls_options())
-        self.ssl_options_values = self.get_options_values(self.ssl_options_list)
-        self.ssl_options = self.combine_ssl_options(self.ssl_options_values)
+        self.ssl_options_list = ssl_options_list
+        self.create_list()
+        self.ssl_options_values = self.get_options_values()
+        self.ssl_options = self.combine_ssl_options()
         self.certification = certification or self.get_default_certification()
-        self.ssl_context = self.get_context(ciphers=ciphers, options=self.ssl_options, cert_reqs=self.certification)
+        self.cipher_manager = CipherManagerSingleton().get_instance(cipher_list=ciphers)
+        self.ssl_context = self.get_context(ciphers=self.cipher_manager.ciphers_string, options=self.ssl_options, cert_reqs=self.certification)
+    def create_list(self):
+        if self.ssl_options_list == None:
+            self.ssl_options_list= []
+        elif isinstance(self.ssl_options_list, str):
+            self.ssl_options_list=self.ssl_options_list.split(',')
+        if isinstance(self.ssl_options_list, str):
+            self.ssl_options_list=[self.ssl_options_list]
+    def get_options_values(self):
+        return [getattr(ssl, option_name) for option_name in self.ssl_options_list]
+    def combine_ssl_options(self):
+        combined_options = 0
+        for option in self.ssl_options_values:
+            combined_options |= option
+        return combined_options
+class SSLManagerSingleton:
+    _instance = None
+    @staticmethod
+    def get_instance(ciphers=None, ssl_options_list=None, certification=None):
+        if SSLManagerSingleton._instance is None:
+            SSLManagerSingleton._instance = SSLManager(ciphers=ciphers, ssl_options_list=ssl_options_list, certification=certification)
+        elif SSLManagerSingleton._instance.cipher_manager.ciphers_string != ciphers or SSLManagerSingleton._instance.ssl_options_list !=ssl_options_list or SSLManagerSingleton._instance.certification !=certification:
+            SSLManagerSingleton._instance = SSLManager(ciphers=ciphers, ssl_options_list=ssl_options_list, certification=certification)
+        return SSLManagerSingleton._instance
 class TLSAdapter(HTTPAdapter):
-    def __init__(self, ciphers=None, certification=None, ssl_options=None):
-        ssl_manager = SSLManager(ciphers, ssl_options, certification)
-        self.ssl_context = ssl_manager.ssl_context
+    def __init__(self, ciphers: Optional[List[str]] = None, certification: Optional[str] = None, ssl_options: Optional[List[str]] = None):
         super().__init__()
+        self.ciphers = ciphers
+        self.certification = certification
+        self.ssl_options = ssl_options
+        
+        self.cipher_manager = CipherManagerSingleton.get_instance(cipher_list=self.ciphers)
+        self.ssl_manager = SSLManagerSingleton.get_instance(
+            ciphers=self.cipher_manager.ciphers_string, 
+            ssl_options_list=ssl_options, 
+            certification=certification
+        )
+        self.ssl_context = self.ssl_manager.ssl_context
 
     def init_poolmanager(self, *args, **kwargs):
-        kwargs['ssl_context'] = self.ssl_context
         return super().init_poolmanager(*args, **kwargs)
+
+
+class TLSAdapterSingleton:
+    _instance = None
+    @staticmethod
+    def get_instance(ciphers=None, certification=None, ssl_options=None):
+        if TLSAdapterSingleton._instance is None:
+            TLSAdapterSingleton._instance = TLSAdapter(ciphers=ciphers, certification=certification, ssl_options=ssl_options)
+        elif TLSAdapterSingleton._instance.ciphers != ciphers or SSLManagerSingleton._instance.certification !=certification or SSLManagerSingleton._instance.ssl_options_list !=ssl_options:
+            TLSAdapterSingleton._instance = TLSAdapter(ciphers=ciphers, certification=certification, ssl_options=ssl_options)
+        return TLSAdapterSingleton._instance
 class UserAgentManager:
     @staticmethod
     def desktop_user_agents() -> list:
@@ -382,39 +425,49 @@ class UserAgentManager:
         """
         return {"user-agent": user_agent}
     def __init__(self,user_agent=desktop_user_agents()[0]):
-        self.user_agent=self.get_user_agent(user_agent=user_agent)
+        self.user_agent = user_agent
+        self.user_agent_header=self.get_user_agent(user_agent=user_agent)
+class UserAgentManagerSingleton:
+    _instance = None
+    @staticmethod
+    def get_instance(user_agent=UserAgentManager.desktop_user_agents()[0]):
+        if UserAgentManagerSingleton._instance is None:
+            UserAgentManagerSingleton._instance = UserAgentManager(user_agent=user_agent)
+        elif UserAgentManagerSingleton._instance.user_agent != user_agent:
+            UserAgentManagerSingleton._instance = UserAgentManager(user_agent=user_agent)
+        return UserAgentManagerSingleton._instance
 class SafeRequest:
     def __init__(self,
                  url=None,
-                 headers:dict=UserAgentManager().user_agent,
+                 headers:dict=UserAgentManagerSingleton().get_instance().user_agent_header,
                  max_retries=3,
                  last_request_time=None,
                  request_wait_limit=1.5,
                  ):
+        self.url =url 
         if isinstance(headers,str):
-            get_user_agent(headers)
+            headers = UserAgentManagerSingleton().get_instance(user_agent=headers).user_agent_header
         self.headers = headers
+        self.max_retries=max_retries
+        
+        self.request_wait_limit = request_wait_limit
+        
+        self.url_manager = URLManagerSingleton().get_instance(url=url)
         self.session = self.initialize_session()
-        self.max_retries = max_retries
+        
         self.last_request_time = last_request_time
-        self.request_wait_limit = request_wait_limit
-        if url == None:
-            return
-        self.url = url
-        self.url_manager = URLManager(url=self.url)
+        
         self.response = self.make_request()
-        self.status_code = self.response.status_code
-        self.source_code = self.response.text if self.response else None
         
-    def initialize_session(self,user_agent:str= None):
-        if isinstance(user_agent,str):
-            user_agent = get_user_agent(user_agent)
+        self.status_code = None if self.response ==  None else self.response.status_code
+        
+        self.source_code = '' if self.response == None else self.response.text
+    def initialize_session(self):
         s = requests.Session()
         s.cookies["cf_clearance"] = "cb4c883efc59d0e990caf7508902591f4569e7bf-1617321078-0-150"
         s.headers.update(self.headers)
         # Add any other headers or cookie settings here
-        adapter = TLSAdapter()
-        
+        adapter = TLSAdapterSingleton().get_instance()
         s.mount('https://', adapter)
         return s
 
@@ -462,18 +515,15 @@ class SafeRequest:
             
         
         self.wait_between_requests()
-
-        cleaned_urls = self.clean_url(self.url)
+        
         for _ in range(self.max_retries):
-            cleaned_url = self.url_manager.correct_url
             try:
-                response = self.session.get(cleaned_url, timeout=10)  # 10 seconds timeout
-                    
+                response = self.session.get(url=self.url_manager.correct_url, timeout=10)  # 10 seconds timeout
                 if response.status_code == 200:
                     self.last_request_time = get_time_stamp()
                     return response
                 elif response.status_code == 429:
-                    logging.warning(f"Rate limited by {cleaned_url}. Retrying...")
+                    logging.warning(f"Rate limited by {self.url_manager.correct_url}. Retrying...")
                     get_sleep(5)  # adjust this based on the server's rate limit reset time
             except requests.Timeout as e:
                 logging.error(f"Request to {cleaned_url} timed out: {e}")
@@ -484,10 +534,8 @@ class SafeRequest:
             except requests.RequestException as e:
                 logging.error(f"Request exception for URL {cleaned_url}: {e}")
 
-        logging.error(f"Failed to retrieve content from {cleaned_url} after {self.max_retries} retries.")
+        logging.error(f"Failed to retrieve content from {self.url_manager.correct_url} after {self.max_retries} retries.")
         return None
-
-
     @staticmethod
     def is_valid_url(url):
         """
@@ -497,12 +545,21 @@ class SafeRequest:
         return bool(parsed.netloc) and bool(parsed.scheme)
     def get_source_code(self, url=None,response=None):
         if self.response:
-            input(self.response)
             return self.response.text if self.response else None
         else:
             logging.error(f"Invalid URL: {url}")
             return None
-        self.clean_url(self.url)
+        self.clean_url(self.url_manager.correct_url)
+class SafeRequestSingleton:
+    _instance = None
+    @staticmethod
+    
+    def get_instance(url=None,headers:dict=UserAgentManager().user_agent_header,max_retries=3,last_request_time=None,request_wait_limit=1.5):
+        if SafeRequestSingleton._instance is None:
+            SafeRequestSingleton._instance = SafeRequest(url=url,headers=headers,max_retries=max_retries,last_request_time=last_request_time,request_wait_limit=request_wait_limit)
+        elif SafeRequestSingleton._instance.url != url or SafeRequestSingleton._instance.headers != headers or SafeRequestSingleton._instance.max_retries != max_retries or SafeRequestSingleton._instance.request_wait_limit != request_wait_limit:
+            SafeRequestSingleton._instance = SafeRequest(url=url,headers=headers,max_retries=max_retries,last_request_time=last_request_time,request_wait_limit=request_wait_limit)
+        return SafeRequestSingleton._instance
 ## ## 
 # Usage
 ## safe_requester = SafeRequest()
@@ -523,7 +580,7 @@ class SafeRequest:
 class URLManager:
     def __init__(self,url=None,session=requests):
         self.url = url
-        self.session = None if url ==  None else session
+        self.session = session
         self.striped_url = None if url ==  None else self.strip_web()
         self.clean_urls = None if url ==  None else self.clean_url()
         self.correct_url = None if url ==  None else self.get_correct_url()
@@ -584,6 +641,16 @@ class URLManager:
             except requests.exceptions.RequestException as e:
                 print(e)
         return None
+class URLManagerSingleton:
+    _instance = None
+    @staticmethod
+    def get_instance(url=None,session=requests):
+        if URLManagerSingleton._instance is None:
+            URLManagerSingleton._instance = URLManager(url=url,session=session)
+        elif URLManagerSingleton._instance.session != session or URLManagerSingleton._instance.url != url:
+            URLManagerSingleton._instance = URLManager(url=url,session=session)
+        return URLManagerSingleton._instance
+    
 def get_limited_request(request_url=str,service_name="default"):
     manager = DynamicRateLimiterManagerSingleton.get_instance()  # Get the singleton instance
     unwanted_response=True
@@ -678,16 +745,15 @@ def get_Source_code(url: str ='https://www.example.com' , user_agent:str= UserAg
     Returns:
         str or None: The source code of the URL if the request is successful, or None if the request fails.
     """
-    url = get_correct_url(url)
-    if url is None:
-        return None
-    
+    url_manager = URLManagerSingleton().get_source_code(url=url)
+    if url_manager.url is None:
+        return []
     s = requests.Session()
     s.cookies["cf_clearance"] = "cb4c883efc59d0e990caf7508902591f4569e7bf-1617321078-0-150"
     s.headers.update(get_user_agent(user_agent))
     adapter = TLSAdapter()
     s.mount('https://', adapter)
-    r = try_request(url=url, session=s)
+    r = try_request(url=url_manager.correct_url, session=s)
 
     if r is None:
         return None
@@ -703,12 +769,11 @@ def parse_react_source(url:str=None) -> list:
     Returns:
         list: A list of strings containing JavaScript and JSX source code found in <script> tags.
     """
-    url = get_correct_url(url)
-    if url is None:
+    url_manager = URLManagerSingleton().get_source_code(url=url)
+    if url_manager.url is None:
         return []
-    
-    data = get_Source_code(url)
-    soup = BeautifulSoup(data, 'html.parser')
+    request_manager = SafeRequestSingleton().get_instance(url=url_manager.url)
+    soup = BeautifulSoup(request_manager.source_code, "html.parser")
     script_tags = soup.find_all('script', type=lambda t: t and ('javascript' in t or 'jsx' in t))
     react_source_code = []
     for script_tag in script_tags:
@@ -725,14 +790,15 @@ def get_all_website_links(url:str=None) -> list:
     Returns:
         list: A list of URLs that belong to the same website as the specified URL.
     """
-    url = get_correct_url(url)
-    if url is None:
+    request_manager = SafeRequestSingleton().get_instance(url=url)
+    if request_manager.url is None:
         return []
     
-    urls = [url]
+    
+    urls = [request_manager.url]
     # domain name of the URL without the protocol
-    domain_name = urlparse(url).netloc
-    soup = BeautifulSoup(get_Source_code(url=url), "html.parser")
+    domain_name = urlparse(request_manager.url).netloc
+    soup = BeautifulSoup(request_manager.source_code, "html.parser")
     for a_tag in soup.findAll("a"):
         href = a_tag.attrs.get("href")
         if href == "" or href is None:
@@ -765,11 +831,12 @@ def parse_all(url:str=None):
     Returns:
         dict: A dict containing keys: [element_types, attribute_values, attribute_names, class_names] with values as lists for keys element types, attribute values, attribute names, and class names found in the source code.
     """
-    url = get_correct_url(url)
-    if url is None:
+    request_manager = SafeRequest().get_source_code(url=url)
+    if request_manager.url is None:
         return [], [], [], []
     
-    data = get_Source_code(url)
+    
+    soup = BeautifulSoup(request_manager.source_code, 'html.parser')
     element_types, attribute_values, attribute_names, class_names = [], [], [], []
     data = str(data).split('<')
     for k in range(1, len(data)):
@@ -801,13 +868,10 @@ def extract_elements(url:str=None, element_type:str=None, attribute_name:str=Non
     Returns:
         list: A list of strings containing portions of the source code that match the provided filters.
     """
-    url = get_correct_url(url)
-    if url is None:
-        return []
-
-    data = SafeRequest().get_source_code(url=url)
-    soup = BeautifulSoup(data, 'html.parser')
-
+    request_manager = SafeRequest().get_source_code(url=url)
+    if request_manager.url is None:
+        return [], [], [], []
+    soup = BeautifulSoup(request_manager.source_code, 'html.parser')
     elements = []
 
     # If no filters are provided, return the entire source code
@@ -832,4 +896,3 @@ def get_response(response):
         if data:
             return data.get("response", data)
     return response.text
-

{abstract_webtools-0.1.4.1.dist-info → abstract_webtools-0.1.4.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: abstract-webtools
-Version: 0.1.4.1
+Version: 0.1.4.12
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff

abstract_webtools-0.1.4.12.dist-info/RECORD

@@ -0,0 +1,11 @@
+abstract_webtools/__init__.py,sha256=2SWEfdPDHqqjUYsOQYlaOHF644ZYcO160nWKiAjga4w,34
+abstract_webtools/abstract_crawler.py,sha256=e8jVVv1_EB8poqlrdQaJ19z9Z0t8un5uc-DKnj1Ud5s,8002
+abstract_webtools/abstract_webtools.py,sha256=sjrsbUa4AkFD80usyU2D2Ez966N_kDt_j4hf2-c7AkQ,40159
+abstract_webtools/abstract_webtools2.py,sha256=dlhhgmUTaN_NgkT6GcJMVBLuXjmW38gAOeCrKxYqytk,30685
+abstract_webtools/grab_source_gui.py,sha256=w7vDsEu1IfOIzcsfWeP2IpdS3yhFA_x5IVIeLFlfppw,14708
+abstract_webtools/main.py,sha256=_I7pPXPkoLZOoYGLQDrSLGhGuQt6-PVyXEHZSmglk2g,1329
+abstract_webtools-0.1.4.12.dist-info/LICENSE,sha256=g3WEJFiVS27HyCGRTwKSsMLyciMaGFdWcZGOe1QalZk,3877
+abstract_webtools-0.1.4.12.dist-info/METADATA,sha256=d4kh-bQrfhjqlmyDnxiyoGbm7xAvsVR7ot5yOmiMcNY,8963
+abstract_webtools-0.1.4.12.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
+abstract_webtools-0.1.4.12.dist-info/top_level.txt,sha256=2DMJ7RmjTcjCsa-uwAV0K6eXXlIIkFDEjBLg_uyCmCI,18
+abstract_webtools-0.1.4.12.dist-info/RECORD,,

abstract_webtools-0.1.4.1.dist-info/RECORD

@@ -1,11 +0,0 @@
-abstract_webtools/__init__.py,sha256=2SWEfdPDHqqjUYsOQYlaOHF644ZYcO160nWKiAjga4w,34
-abstract_webtools/abstract_crawler.py,sha256=5mbFI9tFW0OEeESnC9FjL4uITv1Vvp5CoEH6glFuND8,7994
-abstract_webtools/abstract_webtools.py,sha256=BIT-ZzoAOhGPPeXa75Qudnl_qQ0WZSU5R6a7x9HJXbo,34926
-abstract_webtools/abstract_webtools2.py,sha256=dlhhgmUTaN_NgkT6GcJMVBLuXjmW38gAOeCrKxYqytk,30685
-abstract_webtools/grab_source_gui.py,sha256=w7vDsEu1IfOIzcsfWeP2IpdS3yhFA_x5IVIeLFlfppw,14708
-abstract_webtools/main.py,sha256=_I7pPXPkoLZOoYGLQDrSLGhGuQt6-PVyXEHZSmglk2g,1329
-abstract_webtools-0.1.4.1.dist-info/LICENSE,sha256=g3WEJFiVS27HyCGRTwKSsMLyciMaGFdWcZGOe1QalZk,3877
-abstract_webtools-0.1.4.1.dist-info/METADATA,sha256=cjZwVqntAL286xh80p0sbzba5sATSFl1bzM75ENlPjU,8962
-abstract_webtools-0.1.4.1.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
-abstract_webtools-0.1.4.1.dist-info/top_level.txt,sha256=2DMJ7RmjTcjCsa-uwAV0K6eXXlIIkFDEjBLg_uyCmCI,18
-abstract_webtools-0.1.4.1.dist-info/RECORD,,