abstract-webtools 0.1.6.27__py3-none-any.whl → 0.1.6.29__py3-none-any.whl

abstract_webtools/managers/linkManager/__init__.py

@@ -0,0 +1 @@
+from .linkManager import *

abstract_webtools/managers/linkManager/linkManager.py

@@ -0,0 +1,189 @@
+from ..abstract_webtools import *
+from .urlManager import *
+from .requestManager import *
+from .soupManager import *
+def get_url(url=None,url_mgr=None):
+    if not url and not url_mgr:
+        return None
+    if url:
+        url_mgr = urlManager(url)
+    return url_mgr.url
+def get_url_mgr(url=None,url_mgr=None):
+    if not url and not url_mgr:
+        return None
+    if url:
+        url_mgr = urlManager(url)
+    return url_mgr 
+class linkManager:
+    """
+    LinkManager is a class for managing and extracting links and image links from a web page.
+
+    Args:
+        url (str): The URL of the web page (default is "https://example.com").
+        source_code (str or None): The source code of the web page (default is None).
+        url_mgr (UrlManager or None): An instance of UrlManager (default is None).
+        request_manager (requestManager or None): An instance of requestManager (default is None).
+        soup_manager (SoupManager or None): An instance of SoupManager (default is None).
+        image_link_tags (str): HTML tags to identify image links (default is 'img').
+        img_link_attrs (str): HTML attributes to identify image link URLs (default is 'src').
+        link_tags (str): HTML tags to identify links (default is 'a').
+        link_attrs (str): HTML attributes to identify link URLs (default is 'href').
+        strict_order_tags (bool): Flag to indicate if tags and attributes should be matched strictly (default is False).
+        img_attr_value_desired (list or None): Desired attribute values for image links (default is None).
+        img_attr_value_undesired (list or None): Undesired attribute values for image links (default is None).
+        link_attr_value_desired (list or None): Desired attribute values for links (default is None).
+        link_attr_value_undesired (list or None): Undesired attribute values for links (default is None).
+        associated_data_attr (list): HTML attributes to associate with the extracted links (default is ["data-title", 'alt', 'title']).
+        get_img (list): HTML attributes used to identify associated images (default is ["data-title", 'alt', 'title']).
+
+    Methods:
+        re_initialize(): Reinitialize the LinkManager with the current settings.
+        update_url_mgr(url_mgr): Update the URL manager with a new instance.
+        update_url(url): Update the URL and reinitialize the LinkManager.
+        update_source_code(source_code): Update the source code and reinitialize the LinkManager.
+        update_soup_manager(soup_manager): Update the SoupManager and reinitialize the LinkManager.
+        update_desired(...): Update the desired settings and reinitialize the LinkManager.
+        find_all_desired(...): Find all desired links or image links based on the specified criteria.
+        find_all_domain(): Find all unique domain names in the extracted links.
+
+    Note:
+        - The LinkManager class helps manage and extract links and image links from web pages.
+        - The class provides flexibility in specifying criteria for link extraction.
+    """
+    def __init__(self,url=None,source_code=None,url_mgr=None,req_mgr=None,soup_manager=None,image_link_tags='img',img_link_attrs='src',link_tags='a',link_attrs='href',strict_order_tags=False,img_attr_value_desired=None,img_attr_value_undesired=None,link_attr_value_desired=None,link_attr_value_undesired=None,associated_data_attr=["data-title",'alt','title'],get_img=["data-title",'alt','title']):
+        self.url_mgr = get_url_mgr(url=url,url_mgr=url_mgr)
+        self.url=self.url_mgr.url
+        self.req_mgr = get_req_mgr(req_mgr=req_mgr,url=self.url,url_mgr=self.url_mgr,source_code=source_code)
+        self.source_code = source_code or self.req_mgr.source_code or self.req_mgr.source_code_bytes
+        self.soup_mgr = get_soup_mgr(req_mgr=self.req_mgr,url_mgr=self.url_mgr,source_code = source_code)
+
+        self.strict_order_tags=strict_order_tags
+        self.image_link_tags=image_link_tags
+        self.img_link_attrs=img_link_attrs
+        self.link_tags=link_tags
+        self.link_attrs=link_attrs
+        self.img_attr_value_desired=img_attr_value_desired
+        self.img_attr_value_undesired=img_attr_value_undesired
+        self.link_attr_value_desired=link_attr_value_desired
+        self.link_attr_value_undesired=link_attr_value_undesired
+        self.associated_data_attr=associated_data_attr
+        self.get_img=get_img
+        self.all_desired_image_links=self.find_all_desired_links(tag=self.image_link_tags,
+                                                                 attr=self.img_link_attrs,
+                                                                 attr_value_desired=self.img_attr_value_desired,
+                                                                 attr_value_undesired=self.img_attr_value_undesired)
+        self.all_desired_links=self.find_all_desired_links(tag=self.link_tags,
+                                                           attr=self.link_attrs,
+                                                           attr_value_desired=self.link_attr_value_desired,
+                                                           attr_value_undesired=self.link_attr_value_undesired,
+                                                           associated_data_attr=self.associated_data_attr,
+                                                           get_img=get_img)
+    def re_initialize(self):
+        self.all_desired_image_links=self.find_all_desired_links(tag=self.image_link_tags,attr=self.img_link_attrs,strict_order_tags=self.strict_order_tags,attr_value_desired=self.img_attr_value_desired,attr_value_undesired=self.img_attr_value_undesired)
+        self.all_desired_links=self.find_all_desired_links(tag=self.link_tags,attr=self.link_attrs,strict_order_tags=self.strict_order_tags,attr_value_desired=self.link_attr_value_desired,attr_value_undesired=self.link_attr_value_undesired,associated_data_attr=self.associated_data_attr,get_img=self.get_img)
+    def update_url_mgr(self,url_mgr):
+        self.url_mgr=url_mgr
+        self.url=self.url_mgr.url
+        self.req_mgr.update_url_mgr(url_mgr=self.url_mgr)
+        self.soup_mgr.update_url_mgr(url_mgr=self.url_mgr)
+        self.source_code=self.soup_mgr.source_code
+        self.re_initialize()
+    def update_url(self,url):
+        self.url=url
+        self.url_mgr.update_url(url=self.url)
+        self.url=self.url_mgr.url
+        self.req_mgr.update_url(url=self.url)
+        self.soup_mgr.update_url(url=self.url)
+        self.source_code=self.soup_mgr.source_code
+        self.re_initialize()
+    def update_source_code(self,source_code):
+        self.source_code=source_code
+        if self.source_code != self.soup_mgr.source_code:
+            self.soup_mgr.update_source_code(source_code=self.source_code)
+        self.re_initialize()
+    def update_soup_manager(self,soup_manager):
+        self.soup_mgr=soup_manager
+        self.source_code=self.soup_mgr.source_code
+        self.re_initialize()
+    def update_desired(self,img_attr_value_desired=None,img_attr_value_undesired=None,link_attr_value_desired=None,link_attr_value_undesired=None,image_link_tags=None,img_link_attrs=None,link_tags=None,link_attrs=None,strict_order_tags=None,associated_data_attr=None,get_img=None):
+       self.strict_order_tags = strict_order_tags or self.strict_order_tags
+       self.img_attr_value_desired=img_attr_value_desired or self.img_attr_value_desired
+       self.img_attr_value_undesired=img_attr_value_undesired or self.img_attr_value_undesired
+       self.link_attr_value_desired=link_attr_value_desired or self.link_attr_value_desired
+       self.link_attr_value_undesired=link_attr_value_undesired or self.link_attr_value_undesired
+       self.image_link_tags=image_link_tags or self.image_link_tags
+       self.img_link_attrs=img_link_attrs or self.img_link_attrs
+       self.link_tags=link_tags or self.link_tags
+       self.link_attrs=link_attrs or self.link_attrs
+       self.associated_data_attr=associated_data_attr or self.associated_data_attr
+       self.get_img=get_img or self.get_img
+       self.re_initialize()
+    def find_all_desired(self,tag='img',attr='src',strict_order_tags=False,attr_value_desired=None,attr_value_undesired=None,associated_data_attr=None,get_img=None):
+        def make_list(obj):
+            if isinstance(obj,list) or obj==None:
+                return obj
+            return [obj]
+        def get_desired_value(attr,attr_value_desired=None,attr_value_undesired=None):
+            if attr_value_desired:
+                for value in attr_value_desired:
+                    if value not in attr:
+                        return False
+            if attr_value_undesired:
+                for value in attr_value_undesired:
+                    if value in attr:
+                        return False
+            return True
+        attr_value_desired,attr_value_undesired,associated_data_attr,tags,attribs=make_list(attr_value_desired),make_list(attr_value_undesired),make_list(associated_data_attr),make_list(tag),make_list(attr)
+        desired_ls = []
+        assiciated_data=[]
+        for i,tag in enumerate(tags):
+            attribs_list=attribs
+            if strict_order_tags:
+                if len(attribs)<=i:
+                    attribs_list=[None]
+                else:
+                    attribs_list=make_list(attribs[i])
+            for attr in attribs_list:
+                for component in self.soup_mgr.soup.find_all(tag):
+                    if attr in component.attrs and get_desired_value(attr=component[attr],attr_value_desired=attr_value_desired,attr_value_undesired=attr_value_undesired):
+                        if component[attr] not in desired_ls:
+                            desired_ls.append(component[attr])
+                            assiciated_data.append({"value":component[attr]})
+                            if associated_data_attr:
+                                for data in associated_data_attr:
+                                    if data in component.attrs:
+                                        assiciated_data[-1][data]=component.attrs[data]
+                                        if get_img and component.attrs[data]:
+                                            if data in get_img and len(component.attrs[data])!=0:
+                                                for each in self.soup_mgr.soup.find_all('img'):
+                                                    if 'alt' in each.attrs:
+                                                        if each.attrs['alt'] == component.attrs[data] and 'src' in each.attrs:
+                                                            assiciated_data[-1]['image']=each.attrs['src']
+        desired_ls.append(assiciated_data)
+        return desired_ls
+    def find_all_domain(self):
+        domain = urlparse(self.url_mgr.url).netloc
+        domains_ls=[self.url_mgr.url]
+        for url in self.all_desired_links[:-1]:
+            if self.url_mgr.is_valid_url(url):
+                parse = urlparse(url)
+                comp_domain = parse.netloc
+                if url not in domains_ls and comp_domain == domain:
+                    domains_ls.append(url)
+        return domains_ls
+    def find_all_desired_links(self,tag='img', attr='src',attr_value_desired=None,strict_order_tags=False,attr_value_undesired=None,associated_data_attr=None,all_desired=None,get_img=None):
+        all_desired = all_desired or self.find_all_desired(tag=tag,attr=attr,strict_order_tags=strict_order_tags,attr_value_desired=attr_value_desired,attr_value_undesired=attr_value_undesired,associated_data_attr=associated_data_attr,get_img=get_img)
+        assiciated_attrs = all_desired[-1]
+        valid_assiciated_attrs = []
+        desired_links=[]
+        for i,attr in enumerate(all_desired[:-1]):
+            valid_attr=self.url_mgr.make_valid(attr,self.url_mgr.protocol+'://'+self.url_mgr.domain) 
+            if valid_attr:
+                desired_links.append(valid_attr)
+                valid_assiciated_attrs.append(assiciated_attrs[i])
+                valid_assiciated_attrs[-1]["link"]=valid_attr
+        desired_links.append(valid_assiciated_attrs)
+        return desired_links
+
+
+    

abstract_webtools/managers/requestManager/__init__.py

@@ -0,0 +1 @@
+from .requestManager import *

abstract_webtools/managers/requestManager/requestManager.py

@@ -0,0 +1,353 @@
+from ..abstract_webtools import *
+
+from .userAgentManager import *
+from .cipherManager import *
+from .sslManager import *
+from .tlsAdapter import *
+from .networkManager import *
+from .seleniumManager import *
+from .urlManager import *
+class requestManager:
+    """
+    SafeRequest is a class for making HTTP requests with error handling and retries.
+
+    Args:
+        url (str or None): The URL to make requests to (default is None).
+        url_mgr (urlManager or None): An instance of urlManager (default is None).
+        network_manager (NetworkManager or None): An instance of NetworkManager (default is None).
+        user_agent_manager (UserAgentManager or None): An instance of UserAgentManager (default is None).
+        ssl_manager (SSlManager or None): An instance of SSLManager (default is None).
+        tls_adapter (TLSAdapter or None): An instance of TLSAdapter (default is None).
+        user_agent (str or None): The user agent string to use for requests (default is None).
+        proxies (dict or None): Proxy settings for requests (default is None).
+        headers (dict or None): Additional headers for requests (default is None).
+        cookies (dict or None): Cookie settings for requests (default is None).
+        session (requests.Session or None): A custom requests session (default is None).
+        adapter (str or None): A custom adapter for requests (default is None).
+        protocol (str or None): The protocol to use for requests (default is 'https://').
+        ciphers (str or None): Cipher settings for requests (default is None).
+        auth (tuple or None): Authentication credentials (default is None).
+        login_url (str or None): The URL for authentication (default is None).
+        email (str or None): Email for authentication (default is None).
+        password (str or None): Password for authentication (default is None).
+        certification (str or None): Certification settings for requests (default is None).
+        ssl_options (str or None): SSL options for requests (default is None).
+        stream (bool): Whether to stream the response content (default is False).
+        timeout (float or None): Timeout for requests (default is None).
+        last_request_time (float or None): Timestamp of the last request (default is None).
+        max_retries (int or None): Maximum number of retries for requests (default is None).
+        request_wait_limit (float or None): Wait time between requests (default is None).
+
+    Methods:
+        update_url_mgr(url_mgr): Update the URL manager and reinitialize the SafeRequest.
+        update_url(url): Update the URL and reinitialize the SafeRequest.
+        re_initialize(): Reinitialize the SafeRequest with the current settings.
+        authenticate(s, login_url=None, email=None, password=None, checkbox=None, dropdown=None): Authenticate and make a request.
+        fetch_response(): Fetch the response from the server.
+        initialize_session(): Initialize the requests session with custom settings.
+        process_response_data(): Process the fetched response data.
+        get_react_source_code(): Extract JavaScript and JSX source code from <script> tags.
+        get_status(url=None): Get the HTTP status code of a URL.
+        wait_between_requests(): Wait between requests based on the request_wait_limit.
+        make_request(): Make a request and handle potential errors.
+        try_request(): Try to make an HTTP request using the provided session.
+
+    Note:
+        - The SafeRequest class is designed for making HTTP requests with error handling and retries.
+        - It provides methods for authentication, response handling, and error management.
+    """
+    def __init__(self,
+                 url=None,
+                 source_code=None,
+                 url_mgr=None,
+                 network_manager=None,
+                 user_agent_manager=None,
+                 ssl_manager=None,
+                 ssl_options=None,
+                 tls_adapter=None,
+                 user_agent=None,
+                 proxies=None,
+                 headers=None,
+                 cookies=None,
+                 session=None,
+                 adapter=None,
+                 protocol=None,
+                 ciphers=None,
+                 spec_login=False,
+                 login_referer=None,
+                 login_user_agent=None,
+                 auth=None,
+                 login_url=None,
+                 email = None,
+                 password=None,
+                 checkbox=None,
+                 dropdown=None,
+                 certification=None,
+                 stream=False,
+                 timeout = None,
+                 last_request_time=None,
+                 max_retries=None,
+                 request_wait_limit=
+                 None):
+        self.url_mgr = get_url_mgr(url=url,url_mgr=url_mgr)
+        self.url=get_url(url=url,url_mgr=self.url_mgr)      
+        self._url_mgr = self.url_mgr
+        self._url=self.url       
+        self.user_agent = user_agent
+        self.user_agent_manager = user_agent_manager or UserAgentManager(user_agent=self.user_agent)
+        self.headers= headers or self.user_agent_manager.header or {'Accept': '*/*'}
+        self.user_agent= self.user_agent_manager.user_agent
+        self.ciphers=ciphers or CipherManager().ciphers_string
+        self.certification=certification
+        self.ssl_options=ssl_options
+        self.ssl_manager = ssl_manager or SSLManager(ciphers=self.ciphers, ssl_options=self.ssl_options, certification=self.certification)
+        self.tls_adapter=tls_adapter or  TLSAdapter(ssl_manager=self.ssl_manager,certification=self.certification,ssl_options=self.ssl_manager.ssl_options)
+        self.network_manager= network_manager or NetworkManager(user_agent_manager=self.user_agent_manager,ssl_manager=self.ssl_manager, tls_adapter=self.tls_adapter,user_agent=user_agent,proxies=proxies,cookies=cookies,ciphers=ciphers, certification=certification, ssl_options=ssl_options)
+        self.stream=stream
+        self.tls_adapter=self.network_manager.tls_adapter
+        self.ciphers=self.network_manager.ciphers
+        self.certification=self.network_manager.certification
+        self.ssl_options=self.network_manager.ssl_options
+        self.proxies=self.network_manager.proxies
+        self.timeout=timeout
+        self.cookies=self.network_manager.cookies
+        self.session = session or requests.session()
+        self.auth = auth
+        self.spec_login=spec_login
+        self.password=password
+        self.email = email
+        self.checkbox=checkbox
+        self.dropdown=dropdown
+        self.login_url=login_url
+        self.login_user_agent=login_user_agent
+        self.login_referer=login_referer
+        self.protocol=protocol or 'https://'
+        
+        self.stream=stream if isinstance(stream,bool) else False
+        self.initialize_session()
+        self.last_request_time=last_request_time
+        self.max_retries = max_retries or 3
+        self.request_wait_limit = request_wait_limit or 1.5
+        self._response=None
+        self.source_code = get_selenium_source(self.url)
+        self.source_code_bytes=None
+        self.source_code_json = {}
+        self.react_source_code=[]
+        self._response_data = None
+        self.process_response_data()
+    def update_url_mgr(self,url_mgr):
+        self.url_mgr=url_mgr
+        self.re_initialize()
+    def update_url(self,url):
+        self.url_mgr.update_url(url=url)
+        self.re_initialize()
+    def re_initialize(self):
+        self._response=None
+        self.make_request()
+        self.source_code = None
+        self.source_code_bytes=None
+        self.source_code_json = {}
+        self.react_source_code=[]
+        self._response_data = None
+        self.process_response_data()
+    @property
+    def response(self):
+        """Lazy-loading of response."""
+        if self._response is None:
+            self._response = self.fetch_response()
+            
+        return self._response
+    def authenticate(self,session, login_url=None, email=None, password=None,checkbox=None,dropdown=None):
+        login_urls = login_url or [self.url_mgr.url,self.url_mgr.domain,self.url_mgr.url_join(url=self.url_mgr.domain,path='login'),self.url_mgr.url_join(url=self.url_mgr.domain,path='auth')]
+        s = session
+        if not isinstance(login_urls,list):
+            login_urls=[login_urls]
+        for login_url in login_urls:
+            login_url_mgr = urlManager(login_url)
+            login_url = login_url_mgr.url
+            
+            r = s.get(login_url)
+            soup = BeautifulSoup(r.content, "html.parser")
+            # Find the token or any CSRF protection token
+            token = soup.find('input', {'name': 'token'}).get('value') if soup.find('input', {'name': 'token'}) else None
+            if token != None:
+                break
+        login_data = {}
+        if email != None:
+            login_data['email']=email
+        if password != None:
+            login_data['password'] = password
+        if checkbox != None:
+            login_data['checkbox'] = checkbox
+        if dropdown != None:
+            login_data['dropdown']=dropdown
+        if token != None:
+            login_data['token'] = token
+        s.post(login_url, data=login_data)
+        return s
+
+    def fetch_response(self) -> Union[requests.Response, None]:
+        """Actually fetches the response from the server."""
+        # You can further adapt this method to use retries or other logic you had
+        # in your original code, but the main goal here is to fetch and return the response
+        return self.try_request()
+    def spec_auth(self, session=None, email=None, password=None, login_url=None, login_referer=None, login_user_agent=None):
+        s = session or requests.session()
+        
+        domain = self.url_mgr.url_join(self.url_mgr.get_correct_url(self.url_mgr.domain),'login') if login_url is None else login_url
+        login_url = self.url_mgr.get_correct_url(url=domain)
+        
+        login_referer = login_referer or self.url_mgr.url_join(url=login_url, path='?role=fast&to=&s=1&m=1&email=YOUR_EMAIL')
+        login_user_agent = login_user_agent or 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0'
+        
+        headers = {"Referer": login_referer, 'User-Agent': login_user_agent}
+        payload = {'email': email, 'pass': password}
+        
+        page = s.get(login_url)
+        soup = BeautifulSoup(page.content, 'lxml')
+        action_url = soup.find('form')['action']
+        s.post(action_url, data=payload, headers=headers)
+        return s
+    def initialize_session(self):
+        s = self.session  
+        if self.auth:
+            s= self.auth
+        elif self.spec_login:
+            s=self.spec_auth(session=s,email=self.email, password=self.password, login_url=self.login_url, login_referer=self.login_referer, login_user_agent=self.login_user_agent)
+        elif any([self.password, self.email, self.login_url, self.checkbox, self.dropdown]):
+            s=self.authenticate(session=s, login_url=self.login_url, email=self.email, password=self.password, checkbox=self.checkbox, dropdown=self.dropdown)
+        s.proxies = self.proxies
+        s.cookies["cf_clearance"] = self.network_manager.cookies
+        s.headers.update(self.headers)
+        s.mount(self.protocol, self.network_manager.tls_adapter)
+        return s
+    def process_response_data(self):
+        """Processes the fetched response data."""
+        if not self.response:
+            return  # No data to process
+        if  isinstance(self.response,str):
+            self.source_code = self.response
+        else:
+            self.source_code = self.response.text
+            self.source_code_bytes = self.response.content
+            if self.response.headers.get('content-type') == 'application/json':
+                data = convert_to_json(self.source_code)
+                if data:
+                    self.source_code_json = data.get("response", data)
+            
+            self.get_react_source_code()
+    def get_react_source_code(self) -> list:
+        """
+        Fetches the source code of the specified URL and extracts JavaScript and JSX source code (React components).
+
+        Args:
+            url (str): The URL to fetch the source code from.
+
+        Returns:
+            list: A list of strings containing JavaScript and JSX source code found in <script> tags.
+        """
+        if self.url_mgr.url is None:
+            return []
+        soup = BeautifulSoup(self.source_code_bytes,"html.parser")
+        script_tags = soup.find_all('script', type=lambda t: t and ('javascript' in t or 'jsx' in t))
+        for script_tag in script_tags:
+            self.react_source_code.append(script_tag.string)
+
+
+    def get_status(url:str=None) -> int:
+        """
+        Gets the HTTP status code of the given URL.
+
+        Args:
+            url (str): The URL to check the status of.
+
+        Returns:
+            int: The HTTP status code of the URL, or None if the request fails.
+        """
+        # Get the status code of the URL
+        return try_request(url=url).status_code
+    def wait_between_requests(self):
+        """
+        Wait between requests based on the request_wait_limit.
+        """
+        if self.last_request_time:
+            sleep_time = self.request_wait_limit - (get_time_stamp() - self.last_request_time)
+            if sleep_time > 0:
+                logging.info(f"Sleeping for {sleep_time:.2f} seconds.")
+                get_sleep(sleep_time)
+
+    def make_request(self):
+        """
+        Make a request and handle potential errors.
+        """
+        # Update the instance attributes if they are passed
+        
+        self.wait_between_requests()
+        for _ in range(self.max_retries):
+            try:
+                self.try_request()  # 10 seconds timeout
+                if self.response:
+                    if self.response.status_code == 200:
+                        self.last_request_time = get_time_stamp()
+                        return self.response
+                    elif self.response.status_code == 429:
+                        logging.warning(f"Rate limited by {self.url_mgr.url}. Retrying...")
+                        get_sleep(5)  # adjust this based on the server's rate limit reset time
+            except requests.Timeout as e:
+                logging.error(f"Request to {cleaned_url} timed out: {e}")
+            except requests.ConnectionError:
+                logging.error(f"Connection error for URL {self.url_mgr.url}.")
+            except requests.Timeout:
+                logging.error(f"Request timeout for URL {self.url_mgr.url}.")
+            except requests.RequestException as e:
+                logging.error(f"Request exception for URL {self.url_mgr.url}: {e}")
+        try:
+            response = get_selenium_source(self.url_mgr.url)
+            if response:
+                self.response = response
+                return self.response
+        except:
+            logging.error(f"Failed to retrieve content from {self.url_mgr.url} after {self.max_retries} retries.")
+            return None
+    def try_request(self) -> Union[requests.Response, None]:
+        """
+        Tries to make an HTTP request to the given URL using the provided session.
+
+        Args:
+            timeout (int): Timeout for the request.
+
+        Returns:
+            requests.Response or None: The response object if the request is successful, or None if the request fails.
+        """
+        try:
+            return get_selenium_source(self.url_mgr.url)#self.session.get(url=self.url_mgr.url, timeout=self.timeout,stream=self.stream)
+        except requests.exceptions.RequestException as e:
+            print(e)
+            return None
+
+
+    @property
+    def url(self):
+        return self.url_mgr.url
+
+    @url.setter
+    def url(self, new_url):
+        self._url = new_url
+class SafeRequestSingleton:
+    _instance = None
+    @staticmethod
+    def get_instance(url=None,headers:dict=None,max_retries=3,last_request_time=None,request_wait_limit=1.5):
+        if SafeRequestSingleton._instance is None:
+            SafeRequestSingleton._instance = SafeRequest(url,url_mgr=urlManagerSingleton,headers=headers,max_retries=max_retries,last_request_time=last_request_time,request_wait_limit=request_wait_limit)
+        elif SafeRequestSingleton._instance.url != url or SafeRequestSingleton._instance.headers != headers or SafeRequestSingleton._instance.max_retries != max_retries or SafeRequestSingleton._instance.request_wait_limit != request_wait_limit:
+            SafeRequestSingleton._instance = SafeRequest(url,url_mgr=urlManagerSingleton,headers=headers,max_retries=max_retries,last_request_time=last_request_time,request_wait_limit=request_wait_limit)
+        return SafeRequestSingleton._instance
+def get_req_mgr(url=None,url_mgr=None,source_code=None,req_mgr=None):
+    url = get_url(url=url,url_mgr=url_mgr)
+    url_mgr = get_url_mgr(url=url,url_mgr=url_mgr )
+    req_mgr = req_mgr  or requestManager(url_mgr=url_mgr,url=url,source_code=source_code)
+    return req_mgr
+def get_source(url=None,url_mgr=None,source_code=None,req_mgr=None):
+    # Placeholder for actual implementation.
+    req_mgr = get_req_mgr(req_mgr=req_mgr,url=url,url_mgr=url_mgr,source_code=source_code)
+    return req_mgr.source_code

abstract_webtools/managers/soupManager/__init__.py

@@ -0,0 +1 @@
+from .soupManager import *

abstract_webtools/managers/soupManager/soupManager.py

@@ -0,0 +1,362 @@
+from ..abstract_webtools import *
+from .urlManager import *
+from .requestManager import *
+class soupManager:
+    """
+    SoupManager is a class for managing and parsing HTML source code using BeautifulSoup.
+
+    Args:
+        url (str or None): The URL to be parsed (default is None).
+        source_code (str or None): The HTML source code (default is None).
+        url_mgr (UrlManager or None): An instance of UrlManager (default is None).
+        requestManager (SafeRequest or None): An instance of SafeRequest (default is None).
+        parse_type (str): The type of parser to be used by BeautifulSoup (default is "html.parser").
+
+    Methods:
+        re_initialize(): Reinitialize the SoupManager with the current settings.
+        update_url(url): Update the URL and reinitialize the SoupManager.
+        update_source_code(source_code): Update the source code and reinitialize the SoupManager.
+        update_requestManager(requestManager): Update the request manager and reinitialize the SoupManager.
+        update_url_mgr(url_mgr): Update the URL manager and reinitialize the SoupManager.
+        update_parse_type(parse_type): Update the parsing type and reinitialize the SoupManager.
+        all_links: A property that provides access to all discovered links.
+        _all_links_get(): A method to load all discovered links.
+        get_all_website_links(tag="a", attr="href"): Get all URLs belonging to the same website.
+        meta_tags: A property that provides access to all discovered meta tags.
+        _meta_tags_get(): A method to load all discovered meta tags.
+        get_meta_tags(): Get all meta tags in the source code.
+        find_all(element, soup=None): Find all instances of an HTML element in the source code.
+        get_class(class_name, soup=None): Get the specified class from the HTML source code.
+        has_attributes(tag, *attrs): Check if an HTML tag has the specified attributes.
+        get_find_all_with_attributes(*attrs): Find all HTML tags with specified attributes.
+        get_all_desired_soup(tag=None, attr=None, attr_value=None): Get HTML tags based on specified criteria.
+        extract_elements(url, tag=None, class_name=None, class_value=None): Extract portions of source code based on filters.
+        find_all_with_attributes(class_name=None, *attrs): Find classes with associated href or src attributes.
+        get_images(tag_name, class_name, class_value): Get images with specific class and attribute values.
+        discover_classes_and_meta_images(tag_name, class_name_1, class_name_2, class_value, attrs): Discover classes and meta images.
+
+    Note:
+        - The SoupManager class is designed for parsing HTML source code using BeautifulSoup.
+        - It provides various methods to extract data and discover elements within the source code.
+    """
+    def __init__(self,url=None,source_code=None,url_mgr=None,req_mgr=None, parse_type="html.parser"):
+        self.soup=[]
+        url = get_url(url=url,url_mgr=url_mgr)
+        self.url_mgr = get_url_mgr(url=url,url_mgr=url_mgr)
+        self.url=self.url_mgr.url
+        self.req_mgr = get_req_mgr(req_mgr=req_mgr,url=self.url,url_mgr=self.url_mgr,source_code=source_code)
+        self.parse_type = parse_type
+        source_code = source_code or self.req_mgr.source_code or self.req_mgr.source_code_bytes
+        if source_code:
+            source_code = str(source_code)
+        self.source_code = source_code
+        self.soup= BeautifulSoup(self.source_code, self.parse_type)
+        self.all_tags_and_attribute_names = self.get_all_tags_and_attribute_names()
+        self.all_tags = self.all_tags_and_attribute_names.get('tags')
+        self.all_attribute_names = self.all_tags_and_attribute_names.get('attributes')
+        self.all_tags_and_attributes = self.all_tags + self.all_attribute_names
+        
+        self._all_links_data = None
+        self._meta_tags_data = None
+    def re_initialize(self):
+        self.soup= BeautifulSoup(self.source_code, self.parse_type)
+        self._all_links_data = None
+        self._meta_tags_data = None
+    def update_url(self,url):
+        self.url_mgr.update_url(url=url)
+        self.url=self.url_mgr.url
+        self.req_mgr.update_url(url=url)
+        self.source_code = self.req_mgr.source_code_bytes
+        self.re_initialize()
+    def update_source_code(self,source_code):
+        if source_code:
+            source_code = str(source_code)
+        self.source_code = source_code
+        self.re_initialize()
+    def update_requestManager(self,requestManager):
+        self.req_mgr = requestManager
+        self.url_mgr=self.req_mgr.url_mgr
+        self.url=self.url_mgr.url
+        self.source_code = self.req_mgr.source_code_bytes
+        self.re_initialize()
+    def update_url_mgr(self,url_mgr):
+        self.url_mgr=url_mgr
+        self.url=self.url_mgr.url
+        self.req_mgr.update_url_mgr(url_mgr=self.url_mgr)
+        self.source_code = self.req_mgr.source_code_bytes
+        self.re_initialize()
+    def update_parse_type(self,parse_type):
+        self.parse_type=parse_type
+        self.re_initialize()
+    @property
+    def all_links(self):
+        """This is a property that provides access to the _all_links_data attribute.
+        The first time it's accessed, it will load the data."""
+        if self._all_links_data is None:
+            print("Loading all links for the first time...")
+            self._all_links_data = self._all_links_get()
+        return self._all_links_data
+    def _all_links_get(self):
+        """A method that loads the data (can be replaced with whatever data loading logic you have)."""
+        return self.get_all_website_links()
+    def get_all_website_links(self,tag="a",attr="href") -> list:
+        """
+        Returns all URLs that are found on the specified URL and belong to the same website.
+
+        Args:
+            url (str): The URL to search for links.
+
+        Returns:
+            list: A list of URLs that belong to the same website as the specified URL.
+        """
+        all_urls=[self.url_mgr.url]
+        domain = self.url_mgr.domain
+        all_desired=self.get_all_website_links(tag=tag,attr=attr)
+        for tag in all_desired:
+            href = tag.attrs.get(attr)
+            if href == "" or href is None:
+                # href empty tag
+                continue
+            href=self.url_mgr.get_relative_href(self.url_mgr.url,href)
+            if not self.url_mgr.is_valid_url(href):
+                # not a valid URL
+                continue
+            if href in all_urls:
+                # already in the set
+                continue
+            if domain not in href:
+                # external link
+                continue
+            all_urls.append(href)
+                
+        return all_urls
+
+
+    @property
+    def meta_tags(self):
+        """This is a property that provides access to the _all_links_data attribute.
+        The first time it's accessed, it will load the data."""
+        if self._meta_tags_data is None:
+            print("Loading all links for the first time...")
+            self._meta_tags_data = self._all_links_get()
+        return self._meta_tags_data
+    def _meta_tags_get(self):
+        """A method that loads the data (can be replaced with whatever data loading logic you have)."""
+        return self.get_meta_tags()
+    def get_meta_tags(self):
+        tags = self.find_all("meta")
+        for meta_tag in tags:
+            for attr, values in meta_tag.attrs.items():
+                if attr not in self.meta_tags:
+                    self.meta_tags[attr] = []
+                if values not in self.meta_tags[attr]:
+                    self.meta_tags[attr].append(values)
+
+                    
+    def find_all(self,element,soup=None):
+        soup = self.soup if soup == None else soup
+        return soup.find_all(element)
+    def get_class(self,class_name,soup=None):
+        soup = self.soup if soup == None else soup
+        return soup.get(class_name)
+    @staticmethod
+    def has_attributes(tag, *attrs):
+        return any(tag.has_attr(attr) for attr in attrs)
+    def get_find_all_with_attributes(self, *attrs):
+        return self.soup.find_all(lambda t: self.has_attributes(t, *attrs))
+    def find_tags_by_attributes(self, tag: str = None, attr: str = None, attr_values: List[str] = None) ->List:
+        if not tag:
+            tags = self.soup.find_all(True)  # get all tags
+        else:
+            tags = self.soup.find_all(tag)  # get specific tags
+            
+        extracted_tags = []
+        for t in tags:
+            if attr:
+                attribute_value = t.get(attr)
+                if not attribute_value:  # skip tags without the desired attribute
+                    continue
+                if attr_values and not any(value in attribute_value for value in attr_values):  # skip tags without any of the desired attribute values
+                    continue
+            extracted_tags.append(t)
+        return extracted_tags
+
+
+    def extract_elements(self,url:str=None, tag:str=None, class_name:str=None, class_value:str=None) -> list:
+        """
+        Extracts portions of the source code from the specified URL based on provided filters.
+
+        Args:
+            url (str): The URL to fetch the source code from.
+            element_type (str, optional): The HTML element type to filter by. Defaults to None.
+            attribute_name (str, optional): The attribute name to filter by. Defaults to None.
+            class_name (str, optional): The class name to filter by. Defaults to None.
+
+        Returns:
+            list: A list of strings containing portions of the source code that match the provided filters.
+        """
+        elements = []
+        # If no filters are provided, return the entire source code
+        if not tag and not class_name and not class_value:
+            elements.append(str(self.soup))
+            return elements
+        # Find elements based on the filters provided
+        if tag:
+            elements.extend([str(tags) for tags in self.get_all_desired(tag)])
+        if class_name:
+            elements.extend([str(tags) for tags in self.get_all_desired(tag={class_name: True})])
+        if class_value:
+            elements.extend([str(tags) for tags in self.get_all_desired(class_name=class_name)])
+        return elements
+    def find_all_with_attributes(self, class_name=None, *attrs):
+        """
+        Discovers classes in the HTML content of the provided URL 
+        that have associated href or src attributes.
+
+        Args:
+            base_url (str): The URL from which to discover classes.
+
+        Returns:
+            set: A set of unique class names.
+        """
+
+    
+        unique_classes = set()
+        for tag in self.get_find_all_with_attributes(*attrs):
+            class_list = self.get_class(class_name=class_name, soup=tag)
+            unique_classes.update(class_list)
+        return unique_classes
+    def get_images(self, tag_name, class_name, class_value):
+        images = []
+        for tag in self.soup.find_all(tag_name):
+            if class_name in tag.attrs and tag.attrs[class_name] == class_value:
+                content = tag.attrs.get('content', '')
+                if content:
+                    images.append(content)
+        return images
+    def extract_text_sections(self) -> list:
+        """
+        Extract all sections of text from an HTML content using BeautifulSoup.
+
+        Args:
+            html_content (str): The HTML content to be parsed.
+
+        Returns:
+            list: A list containing all sections of text.
+        """
+        # Remove any script or style elements to avoid extracting JavaScript or CSS code
+        for script in self.soup(['script', 'style']):
+            script.decompose()
+
+        # Extract text from the remaining elements
+        text_sections = self.soup.stripped_strings
+        return [text for text in text_sections if text]
+    def discover_classes_and_meta_images(self, tag_name, class_name_1, class_name_2, class_value, attrs):
+        """
+        Discovers classes in the HTML content of the provided URL 
+        that have associated href or src attributes. Also, fetches 
+        image references from meta tags.
+
+        Args:
+            base_url (str): The URL from which to discover classes and meta images.
+
+        Returns:
+            tuple: A set of unique class names and a list of meta images.
+        """
+    
+        unique_classes = self.find_all_with_attributes(class_name=class_name_1, *attrs)
+        images = self.get_images(tag_name=tag_name, class_name=class_name_2, class_value=class_value)
+        return unique_classes, images
+    def get_all_tags_and_attribute_names(self):
+        tag_names = set()  # Using a set to ensure uniqueness
+        attribute_names = set()
+        get_all = self.find_tags_by_attributes()
+        for tag in get_all:  # True matches all tags
+            tag_names.add(tag.name)
+            for attr in tag.attrs:
+                attribute_names.add(attr)
+        tag_names_list = list(tag_names)
+        attribute_names_list = list(attribute_names)
+        return {"tags":tag_names_list,"attributes":attribute_names_list}
+
+    def get_all_attribute_values(self, tags_list=None):
+        """
+        Collects all attribute values for each specified tag or all tags if none are specified.
+        
+        Parameters:
+        - tags_list: List of specific tags to retrieve attributes from, e.g., ['script', 'img'].
+                    If None, retrieves attributes for all tags.
+        
+        Returns:
+        - attribute_values: Dictionary where each key is an attribute and the value is a list of unique values for that attribute.
+        """
+        attribute_values = {}
+        tags_list = tags_list or self.all_tags_and_attributes
+        # Get all tags matching tags_list criteria
+        for tag_name in tags_list:
+            for tag in self.soup.find_all(tag_name):
+                for attr, value in tag.attrs.items():
+                    if attr not in attribute_values:
+                        attribute_values[attr] = set()
+                    
+                    # Add attribute values
+                    if isinstance(value, list):
+                        attribute_values[attr].update(value)
+                    else:
+                        attribute_values[attr].add(value)
+        
+        # Convert each set to a list for consistency
+        for attr, values in attribute_values.items():
+            attribute_values[attr] = list(values)
+
+        # Capture JavaScript URLs inside <script> tags
+        attribute_values['script_links'] = self.get_js_links()
+
+        return attribute_values
+
+    def get_js_links(self):
+        """Extract URLs embedded in JavaScript within <script> tags."""
+        js_links = []
+        script_tags = self.soup.find_all('script')
+        for script in script_tags:
+            # Find URLs in the JavaScript code
+            urls_in_js = re.findall(r'["\'](https?://[^"\']+|/[^"\']+)["\']', script.get_text())
+            js_links.extend(urls_in_js)
+        return list(set(js_links))  # Remove duplicates
+    
+    @property
+    def url(self):
+        return self._url
+    @url.setter
+    def url(self, new_url):
+        self._url = new_url
+
+class SoupManagerSingleton():
+    _instance = None
+    @staticmethod
+    def get_instance(url_mgr,requestManager,parse_type="html.parser",source_code=None):
+        if SoupManagerSingleton._instance is None:
+            SoupManagerSingleton._instance = SoupManager(url_mgr,requestManager,parse_type=parse_type,source_code=source_code)
+        elif parse_type != SoupManagerSingleton._instance.parse_type  or source_code != SoupManagerSingleton._instance.source_code:
+            SoupManagerSingleton._instance = SoupManager(url_mgr,requestManager,parse_type=parse_type,source_code=source_code)
+        return SoupManagerSingleton._instance
+def get_soup_mgr(url=None,url_mgr=None,source_code=None,req_mgr=None,soup_mgr=None,parse_type="html.parser"):
+    url_mgr = get_url_mgr(url=url,url_mgr=url_mgr)
+    url = get_url(url=url,url_mgr=url_mgr)
+    req_mgr = get_req_mgr(url_mgr=url_mgr,url=url,source_code=source_code)
+    soup_mgr = soup_mgr or soupManager(url_mgr=url_mgr,req_mgr=req_mgr,url=url,source_code=source_code)
+    return soup_mgr
+def get_all_attribute_values(url=None,url_mgr=None,source_code=None,req_mgr=None,soup_mgr=None,tags_list = None,parse_type="html.parser"):
+    soup_mgr = get_soup_mgr(url=url,url_mgr=url_mgr,source_code=source_code,req_mgr=req_mgr,soup_mgr=soup_mgr)
+    return soup_mgr.get_all_attribute_values(tags_list=tags_list)
+def get_soup(url=None,url_mgr=None,req_mgr=None,source_code=None,soup_mgr=None,parse_type="html.parser"):
+    if source_code or soup_mgr:
+        if soup_mgr:
+            return soup_mgr.soup
+        return BeautifulSoup(source_code, parse_type)
+    url_mgr = get_url_mgr(url=url,url_mgr=url_mgr)
+    url = get_url(url=url,url_mgr=url_mgr)
+    req_mgr = req_mgr or get_req_mgr(url_mgr=url_mgr,url=url,source_code=source_code)
+    source_code = req_mgr.source_code
+    soup_mgr = get_soup_mgr(url=url,url_mgr=url_mgr,source_code=source_code,req_mgr=req_mgr,soup_mgr=soup_mgr)
+    return soup_mgr.soup

abstract_webtools/managers/urlManager/__init__.py

@@ -0,0 +1 @@
+from .urlManager import *

abstract_webtools/managers/urlManager/urlManager.py

@@ -0,0 +1,230 @@
+from ..abstract_webtools import *
+class urlManager:
+    """
+    urlManager is a class for managing URLs, including cleaning, validating, and finding the correct version.
+
+    Args:
+        url (str or None): The URL to manage (default is None).
+        session (requests.Session): A custom requests session (default is the requests module's session).
+
+    Attributes:
+        session (requests.Session): The requests session used for making HTTP requests.
+        clean_urls (list): List of cleaned URL variations.
+        url (str): The current URL.
+        protocol (str): The protocol part of the URL (e.g., "https").
+        domain (str): The domain part of the URL (e.g., "example.com").
+        path (str): The path part of the URL (e.g., "/path/to/resource").
+        query (str): The query part of the URL (e.g., "?param=value").
+        all_urls (list): List of all URLs (not used in the provided code).
+
+    Methods:
+        url_to_pieces(url): Split a URL into its protocol, domain, path, and query components.
+        clean_url(url): Return a list of potential URL versions with and without 'www' and 'http(s)'.
+        get_correct_url(url): Get the correct version of the URL from possible variations.
+        update_url(url): Update the URL and related attributes.
+        get_domain(url): Get the domain name from a URL.
+        url_join(url, path): Join a base URL with a path.
+        is_valid_url(url): Check if a URL is valid.
+        make_valid(href, url): Make a URL valid by joining it with a base URL.
+        get_relative_href(url, href): Get the relative href URL by joining it with a base URL.
+
+    Note:
+        - The urlManager class provides methods for managing URLs, including cleaning and validating them.
+        - It also includes methods for joining and validating relative URLs.
+    """
+
+    def __init__(self, url=None, session=None):
+        """
+        Initialize a urlManager instance.
+
+        Args:
+            url (str or None): The URL to manage (default is None).
+            session (requests.Session): A custom requests session (default is the requests module's session).
+        """
+        url = url or 'www.example.com'
+        self._url=url
+        self.url = url
+        self.session= session or requests
+        self.clean_urls = self.clean_url(url=url)
+        self.url = self.get_correct_url(clean_urls=self.clean_urls)
+        url_pieces = self.url_to_pieces(url=self.url)
+        self.protocol,self.domain,self.path,self.query=url_pieces
+        self.all_urls = []
+    def url_to_pieces(self, url):
+        
+        try:
+            match = re.match(r'^(https?)?://?([^/]+)(/[^?]+)?(\?.+)?', url)
+            if match:
+                protocol = match.group(1) if match.group(1) else None
+                domain = match.group(2) if match.group(1) else None
+                path = match.group(3) if match.group(3) else ""  # Handle None
+                query = match.group(4) if match.group(4) else ""  # Handle None
+        except:
+            print(f'the url {url} was not reachable')
+            protocol,domain,path,query=None,None,"",""
+        return protocol, domain, path, query
+
+    def clean_url(self,url=None) -> list:
+        """
+        Given a URL, return a list with potential URL versions including with and without 'www.', 
+        and with 'http://' and 'https://'.
+        """
+        url = url or self.url 
+        urls=[]
+        if url:
+            # Remove http:// or https:// prefix
+            cleaned = url.replace("http://", "").replace("https://", "")
+            no_subdomain = cleaned.replace("www.", "", 1)
+            
+            urls = [
+                f"https://{cleaned}",
+                f"http://{cleaned}",
+            ]
+
+            # Add variants without 'www' if it was present
+            if cleaned != no_subdomain:
+                urls.extend([
+                    f"https://{no_subdomain}",
+                    f"http://{no_subdomain}",
+                ])
+
+            # Add variants with 'www' if it wasn't present
+            else:
+                urls.extend([
+                    f"https://www.{cleaned}",
+                    f"http://www.{cleaned}",
+                ])
+
+        return urls
+
+    def get_correct_url(self,url=None,clean_urls=None) -> (str or None):
+        """
+        Gets the correct URL from the possible variations by trying each one with an HTTP request.
+
+        Args:
+            url (str): The URL to find the correct version of.
+            session (type(requests.Session), optional): The requests session to use for making HTTP requests.
+                Defaults to requests.
+
+        Returns:
+            str: The correct version of the URL if found, or None if none of the variations are valid.
+        """
+        self.url = url
+        if url==None and clean_urls != None:
+            if self.url:
+                url=self.url or clean_urls[0]
+        if url!=None and clean_urls==None:
+            clean_urls=self.clean_url(url)
+        elif url==None and clean_urls==None:
+            url=self.url
+            clean_urls=self.clean_urls
+        # Get the correct URL from the possible variations
+        for url in clean_urls:
+            try:
+                source = self.session.get(url)
+                return url
+            except requests.exceptions.RequestException as e:
+                print(e)
+        return None
+    def update_url(self,url):
+        # These methods seem essential for setting up the urlManager object.
+        self.url = url
+        self.clean_urls = self.clean_url()
+        self.correct_url = self.get_correct_url()
+        self.url =self.correct_url
+        self.protocol,self.domain,self.path,self.query=self.url_to_pieces(url=self.url)
+        self.all_urls = []
+    def get_domain(self,url=None):
+        url = url or self.url 
+        return urlparse(url).netloc
+    def url_join(self,url,path):
+        url = eatOuter(url,['/'])
+        path = eatInner(path,['/'])
+        slash=''
+        if path[0] not in ['?','&']:
+            slash = '/'
+        url = url+slash+path
+        return url
+    @property
+    def url(self):
+        return self._url
+    @url.setter
+    def url(self, new_url):
+        self._url = new_url
+    def is_valid_url(self,url=None):
+        """
+        Check if the given URL is valid.
+        """
+        url = url or self.url 
+        parsed = urlparse(url)
+        return bool(parsed.netloc) and bool(parsed.scheme)
+    
+    def make_valid(self,href,url=None):
+        def is_valid_url(url):
+            url = url or self.url 
+            """
+            Check if the given URL is valid.
+            """
+            parsed = urlparse(url)
+            return bool(parsed.netloc) and bool(parsed.scheme)
+        if is_valid_url(href):
+            return href
+        new_link=urljoin(url,href)
+        if is_valid_url(new_link):
+            return new_link
+        return False
+    
+    def get_relative_href(self,url,href):
+        # join the URL if it's relative (not an absolute link)
+        url = url or self.url 
+        href = urljoin(url, href)
+        parsed_href = urlparse(href)
+        # remove URL GET parameters, URL fragments, etc.
+        href = parsed_href.scheme + "://" + parsed_href.netloc + parsed_href.path
+        return href
+    def url_basename(self,url=None):
+        url = url or self.url 
+        path = urllib.parse.urlparse(url).path
+        return path.strip('/').split('/')[-1]
+
+
+    def base_url(self,url=None):
+        url = url or self.url 
+        return re.match(r'https?://[^?#]+/', url).group()
+
+
+    def urljoin(self,base, path):
+        if isinstance(path, bytes):
+            path = path.decode()
+        if not isinstance(path, str) or not path:
+            return None
+        if re.match(r'^(?:[a-zA-Z][a-zA-Z0-9+-.]*:)?//', path):
+            return path
+        if isinstance(base, bytes):
+            base = base.decode()
+        if not isinstance(base, str) or not re.match(
+                r'^(?:https?:)?//', base):
+            return None
+        return urllib.parse.urljoin(base, path)
+class urlManagerSingleton:
+    _instance = None
+    @staticmethod
+    def get_instance(url=None,session=requests):
+        if urlManagerSingleton._instance is None:
+            urlManagerSingleton._instance = urlManager(url,session=session)
+        elif urlManagerSingleton._instance.session != session or urlManagerSingleton._instance.url != url:
+            urlManagerSingleton._instance = urlManager(url,session=session)
+        return urlManagerSingleton._instance
+
+def get_url(url=None,url_mgr=None):
+    if not url and not url_mgr:
+        return None
+    if url:
+        url_mgr = urlManager(url)
+    return url_mgr.url
+def get_url_mgr(url=None,url_mgr=None):
+    if url_mgr == None and url:
+         url_mgr = urlManager(url=url)
+    if url_mgr and url == None:
+        url = url_mgr.url
+    return url_mgr 

abstract_webtools/managers/videoDownloader.py

@@ -1,5 +1,8 @@
-from abstract_webtools import requestManager, urlManager, soupManager, requests, linkManager
-import threading,os,re,yt_dlp,urllib.request,m3u8_To_MP4,subprocess
+from .requestManager.requestManager import requestManager
+from .urlManager.urlManager import urlManager
+from .soupManager.soupManager import soupManager
+from .linkManager.linkManager import linkManager
+import threading,os,re,yt_dlp,urllib.request,m3u8_To_MP4,subprocess,requests
 from abstract_utilities import get_logFile,safe_dump_to_file
 from m3u8 import M3U8  # Install: pip install m3u8
 from urllib.parse import urljoin

{abstract_webtools-0.1.6.27.dist-info → abstract_webtools-0.1.6.29.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: abstract_webtools
-Version: 0.1.6.27
+Version: 0.1.6.29
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff

{abstract_webtools-0.1.6.27.dist-info → abstract_webtools-0.1.6.29.dist-info}/RECORD

@@ -23,11 +23,19 @@ abstract_webtools/managers/sslManager.py,sha256=C-QgQw9CW84uOE5kx2MPjC3RsLbE2JQq
 abstract_webtools/managers/tlsAdapter.py,sha256=XZSMZz9EUOhv-h3_Waf6mjV1dA3oN_M_oWuoo4VZ_HE,1454
 abstract_webtools/managers/urlManager.py,sha256=Dvf-TiSo5j_YjZS2Eq6lFfbhveneD6NA_wEE0xUXy_E,8858
 abstract_webtools/managers/userAgentManager.py,sha256=33SB2p2FG7EYZl7l2iYm1U4gI9PcdkGTZHw5lg_Ogrw,1653
-abstract_webtools/managers/videoDownloader.py,sha256=7RGgk4-ctTED-62rug0u8ubpxReExsjhcVFq5wKc2hk,10271
+abstract_webtools/managers/videoDownloader.py,sha256=RgwPZh2gVdIbToZJFC6nNh_H7hAX9xt6RfTS9xiRVxU,10389
 abstract_webtools/managers/videoDownloader2.py,sha256=v3H6akdhvVWGrB-r35m3cp_-aKkNWadpfCiMylOnv6w,12748
+abstract_webtools/managers/linkManager/__init__.py,sha256=NpfWNzvTLSfsIWSeLYIxPzeLHADk_grSx5rfgCeWERw,27
+abstract_webtools/managers/linkManager/linkManager.py,sha256=m6y9s8jknrTX8RtOAFKeHd4yd23G7Rgf0T7Sp7wmHUw,12180
+abstract_webtools/managers/requestManager/__init__.py,sha256=z2qGtweEoO_OKr959LGxVXEMu1hu7PIkmh89BEh5TI8,30
+abstract_webtools/managers/requestManager/requestManager.py,sha256=zXD31WAYghV1OjnTQzRQnQGqZz6_J4mjHTdNLnBop_0,17343
+abstract_webtools/managers/soupManager/__init__.py,sha256=mqfXfqM9sWlYpOkoXUqtBoVvk2KQx1862NnmRVJwGtY,27
+abstract_webtools/managers/soupManager/soupManager.py,sha256=-_mRCWlyzfKlF64UU53WXBmCvJ98jQ4GyHh8S8Pw3xs,17198
+abstract_webtools/managers/urlManager/__init__.py,sha256=gaJCHeK91Z-eYsBnxgdhbIUten1-gbx-zqx70R6ag-Y,26
+abstract_webtools/managers/urlManager/urlManager.py,sha256=Dvf-TiSo5j_YjZS2Eq6lFfbhveneD6NA_wEE0xUXy_E,8858
 abstract_webtools/managers/videos/Heather brooke swallo from condom.mp4,sha256=h-bKFLAHt7pGLGu4EcMvSSox7BPRK0Nga3u813iMVKQ,8335544
-abstract_webtools-0.1.6.27.dist-info/LICENSE,sha256=g3WEJFiVS27HyCGRTwKSsMLyciMaGFdWcZGOe1QalZk,3877
-abstract_webtools-0.1.6.27.dist-info/METADATA,sha256=UoJxhmemLf8FSnKEGv9i36HMf5xNCqpK3kb4BDiUp1s,16051
-abstract_webtools-0.1.6.27.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-abstract_webtools-0.1.6.27.dist-info/top_level.txt,sha256=2DMJ7RmjTcjCsa-uwAV0K6eXXlIIkFDEjBLg_uyCmCI,18
-abstract_webtools-0.1.6.27.dist-info/RECORD,,
+abstract_webtools-0.1.6.29.dist-info/LICENSE,sha256=g3WEJFiVS27HyCGRTwKSsMLyciMaGFdWcZGOe1QalZk,3877
+abstract_webtools-0.1.6.29.dist-info/METADATA,sha256=6rXZpsWEuV75cR7_4MH63_E2cT6lydRCi9Fxv1XtH00,16051
+abstract_webtools-0.1.6.29.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+abstract_webtools-0.1.6.29.dist-info/top_level.txt,sha256=2DMJ7RmjTcjCsa-uwAV0K6eXXlIIkFDEjBLg_uyCmCI,18
+abstract_webtools-0.1.6.29.dist-info/RECORD,,