abstract-webtools 0.1.5.81__py3-none-any.whl → 0.1.5.83__py3-none-any.whl

abstract_webtools/abstract_webtools.py

@@ -84,9 +84,12 @@ from requests.adapters import HTTPAdapter
 from urllib.parse import urlparse, urljoin
 from requests.packages.urllib3.util import ssl_
 from requests.packages.urllib3.poolmanager import PoolManager
+from urllib.parse import urlparse, parse_qs
+import time
+import requests
+from .managers import *
 from abstract_utilities import get_time_stamp,get_sleep,sleep_count_down,eatInner,eatAll,eatOuter,ThreadManager
 logging.basicConfig(level=logging.INFO)
-
 def try_request(request):
     try:
         respnse = requests.get(url)
@@ -95,1758 +98,3 @@ def try_request(request):
         response = None
     return response
 
-class CipherManager:
-    @staticmethod
-    def  get_default_ciphers()-> list:
-        return [
-            "ECDHE-RSA-AES256-GCM-SHA384", "ECDHE-ECDSA-AES256-GCM-SHA384",
-            "ECDHE-RSA-AES256-SHA384", "ECDHE-ECDSA-AES256-SHA384",
-            "ECDHE-RSA-AES256-SHA", "ECDHE-ECDSA-AES256-SHA",
-            "ECDHE-RSA-AES128-GCM-SHA256", "ECDHE-RSA-AES128-SHA256",
-            "ECDHE-ECDSA-AES128-GCM-SHA256", "ECDHE-ECDSA-AES128-SHA256",
-            "AES256-SHA", "AES128-SHA"
-        ]
-
-    def __init__(self,cipher_list=None):
-        if cipher_list == None:
-            cipher_list=self.get_default_ciphers()
-        self.cipher_list = cipher_list
-        self.create_list()
-        self.ciphers_string = self.add_string_list()
-    def add_string_list(self):
-        if len(self.cipher_list)==0:
-            return ''
-        return','.join(self.cipher_list)
-    def create_list(self):
-        if self.cipher_list == None:
-            self.cipher_list= []
-        elif isinstance(self.cipher_list, str):
-            self.cipher_list=self.cipher_list.split(',')
-        if isinstance(self.cipher_list, str):
-            self.cipher_list=[self.cipher_list]
-class CipherManagerSingleton:
-    _instance = None
-    @staticmethod
-    def get_instance(cipher_list=None):
-        if CipherManagerSingleton._instance is None:
-            CipherManagerSingleton._instance = CipherManager(cipher_list=cipher_list)
-        elif CipherManagerSingleton._instance.cipher_list != cipher_list:
-            CipherManagerSingleton._instance = CipherManager(cipher_list=cipher_list)
-        return CipherManagerSingleton._instance
-class SSLManager:
-    def __init__(self, ciphers=None, ssl_options=None, certification=None):
-        self.ciphers = ciphers or CipherManager().ciphers_string
-        self.ssl_options = ssl_options or self.get_default_ssl_settings()
-        self.certification = certification or ssl.CERT_REQUIRED
-        self.ssl_context = self.get_context()
-    def get_default_ssl_settings(self):
-        return ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1 | ssl.OP_NO_COMPRESSION
-    def get_context(self):
-        return ssl_.create_urllib3_context(ciphers=self.ciphers, cert_reqs=self.certification, options=self.ssl_options)
-
-class SSLManagerSingleton:
-    _instance = None
-    @staticmethod
-    def get_instance(ciphers=None, ssl_options_list=None, certification=None):
-        if SSLManagerSingleton._instance is None:
-            SSLManagerSingleton._instance = SSLManager(ciphers=ciphers, ssl_options_list=ssl_options_list, certification=certification)
-        elif SSLManagerSingleton._instance.cipher_manager.ciphers_string != ciphers or SSLManagerSingleton._instance.ssl_options_list !=ssl_options_list or SSLManagerSingleton._instance.certification !=certification:
-            SSLManagerSingleton._instance = SSLManager(ciphers=ciphers, ssl_options_list=ssl_options_list, certification=certification)
-        return SSLManagerSingleton._instance
-class TLSAdapter(HTTPAdapter):
-    def __init__(self, ssl_manager=None,ciphers=None, certification: Optional[str] = None, ssl_options: Optional[List[str]] = None):
-        if ssl_manager == None:
-            ssl_manager = SSLManager(ciphers=ciphers, ssl_options=ssl_options, certification=certification)
-        self.ssl_manager = ssl_manager
-        self.ciphers = ssl_manager.ciphers
-        self.certification = ssl_manager.certification
-        self.ssl_options = ssl_manager.ssl_options
-        self.ssl_context = self.ssl_manager.ssl_context
-        super().__init__()
-
-    def init_poolmanager(self, *args, **kwargs):
-        kwargs['ssl_context'] = self.ssl_context
-        return super().init_poolmanager(*args, **kwargs)
-class TLSAdapterSingleton:
-    _instance: Optional[TLSAdapter] = None
-    
-    @staticmethod
-    def get_instance(ciphers: Optional[List[str]] = None, certification: Optional[str] = None, ssl_options: Optional[List[str]] = None) -> TLSAdapter:
-        if (not TLSAdapterSingleton._instance) or (
-            TLSAdapterSingleton._instance.ciphers != ciphers or 
-            TLSAdapterSingleton._instance.certification != certification or 
-            TLSAdapterSingleton._instance.ssl_options != ssl_options
-        ):
-            TLSAdapterSingleton._instance = TLSAdapter(ciphers=ciphers, certification=certification, ssl_options=ssl_options)
-        return TLSAdapterSingleton._instance
-class UserAgentManager:
-    def __init__(self, os=None, browser=None, version=None,user_agent=None):
-        self.os = os or 'Windows'
-        self.browser = browser or "Firefox"
-        self.version = version or '42.0'
-        self.user_agent = user_agent or self.get_user_agent()
-        self.header = self.user_agent_header()
-    @staticmethod
-    def user_agent_db():
-        from .big_user_agent_list import big_user_agent_dict
-        return big_user_agent_dict
-
-    def get_user_agent(self):
-        ua_db = self.user_agent_db()
-
-        if self.os and self.os in ua_db:
-            os_db = ua_db[self.os]
-        else:
-            os_db = random.choice(list(ua_db.values()))
-
-        if self.browser and self.browser in os_db:
-            browser_db = os_db[self.browser]
-        else:
-            browser_db = random.choice(list(os_db.values()))
-
-        if self.version and self.version in browser_db:
-            return browser_db[self.version]
-        else:
-            return random.choice(list(browser_db.values()))
-
-    def user_agent_header(self):
-        return {"user-agent": self.user_agent}
-class UserAgentManagerSingleton:
-    _instance = None
-    @staticmethod
-    def get_instance(user_agent=UserAgentManager().get_user_agent()[0]):
-        if UserAgentManagerSingleton._instance is None:
-            UserAgentManagerSingleton._instance = UserAgentManager(user_agent=user_agent)
-        elif UserAgentManagerSingleton._instance.user_agent != user_agent:
-            UserAgentManagerSingleton._instance = UserAgentManager(user_agent=user_agent)
-        return UserAgentManagerSingleton._instance
-class NetworkManager:
-    def __init__(self, user_agent_manager=None,ssl_manager=None, tls_adapter=None,user_agent=None,proxies=None,cookies=None,ciphers=None, certification: Optional[str] = None, ssl_options: Optional[List[str]] = None):
-        if ssl_manager == None:
-            ssl_manager = SSLManager(ciphers=ciphers, ssl_options=ssl_options, certification=certification)
-        self.ssl_manager=ssl_manager
-        if tls_adapter == None:
-            tls_adapter=TLSAdapter(ssl_manager=ssl_manager,ciphers=ciphers, certification=certification, ssl_options=ssl_options)
-        self.tls_adapter=tls_adapter
-        self.ciphers=tls_adapter.ciphers
-        self.certification=tls_adapter.certification
-        self.ssl_options=tls_adapter.ssl_options
-        self.proxies=None or {}
-        self.cookies=cookies or "cb4c883efc59d0e990caf7508902591f4569e7bf-1617321078-0-150"
-class MySocketClient:
-    def __init__(self, ip_address=None, port=None,domain=None):
-        self.sock
-        self.ip_address= ip_address or None
-        self.port = port  or None
-        
-        self.domain = domain  or None
-    def receive_data(self):
-        chunks = []
-        while True:
-            chunk = self.sock.recv(4096)
-            if chunk:
-                chunks.append(chunk)
-            else:
-                break
-        return b''.join(chunks).decode('utf-8')
-    def _parse_socket_response_as_json(self, data, *args, **kwargs):
-        return self._parse_json(data[data.find('{'):data.rfind('}') + 1], *args, **kwargs)
-    def process_data(self):
-        data = self.receive_data()
-        return self._parse_socket_response_as_json(data)
-    def _parse_json(self,json_string):
-        return json.loads(json_string)
-    def get_ip(self,domain=None):
-        try:
-            return self.sock.gethostbyname(domain if domain != None else self.domain)
-        except self.sock.gaierror:
-            return None
-    def grt_host_name(self,ip_address=None):
-        return self.sock.gethostbyaddr(ip_address if ip_address != None else self.ip_address)
-    def toggle_sock(self):
-        if self.sock != None:
-            self.sock.close()
-        else:
-            self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-            if host and socket:
-                self.sock.connect((host, port))
-class MySocketClient():
-    _instance = None
-    @staticmethod
-    def get_instance(ip_address='local_host',port=22,domain="example.com"):
-        if MySocketClientSingleton._instance is None:
-            MySocketClientSingleton._instance = MySocketClient(ip_address=ip_address,port=port,domain=domain)
-        elif MySocketClientSingleton._instance.ip_address != ip_address or MySocketClientSingleton._instance.port != port or UrlManagerSingleton._instance.domain != domain:
-            MySocketClientSingleton._instance = MySocketClient(ip_address=ip_address,port=port,domain=domain)
-        return MySocketClient
-
-class UrlManager:
-    """
-    UrlManager is a class for managing URLs, including cleaning, validating, and finding the correct version.
-
-    Args:
-        url (str or None): The URL to manage (default is None).
-        session (requests.Session): A custom requests session (default is the requests module's session).
-
-    Attributes:
-        session (requests.Session): The requests session used for making HTTP requests.
-        clean_urls (list): List of cleaned URL variations.
-        url (str): The current URL.
-        protocol (str): The protocol part of the URL (e.g., "https").
-        domain (str): The domain part of the URL (e.g., "example.com").
-        path (str): The path part of the URL (e.g., "/path/to/resource").
-        query (str): The query part of the URL (e.g., "?param=value").
-        all_urls (list): List of all URLs (not used in the provided code).
-
-    Methods:
-        url_to_pieces(url): Split a URL into its protocol, domain, path, and query components.
-        clean_url(url): Return a list of potential URL versions with and without 'www' and 'http(s)'.
-        get_correct_url(url): Get the correct version of the URL from possible variations.
-        update_url(url): Update the URL and related attributes.
-        get_domain(url): Get the domain name from a URL.
-        url_join(url, path): Join a base URL with a path.
-        is_valid_url(url): Check if a URL is valid.
-        make_valid(href, url): Make a URL valid by joining it with a base URL.
-        get_relative_href(url, href): Get the relative href URL by joining it with a base URL.
-
-    Note:
-        - The UrlManager class provides methods for managing URLs, including cleaning and validating them.
-        - It also includes methods for joining and validating relative URLs.
-    """
-
-    def __init__(self, url=None, session=None):
-        """
-        Initialize a UrlManager instance.
-
-        Args:
-            url (str or None): The URL to manage (default is None).
-            session (requests.Session): A custom requests session (default is the requests module's session).
-        """
-        self._url=url or 'www.example.com'
-        self.url = url or 'www.example.com'
-        self.session= session or requests
-        self.clean_urls = self.clean_url(url=url)
-        self.url = self.get_correct_url(clean_urls=self.clean_urls)
-        url_pieces = self.url_to_pieces(url=self.url)
-        self.protocol,self.domain,self.path,self.query=url_pieces
-        self.all_urls = []
-    def url_to_pieces(self, url):
-        
-        try:
-            match = re.match(r'^(https?)?://?([^/]+)(/[^?]+)?(\?.+)?', url)
-            if match:
-                protocol = match.group(1) if match.group(1) else None
-                domain = match.group(2) if match.group(1) else None
-                path = match.group(3) if match.group(3) else ""  # Handle None
-                query = match.group(4) if match.group(4) else ""  # Handle None
-        except:
-            print(f'the url {url} was not reachable')
-            protocol,domain,path,query=None,None,"",""
-        return protocol, domain, path, query
-
-    def clean_url(self,url=None) -> list:
-        """
-        Given a URL, return a list with potential URL versions including with and without 'www.', 
-        and with 'http://' and 'https://'.
-        """
-        if url == None:
-            url=self.url
-        urls=[]
-        if url:
-            # Remove http:// or https:// prefix
-            cleaned = url.replace("http://", "").replace("https://", "")
-            no_subdomain = cleaned.replace("www.", "", 1)
-            
-            urls = [
-                f"https://{cleaned}",
-                f"http://{cleaned}",
-            ]
-
-            # Add variants without 'www' if it was present
-            if cleaned != no_subdomain:
-                urls.extend([
-                    f"https://{no_subdomain}",
-                    f"http://{no_subdomain}",
-                ])
-
-            # Add variants with 'www' if it wasn't present
-            else:
-                urls.extend([
-                    f"https://www.{cleaned}",
-                    f"http://www.{cleaned}",
-                ])
-
-        return urls
-
-    def get_correct_url(self,url=None,clean_urls=None) -> (str or None):
-        """
-        Gets the correct URL from the possible variations by trying each one with an HTTP request.
-
-        Args:
-            url (str): The URL to find the correct version of.
-            session (type(requests.Session), optional): The requests session to use for making HTTP requests.
-                Defaults to requests.
-
-        Returns:
-            str: The correct version of the URL if found, or None if none of the variations are valid.
-        """
-        self.url = url
-        if url==None and clean_urls != None:
-            if self.url:
-                url=self.url or clean_urls[0]
-        if url!=None and clean_urls==None:
-            clean_urls=self.clean_url(url)
-        elif url==None and clean_urls==None:
-            url=self.url
-            clean_urls=self.clean_urls
-        # Get the correct URL from the possible variations
-        for url in clean_urls:
-            try:
-                source = self.session.get(url)
-                return url
-            except requests.exceptions.RequestException as e:
-                print(e)
-        return None
-    def update_url(self,url):
-        # These methods seem essential for setting up the UrlManager object.
-        self.url = url
-        self.clean_urls = self.clean_url()
-        self.correct_url = self.get_correct_url()
-        self.url =self.correct_url
-        self.protocol,self.domain,self.path,self.query=self.url_to_pieces(url=self.url)
-        self.all_urls = []
-    def get_domain(self,url):
-        return urlparse(url).netloc
-    def url_join(self,url,path):
-        url = eatOuter(url,['/'])
-        path = eatInner(path,['/'])
-        slash=''
-        if path[0] not in ['?','&']:
-            slash = '/'
-        url = url+slash+path
-        return url
-    @property
-    def url(self):
-        return self._url
-    @url.setter
-    def url(self, new_url):
-        self._url = new_url
-    @staticmethod
-    def is_valid_url(url):
-        """
-        Check if the given URL is valid.
-        """
-        parsed = urlparse(url)
-        return bool(parsed.netloc) and bool(parsed.scheme)
-    @staticmethod
-    def make_valid(href,url):
-        def is_valid_url(url):
-            """
-            Check if the given URL is valid.
-            """
-            parsed = urlparse(url)
-            return bool(parsed.netloc) and bool(parsed.scheme)
-        if is_valid_url(href):
-            return href
-        new_link=urljoin(url,href)
-        if is_valid_url(new_link):
-            return new_link
-        return False
-    @staticmethod
-    def get_relative_href(url,href):
-        # join the URL if it's relative (not an absolute link)
-        href = urljoin(url, href)
-        parsed_href = urlparse(href)
-        # remove URL GET parameters, URL fragments, etc.
-        href = parsed_href.scheme + "://" + parsed_href.netloc + parsed_href.path
-        return href
-    def url_basename(url):
-        path = urllib.parse.urlparse(url).path
-        return path.strip('/').split('/')[-1]
-
-
-    def base_url(url):
-        return re.match(r'https?://[^?#]+/', url).group()
-
-
-    def urljoin(base, path):
-        if isinstance(path, bytes):
-            path = path.decode()
-        if not isinstance(path, str) or not path:
-            return None
-        if re.match(r'^(?:[a-zA-Z][a-zA-Z0-9+-.]*:)?//', path):
-            return path
-        if isinstance(base, bytes):
-            base = base.decode()
-        if not isinstance(base, str) or not re.match(
-                r'^(?:https?:)?//', base):
-            return None
-        return urllib.parse.urljoin(base, path)
-class UrlManagerSingleton:
-    _instance = None
-    @staticmethod
-    def get_instance(url=None,session=requests):
-        if UrlManagerSingleton._instance is None:
-            UrlManagerSingleton._instance = UrlManager(url,session=session)
-        elif UrlManagerSingleton._instance.session != session or UrlManagerSingleton._instance.url != url:
-            UrlManagerSingleton._instance = UrlManager(url,session=session)
-        return UrlManagerSingleton._instance
-class SafeRequest:
-    """
-    SafeRequest is a class for making HTTP requests with error handling and retries.
-
-    Args:
-        url (str or None): The URL to make requests to (default is None).
-        url_manager (UrlManager or None): An instance of UrlManager (default is None).
-        network_manager (NetworkManager or None): An instance of NetworkManager (default is None).
-        user_agent_manager (UserAgentManager or None): An instance of UserAgentManager (default is None).
-        ssl_manager (SSlManager or None): An instance of SSLManager (default is None).
-        tls_adapter (TLSAdapter or None): An instance of TLSAdapter (default is None).
-        user_agent (str or None): The user agent string to use for requests (default is None).
-        proxies (dict or None): Proxy settings for requests (default is None).
-        headers (dict or None): Additional headers for requests (default is None).
-        cookies (dict or None): Cookie settings for requests (default is None).
-        session (requests.Session or None): A custom requests session (default is None).
-        adapter (str or None): A custom adapter for requests (default is None).
-        protocol (str or None): The protocol to use for requests (default is 'https://').
-        ciphers (str or None): Cipher settings for requests (default is None).
-        auth (tuple or None): Authentication credentials (default is None).
-        login_url (str or None): The URL for authentication (default is None).
-        email (str or None): Email for authentication (default is None).
-        password (str or None): Password for authentication (default is None).
-        certification (str or None): Certification settings for requests (default is None).
-        ssl_options (str or None): SSL options for requests (default is None).
-        stream (bool): Whether to stream the response content (default is False).
-        timeout (float or None): Timeout for requests (default is None).
-        last_request_time (float or None): Timestamp of the last request (default is None).
-        max_retries (int or None): Maximum number of retries for requests (default is None).
-        request_wait_limit (float or None): Wait time between requests (default is None).
-
-    Methods:
-        update_url_manager(url_manager): Update the URL manager and reinitialize the SafeRequest.
-        update_url(url): Update the URL and reinitialize the SafeRequest.
-        re_initialize(): Reinitialize the SafeRequest with the current settings.
-        authenticate(s, login_url=None, email=None, password=None, checkbox=None, dropdown=None): Authenticate and make a request.
-        fetch_response(): Fetch the response from the server.
-        initialize_session(): Initialize the requests session with custom settings.
-        process_response_data(): Process the fetched response data.
-        get_react_source_code(): Extract JavaScript and JSX source code from <script> tags.
-        get_status(url=None): Get the HTTP status code of a URL.
-        wait_between_requests(): Wait between requests based on the request_wait_limit.
-        make_request(): Make a request and handle potential errors.
-        try_request(): Try to make an HTTP request using the provided session.
-
-    Note:
-        - The SafeRequest class is designed for making HTTP requests with error handling and retries.
-        - It provides methods for authentication, response handling, and error management.
-    """
-    def __init__(self,
-                 url=None,
-                 source_code=None,
-                 url_manager=None,
-                 network_manager=None,
-                 user_agent_manager=None,
-                 ssl_manager=None,
-                 ssl_options=None,
-                 tls_adapter=None,
-                 user_agent=None,
-                 proxies=None,
-                 headers=None,
-                 cookies=None,
-                 session=None,
-                 adapter=None,
-                 protocol=None,
-                 ciphers=None,
-                 spec_login=False,
-                 login_referer=None,
-                 login_user_agent=None,
-                 auth=None,
-                 login_url=None,
-                 email = None,
-                 password=None,
-                 checkbox=None,
-                 dropdown=None,
-                 certification=None,
-                 stream=False,
-                 timeout = None,
-                 last_request_time=None,
-                 max_retries=None,
-                 request_wait_limit=None):
-        self.url_manager=url_manager or  UrlManager(url=url)
-        self._url=url
-        self.url=url
-        
- 
-        self.url_manager=url_manager
-        self._url_manager = self.url_manager
-        self.user_agent = user_agent
-        self.user_agent_manager = user_agent_manager or UserAgentManager(user_agent=self.user_agent)
-        self.headers= headers or self.user_agent_manager.header or {'Accept': '*/*'}
-        self.user_agent= self.user_agent_manager.user_agent
-        self.ciphers=ciphers or CipherManager().ciphers_string
-        self.certification=certification
-        self.ssl_options=ssl_options
-        self.ssl_manager = ssl_manager or SSLManager(ciphers=self.ciphers, ssl_options=self.ssl_options, certification=self.certification)
-        self.tls_adapter=tls_adapter or  TLSAdapter(ssl_manager=self.ssl_manager,certification=self.certification,ssl_options=self.ssl_manager.ssl_options)
-        self.network_manager= network_manager or NetworkManager(user_agent_manager=self.user_agent_manager,ssl_manager=self.ssl_manager, tls_adapter=self.tls_adapter,user_agent=user_agent,proxies=proxies,cookies=cookies,ciphers=ciphers, certification=certification, ssl_options=ssl_options)
-        self.stream=stream
-        self.tls_adapter=self.network_manager.tls_adapter
-        self.ciphers=self.network_manager.ciphers
-        self.certification=self.network_manager.certification
-        self.ssl_options=self.network_manager.ssl_options
-        self.proxies=self.network_manager.proxies
-        self.timeout=timeout
-        self.cookies=self.network_manager.cookies
-        self.session = session or requests.session()
-        self.auth = auth
-        self.spec_login=spec_login
-        self.password=password
-        self.email = email
-        self.checkbox=checkbox
-        self.dropdown=dropdown
-        self.login_url=login_url
-        self.login_user_agent=login_user_agent
-        self.login_referer=login_referer
-        self.protocol=protocol or 'https://'
-        
-        self.stream=stream if isinstance(stream,bool) else False
-        self.initialize_session()
-        self.last_request_time=last_request_time
-        self.max_retries = max_retries or 3
-        self.request_wait_limit = request_wait_limit or 1.5
-        self._response=None
-        self.make_request()
-        self.source_code = None
-        self.source_code_bytes=None
-        self.source_code_json = {}
-        self.react_source_code=[]
-        self._response_data = None
-        self.process_response_data()
-    def update_url_manager(self,url_manager):
-        self.url_manager=url_manager
-        self.re_initialize()
-    def update_url(self,url):
-        self.url_manager.update_url(url=url)
-        self.re_initialize()
-    def re_initialize(self):
-        self._response=None
-        self.make_request()
-        self.source_code = None
-        self.source_code_bytes=None
-        self.source_code_json = {}
-        self.react_source_code=[]
-        self._response_data = None
-        self.process_response_data()
-    @property
-    def response(self):
-        """Lazy-loading of response."""
-        if self._response is None:
-            self._response = self.fetch_response()
-            
-        return self._response
-    def authenticate(self,session, login_url=None, email=None, password=None,checkbox=None,dropdown=None):
-        login_urls = login_url or [self.url_manager.url,self.url_manager.domain,self.url_manager.url_join(url=self.url_manager.domain,path='login'),self.url_manager.url_join(url=self.url_manager.domain,path='auth')]
-        s = session
-        if not isinstance(login_urls,list):
-            login_urls=[login_urls]
-        for login_url in login_urls:
-            login_url_manager = UrlManager(login_url)
-            login_url = login_url_manager.url
-            
-            r = s.get(login_url)
-            soup = BeautifulSoup(r.content, "html.parser")
-            # Find the token or any CSRF protection token
-            token = soup.find('input', {'name': 'token'}).get('value') if soup.find('input', {'name': 'token'}) else None
-            if token != None:
-                break
-        login_data = {}
-        if email != None:
-            login_data['email']=email
-        if password != None:
-            login_data['password'] = password
-        if checkbox != None:
-            login_data['checkbox'] = checkbox
-        if dropdown != None:
-            login_data['dropdown']=dropdown
-        if token != None:
-            login_data['token'] = token
-        s.post(login_url, data=login_data)
-        return s
-
-    def fetch_response(self) -> Union[requests.Response, None]:
-        """Actually fetches the response from the server."""
-        # You can further adapt this method to use retries or other logic you had
-        # in your original code, but the main goal here is to fetch and return the response
-        return self.try_request()
-    def spec_auth(self, session=None, email=None, password=None, login_url=None, login_referer=None, login_user_agent=None):
-        s = session or requests.session()
-        
-        domain = self.url_manager.url_join(self.url_manager.get_correct_url(self.url_manager.domain),'login') if login_url is None else login_url
-        login_url = self.url_manager.get_correct_url(url=domain)
-        
-        login_referer = login_referer or self.url_manager.url_join(url=login_url, path='?role=fast&to=&s=1&m=1&email=YOUR_EMAIL')
-        login_user_agent = login_user_agent or 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0'
-        
-        headers = {"Referer": login_referer, 'User-Agent': login_user_agent}
-        payload = {'email': email, 'pass': password}
-        
-        page = s.get(login_url)
-        soup = BeautifulSoup(page.content, 'lxml')
-        action_url = soup.find('form')['action']
-        s.post(action_url, data=payload, headers=headers)
-        return s
-    def initialize_session(self):
-        s = self.session  
-        if self.auth:
-            s= self.auth
-        elif self.spec_login:
-            s=self.spec_auth(session=s,email=self.email, password=self.password, login_url=self.login_url, login_referer=self.login_referer, login_user_agent=self.login_user_agent)
-        elif any([self.password, self.email, self.login_url, self.checkbox, self.dropdown]):
-            s=self.authenticate(session=s, login_url=self.login_url, email=self.email, password=self.password, checkbox=self.checkbox, dropdown=self.dropdown)
-        s.proxies = self.proxies
-        s.cookies["cf_clearance"] = self.network_manager.cookies
-        s.headers.update(self.headers)
-        s.mount(self.protocol, self.network_manager.tls_adapter)
-        return s
-    def process_response_data(self):
-        """Processes the fetched response data."""
-        if not self.response:
-            return  # No data to process
-        
-        self.source_code = self.response.text
-        self.source_code_bytes = self.response.content
-        
-        if self.response.headers.get('content-type') == 'application/json':
-            data = convert_to_json(self.source_code)
-            if data:
-                self.source_code_json = data.get("response", data)
-        
-        self.get_react_source_code()
-    def get_react_source_code(self) -> list:
-        """
-        Fetches the source code of the specified URL and extracts JavaScript and JSX source code (React components).
-
-        Args:
-            url (str): The URL to fetch the source code from.
-
-        Returns:
-            list: A list of strings containing JavaScript and JSX source code found in <script> tags.
-        """
-        if self.url_manager.url is None:
-            return []
-        soup = BeautifulSoup(self.source_code_bytes,"html.parser")
-        script_tags = soup.find_all('script', type=lambda t: t and ('javascript' in t or 'jsx' in t))
-        for script_tag in script_tags:
-            self.react_source_code.append(script_tag.string)
-
-
-    def get_status(url:str=None) -> int:
-        """
-        Gets the HTTP status code of the given URL.
-
-        Args:
-            url (str): The URL to check the status of.
-
-        Returns:
-            int: The HTTP status code of the URL, or None if the request fails.
-        """
-        # Get the status code of the URL
-        return try_request(url=url).status_code
-    def wait_between_requests(self):
-        """
-        Wait between requests based on the request_wait_limit.
-        """
-        if self.last_request_time:
-            sleep_time = self.request_wait_limit - (get_time_stamp() - self.last_request_time)
-            if sleep_time > 0:
-                logging.info(f"Sleeping for {sleep_time:.2f} seconds.")
-                get_sleep(sleep_time)
-
-    def make_request(self):
-        """
-        Make a request and handle potential errors.
-        """
-        # Update the instance attributes if they are passed
-
-        self.wait_between_requests()
-        for _ in range(self.max_retries):
-            try:
-                self.try_request()  # 10 seconds timeout
-                if self.response:
-                    if self.response.status_code == 200:
-                        self.last_request_time = get_time_stamp()
-                        return self.response
-                    elif self.response.status_code == 429:
-                        logging.warning(f"Rate limited by {self.url_manager.url}. Retrying...")
-                        get_sleep(5)  # adjust this based on the server's rate limit reset time
-            except requests.Timeout as e:
-                logging.error(f"Request to {cleaned_url} timed out: {e}")
-            except requests.ConnectionError:
-                logging.error(f"Connection error for URL {self.url_manager.url}.")
-            except requests.Timeout:
-                logging.error(f"Request timeout for URL {self.url_manager.url}.")
-            except requests.RequestException as e:
-                logging.error(f"Request exception for URL {self.url_manager.url}: {e}")
-
-        logging.error(f"Failed to retrieve content from {self.url_manager.url} after {self.max_retries} retries.")
-        return None
-    def try_request(self) -> Union[requests.Response, None]:
-        """
-        Tries to make an HTTP request to the given URL using the provided session.
-
-        Args:
-            timeout (int): Timeout for the request.
-
-        Returns:
-            requests.Response or None: The response object if the request is successful, or None if the request fails.
-        """
-        try:
-            return self.session.get(url=self.url_manager.url, timeout=self.timeout,stream=self.stream)
-        except requests.exceptions.RequestException as e:
-            print(e)
-            return None
-
-    @property
-    def url(self):
-        return self.url_manager.url
-
-    @url.setter
-    def url(self, new_url):
-        self._url = new_url
-class SafeRequestSingleton:
-    _instance = None
-    @staticmethod
-    def get_instance(url=None,headers:dict=None,max_retries=3,last_request_time=None,request_wait_limit=1.5):
-        if SafeRequestSingleton._instance is None:
-            SafeRequestSingleton._instance = SafeRequest(url,url_manager=UrlManagerSingleton,headers=headers,max_retries=max_retries,last_request_time=last_request_time,request_wait_limit=request_wait_limit)
-        elif SafeRequestSingleton._instance.url != url or SafeRequestSingleton._instance.headers != headers or SafeRequestSingleton._instance.max_retries != max_retries or SafeRequestSingleton._instance.request_wait_limit != request_wait_limit:
-            SafeRequestSingleton._instance = SafeRequest(url,url_manager=UrlManagerSingleton,headers=headers,max_retries=max_retries,last_request_time=last_request_time,request_wait_limit=request_wait_limit)
-        return SafeRequestSingleton._instance
-class SoupManager:
-    """
-    SoupManager is a class for managing and parsing HTML source code using BeautifulSoup.
-
-    Args:
-        url (str or None): The URL to be parsed (default is None).
-        source_code (str or None): The HTML source code (default is None).
-        url_manager (UrlManager or None): An instance of UrlManager (default is None).
-        request_manager (SafeRequest or None): An instance of SafeRequest (default is None).
-        parse_type (str): The type of parser to be used by BeautifulSoup (default is "html.parser").
-
-    Methods:
-        re_initialize(): Reinitialize the SoupManager with the current settings.
-        update_url(url): Update the URL and reinitialize the SoupManager.
-        update_source_code(source_code): Update the source code and reinitialize the SoupManager.
-        update_request_manager(request_manager): Update the request manager and reinitialize the SoupManager.
-        update_url_manager(url_manager): Update the URL manager and reinitialize the SoupManager.
-        update_parse_type(parse_type): Update the parsing type and reinitialize the SoupManager.
-        all_links: A property that provides access to all discovered links.
-        _all_links_get(): A method to load all discovered links.
-        get_all_website_links(tag="a", attr="href"): Get all URLs belonging to the same website.
-        meta_tags: A property that provides access to all discovered meta tags.
-        _meta_tags_get(): A method to load all discovered meta tags.
-        get_meta_tags(): Get all meta tags in the source code.
-        find_all(element, soup=None): Find all instances of an HTML element in the source code.
-        get_class(class_name, soup=None): Get the specified class from the HTML source code.
-        has_attributes(tag, *attrs): Check if an HTML tag has the specified attributes.
-        get_find_all_with_attributes(*attrs): Find all HTML tags with specified attributes.
-        get_all_desired_soup(tag=None, attr=None, attr_value=None): Get HTML tags based on specified criteria.
-        extract_elements(url, tag=None, class_name=None, class_value=None): Extract portions of source code based on filters.
-        find_all_with_attributes(class_name=None, *attrs): Find classes with associated href or src attributes.
-        get_images(tag_name, class_name, class_value): Get images with specific class and attribute values.
-        discover_classes_and_meta_images(tag_name, class_name_1, class_name_2, class_value, attrs): Discover classes and meta images.
-
-    Note:
-        - The SoupManager class is designed for parsing HTML source code using BeautifulSoup.
-        - It provides various methods to extract data and discover elements within the source code.
-    """
-    def __init__(self,url=None,source_code=None,url_manager=None,request_manager=None, parse_type="html.parser"):
-        self.soup=[]
-        self.url=url
-        if url_manager == None:
-            url_manager=UrlManager(url=self.url)
-        if self.url != None and url_manager != None and url_manager.url != UrlManager(url=url).url:
-            url_manager.update_url(url=self.url)
-        self.url_manager= url_manager
-        self.url=self.url_manager.url
-        if request_manager == None:
-            request_manager = SafeRequest(url_manager=self.url_manager)
-        self.request_manager = request_manager
-        if self.request_manager.url_manager != self.url_manager:
-           self.request_manager.update_url_manager(url_manager=self.url_manager)
-        self.parse_type = parse_type
-        if source_code != None:
-            self.source_code = source_code
-        else:
-            self.source_code = self.request_manager.source_code_bytes
-        self.soup= BeautifulSoup(self.source_code, self.parse_type)
-        self._all_links_data = None
-        self._meta_tags_data = None
-    def re_initialize(self):
-        self.soup= BeautifulSoup(self.source_code, self.parse_type)
-        self._all_links_data = None
-        self._meta_tags_data = None
-    def update_url(self,url):
-        self.url_manager.update_url(url=url)
-        self.url=self.url_manager.url
-        self.request_manager.update_url(url=url)
-        self.source_code = self.request_manager.source_code_bytes
-        self.re_initialize()
-    def update_source_code(self,source_code):
-        self.source_code = source_code
-        self.re_initialize()
-    def update_request_manager(self,request_manager):
-        self.request_manager = request_manager
-        self.url_manager=self.request_manager.url_manager
-        self.url=self.url_manager.url
-        self.source_code = self.request_manager.source_code_bytes
-        self.re_initialize()
-    def update_url_manager(self,url_manager):
-        self.url_manager=url_manager
-        self.url=self.url_manager.url
-        self.request_manager.update_url_manager(url_manager=self.url_manager)
-        self.source_code = self.request_manager.source_code_bytes
-        self.re_initialize()
-    def update_parse_type(self,parse_type):
-        self.parse_type=parse_type
-        self.re_initialize()
-    @property
-    def all_links(self):
-        """This is a property that provides access to the _all_links_data attribute.
-        The first time it's accessed, it will load the data."""
-        if self._all_links_data is None:
-            print("Loading all links for the first time...")
-            self._all_links_data = self._all_links_get()
-        return self._all_links_data
-    def _all_links_get(self):
-        """A method that loads the data (can be replaced with whatever data loading logic you have)."""
-        return self.get_all_website_links()
-    def get_all_website_links(self,tag="a",attr="href") -> list:
-        """
-        Returns all URLs that are found on the specified URL and belong to the same website.
-
-        Args:
-            url (str): The URL to search for links.
-
-        Returns:
-            list: A list of URLs that belong to the same website as the specified URL.
-        """
-        all_urls=[self.url_manager.url]
-        domain = self.url_manager.domain
-        all_desired=self.get_all_desired_soup(tag=tag,attr=attr)
-        for tag in all_desired:
-            href = tag.attrs.get(attr)
-            if href == "" or href is None:
-                # href empty tag
-                continue
-            href=self.url_manager.get_relative_href(self.url_manager.url,href)
-            if not self.url_manager.is_valid_url(href):
-                # not a valid URL
-                continue
-            if href in all_urls:
-                # already in the set
-                continue
-            if domain not in href:
-                # external link
-                continue
-            all_urls.append(href)
-                
-        return all_urls
-
-
-    @property
-    def meta_tags(self):
-        """This is a property that provides access to the _all_links_data attribute.
-        The first time it's accessed, it will load the data."""
-        if self._meta_tags_data is None:
-            print("Loading all links for the first time...")
-            self._meta_tags_data = self._all_links_get()
-        return self._meta_tags_data
-    def _meta_tags_get(self):
-        """A method that loads the data (can be replaced with whatever data loading logic you have)."""
-        return self.get_meta_tags()
-    def get_meta_tags(self):
-        tags = self.find_all("meta")
-        for meta_tag in tags:
-            for attr, values in meta_tag.attrs.items():
-                if attr not in self.meta_tags:
-                    self.meta_tags[attr] = []
-                if values not in self.meta_tags[attr]:
-                    self.meta_tags[attr].append(values)
-
-                    
-    def find_all(self,element,soup=None):
-        soup = self.soup if soup == None else soup
-        return soup.find_all(element)
-    def get_class(self,class_name,soup=None):
-        soup = self.soup if soup == None else soup
-        return soup.get(class_name)
-    @staticmethod
-    def has_attributes(tag, *attrs):
-        return any(tag.has_attr(attr) for attr in attrs)
-    def get_find_all_with_attributes(self, *attrs):
-        return self.soup.find_all(lambda t: self.has_attributes(t, *attrs))
-    def find_tags_by_attributes(self, tag: str = None, attr: str = None, attr_values: List[str] = None) ->List:
-        if not tag:
-            tags = self.soup.find_all(True)  # get all tags
-        else:
-            tags = self.soup.find_all(tag)  # get specific tags
-            
-        extracted_tags = []
-        for t in tags:
-            if attr:
-                attribute_value = t.get(attr)
-                if not attribute_value:  # skip tags without the desired attribute
-                    continue
-                if attr_values and not any(value in attribute_value for value in attr_values):  # skip tags without any of the desired attribute values
-                    continue
-            extracted_tags.append(t)
-        return extracted_tags
-
-
-    def extract_elements(self,url:str=None, tag:str=None, class_name:str=None, class_value:str=None) -> list:
-        """
-        Extracts portions of the source code from the specified URL based on provided filters.
-
-        Args:
-            url (str): The URL to fetch the source code from.
-            element_type (str, optional): The HTML element type to filter by. Defaults to None.
-            attribute_name (str, optional): The attribute name to filter by. Defaults to None.
-            class_name (str, optional): The class name to filter by. Defaults to None.
-
-        Returns:
-            list: A list of strings containing portions of the source code that match the provided filters.
-        """
-        elements = []
-        # If no filters are provided, return the entire source code
-        if not tag and not class_name and not class_value:
-            elements.append(str(self.soup))
-            return elements
-        # Find elements based on the filters provided
-        if tag:
-            elements.extend([str(tags) for tags in self.get_all_desired(tag)])
-        if class_name:
-            elements.extend([str(tags) for tags in self.get_all_desired(tag={class_name: True})])
-        if class_value:
-            elements.extend([str(tags) for tags in self.get_all_desired(class_name=class_name)])
-        return elements
-    def find_all_with_attributes(self, class_name=None, *attrs):
-        """
-        Discovers classes in the HTML content of the provided URL 
-        that have associated href or src attributes.
-
-        Args:
-            base_url (str): The URL from which to discover classes.
-
-        Returns:
-            set: A set of unique class names.
-        """
-
-    
-        unique_classes = set()
-        for tag in self.get_find_all_with_attributes(*attrs):
-            class_list = self.get_class(class_name=class_name, soup=tag)
-            unique_classes.update(class_list)
-        return unique_classes
-    def get_images(self, tag_name, class_name, class_value):
-        images = []
-        for tag in self.soup.find_all(tag_name):
-            if class_name in tag.attrs and tag.attrs[class_name] == class_value:
-                content = tag.attrs.get('content', '')
-                if content:
-                    images.append(content)
-        return images
-    def extract_text_sections(self) -> list:
-        """
-        Extract all sections of text from an HTML content using BeautifulSoup.
-
-        Args:
-            html_content (str): The HTML content to be parsed.
-
-        Returns:
-            list: A list containing all sections of text.
-        """
-        # Remove any script or style elements to avoid extracting JavaScript or CSS code
-        for script in self.soup(['script', 'style']):
-            script.decompose()
-
-        # Extract text from the remaining elements
-        text_sections = self.soup.stripped_strings
-        return [text for text in text_sections if text]
-    def discover_classes_and_meta_images(self, tag_name, class_name_1, class_name_2, class_value, attrs):
-        """
-        Discovers classes in the HTML content of the provided URL 
-        that have associated href or src attributes. Also, fetches 
-        image references from meta tags.
-
-        Args:
-            base_url (str): The URL from which to discover classes and meta images.
-
-        Returns:
-            tuple: A set of unique class names and a list of meta images.
-        """
-    
-        unique_classes = self.find_all_with_attributes(class_name=class_name_1, *attrs)
-        images = self.get_images(tag_name=tag_name, class_name=class_name_2, class_value=class_value)
-        return unique_classes, images
-    def get_all_tags_and_attribute_names(self):
-        tag_names = set()  # Using a set to ensure uniqueness
-        attribute_names = set()
-        get_all = self.find_tags_by_attributes()
-        for tag in get_all:  # True matches all tags
-            tag_names.add(tag.name)
-            for attr in tag.attrs:
-                attribute_names.add(attr)
-        tag_names_list = list(tag_names)
-        attribute_names_list = list(attribute_names)
-        return {"tags":tag_names_list,"attributes":attribute_names_list}
-
-    def get_all_attribute_values(self):
-        attribute_values={}
-        get_all = self.find_tags_by_attributes()
-        for tag in get_all:  # True matches all tags
-            for attr, value in tag.attrs.items():
-                # If attribute is not yet in the dictionary, add it with an empty set
-                if attr not in attribute_values:
-                    attribute_values[attr] = set()
-                # If the attribute value is a list (e.g., class), extend the set with the list
-                if isinstance(value, list):
-                    attribute_values[attr].update(value)
-                else:
-                    attribute_values[attr].add(value)
-        for attr, values in attribute_values.items():
-            attribute_values[attr] = list(values)
-        return attribute_values
-    
-    @property
-    def url(self):
-        return self._url
-    @url.setter
-    def url(self, new_url):
-        self._url = new_url
-
-class SoupManagerSingleton():
-    _instance = None
-    @staticmethod
-    def get_instance(url_manager,request_manager,parse_type="html.parser",source_code=None):
-        if SoupManagerSingleton._instance is None:
-            SoupManagerSingleton._instance = SoupManager(url_manager,request_manager,parse_type=parse_type,source_code=source_code)
-        elif parse_type != SoupManagerSingleton._instance.parse_type  or source_code != SoupManagerSingleton._instance.source_code:
-            SoupManagerSingleton._instance = SoupManager(url_manager,request_manager,parse_type=parse_type,source_code=source_code)
-        return SoupManagerSingleton._instance
-class VideoDownloader:
-    """
-    VideoDownloader is a class for downloading videos from URLs using YouTube-DL.
-
-    Args:
-        link (str or list): The URL(s) of the video(s) to be downloaded.
-        temp_directory (str or None): The directory to store temporary video files (default is None, uses video_directory/temp_files).
-        video_directory (str or None): The directory to store downloaded videos (default is None, uses 'videos' in the current working directory).
-        remove_existing (bool): Whether to remove existing video files with the same name (default is True).
-
-    Methods:
-        count_outliers(speed, threshold): Count speed outliers below the threshold.
-        filter_outliers(speeds): Filter out speed outliers in the list of speeds.
-        remove_temps(file_name): Remove temporary video files based on the file name.
-        move_video(): Move the downloaded video to the final directory.
-        yt_dlp_downloader(url, ydl_opts={}, download=True): Download video information using YouTube-DL.
-        progress_callback(d): Callback function to monitor download progress.
-        download(): Download video(s) based on the provided URL(s).
-        monitor(): Monitor the download progress.
-        start(): Start the download and monitoring threads.
-
-    Note:
-        - The VideoDownloader class uses YouTube-DL to download videos.
-        - It allows downloading from multiple URLs.
-        - You need to have YouTube-DL installed to use this class.
-    """
-    def __init__(self, link,temp_directory=None,video_directory=None,remove_existing=True):
-        if video_directory==None:
-            video_directory=os.path.join(os.getcwd(),'videos')
-        if temp_directory == None:
-            temp_directory=os.path.join(video_directory,'temp_files')
-        self.thread_manager = ThreadManager()
-        self.pause_event = self.thread_manager.add_thread('pause_event')
-        self.link = link
-        self.temp_directory = temp_directory
-        self.video_directory = video_directory
-        self.remove_existing=remove_existing
-        self.video_urls=self.link if isinstance(self.link,list) else [self.link]
-        self.starttime = None
-        self.downloaded = 0
-        self.time_interval=60
-        self.monitoring=True
-        self.temp_file_name = None
-        self.file_name = None
-        self.dl_speed = None
-        self.dl_eta=None
-        self.total_bytes_est=None
-        self.percent_speed=None
-        self.percent=None
-        self.speed_track = []
-        self.video_url=None
-        self.last_checked = get_time_stamp()
-        self.num=0
-        self.start()
-    def count_outliers(self,speed,threshold):
-        if speed < threshold:
-            self.outlier_count+=1
-        else:
-            self.outlier_count=0
-    def filter_outliers(self,speeds):
-        # Step 1: Compute initial average
-        initial_avg = sum(speeds) / len(speeds)
-        
-        # Step 2: Remove speeds 25% under the average
-        threshold = initial_avg * 0.75  # 25% under average
-        filtered_speeds = [speed for speed in speeds if speed >= threshold]
-        
-        # Step 3: Compute the new average of the filtered list
-        if filtered_speeds:  # Ensure the list is not empty
-            self.count_outliers(speeds[-1],threshold)
-            return filtered_speeds
-        else:
-            # This can happen if all values are outliers, it's up to you how to handle it
-            self.outlier_count=0
-            return speeds
-    def remove_temps(self,file_name):
-        for temp_vid in os.listdir(self.temp_directory):
-            if len(file_name)<=len(temp_vid):
-                if temp_vid[:len(file_name)] == file_name:
-                    os.remove(os.path.join(self.temp_directory,temp_vid))
-                    print(f"removing {temp_vid} from {self.temp_directory}")
-    def move_video(self):
-        if os.path.exists(self.temp_file_path):
-            shutil.move(self.temp_file_path, self.video_directory)
-            print(f"moving {self.file_name} from {self.temp_directory} to {self.video_directory}")
-            self.remove_temps(self.file_name)
-            return True
-        if os.path.exists(self.complete_file_path):
-            print(f"{self.file_name} already existed in {self.video_directory}; removing it from {self.temp_directory}")
-            self.remove_temps(self.file_name)
-            return True
-        return False
-    def yt_dlp_downloader(self,url,ydl_opts={},download=True):
-        try:
-            with yt_dlp.YoutubeDL(ydl_opts) as ydl:
-                self.info_dict=ydl.extract_info(url=url, download=download)
-            return True
-        except:
-            return False
-    def progress_callback(self, d):
-        self.status_dict = d
-        keys = ['status',
-                'downloaded_bytes',
-                'fragment_index',
-                'fragment_count',
-                'filename',
-                'tmpfilename',
-                'max_progress',
-                'progress_idx',
-                'elapsed',
-                'total_bytes_estimate',
-                'speed',
-                'eta',
-                '_eta_str',
-                '_speed_str',
-                '_percent_str',
-                '_total_bytes_str',
-                '_total_bytes_estimate_str',
-                '_downloaded_bytes_str',
-                '_elapsed_str',
-                '_default_template']
-        if self.status_dict['status'] == 'finished':
-            print("Done downloading, moving video to final directory...")
-            self.move_video()
-            return
-        if get_time_stamp()-self.last_checked>5:
-            print(self.status_dict['_default_template'])
-            self.last_checked = get_time_stamp()
-            if (get_time_stamp()-self.start_time/5)>6:
-                self.speed_track.append(self.status_dict['speed'])
-                self.speed_track=self.filter_outliers(self.speed_track)
-                
-    def download(self):
-        if not os.path.exists(self.video_directory):
-            os.makedirs(self.video_directory,exist_ok=True)
-        if not os.path.exists(self.temp_directory):
-            os.makedirs(self.temp_directory,exist_ok=True)
-        for self.num,video_url in enumerate(self.video_urls):
-            if video_url != self.video_url or self.video_url == None:
-                self.video_url=video_url
-                self.info_dict=None
-                result = self.yt_dlp_downloader(url=self.video_url,ydl_opts={'quiet': True, 'no_warnings': True},download=False)
-                if self.info_dict != None and result:
-                    self.start_time = get_time_stamp()
-                    self.downloaded = 0
-                    self.video_title = self.info_dict.get('title', None)
-                    self.video_ext = self.info_dict.get('ext', 'mp4')
-                    self.file_name =f"{self.video_title}.{self.video_ext}"
-                    self.temp_file_path = os.path.join(self.temp_directory, self.file_name)
-                    self.complete_file_path = os.path.join(self.video_directory, self.file_name)
-                    if not self.move_video():
-                        self.dl_speed = []
-                        self.percent=None
-                        self.dl_eta=None
-                        self.total_bytes_est=None
-                        self.percent_speed=None
-                        self.speed_track = []
-                        self.outlier_count=0
-                        ydl_opts = {
-                            'outtmpl': self.temp_file_path,
-                            'noprogress':True,
-                            'progress_hooks': [self.progress_callback]  
-                        }
-                        
-                       
-                        print("Starting download...")  # Check if this point in code is reached
-                        result = self.yt_dlp_downloader(url=self.video_url,ydl_opts=ydl_opts,download=True)
-                        if result:
-                            print("Download finished!")  # Check if download completes
-                        else:
-                            print(f'error downloding {self.video_url}')
-                        self.move_video()
-                    else:
-                        print(f"The video from {self.video_url} already exists in the directory {self.video_directory}. Skipping download.")
-                else:
-                    print(f"could not find video info from {self.video_url} Skipping download.")
-        if self.num==len(self.video_urls)-1:
-            self.monitoring=False
-            self.time_interval=0
-            
-    def monitor(self):
-        while self.monitoring:
-            self.thread_manager.wait(name='pause_event',n=self.time_interval)# check every minute
-            if self.monitoring:
-                if 'eta' in self.status_dict:
-                    if self.outlier_count>=3 and (self.status_dict['eta']/60)>10:
-                        self.start()
-
-    def start(self):
-        download_thread = self.thread_manager.add_thread(name='download_thread',target=self.download)
-        monitor_thread = self.thread_manager.add_thread(name='monitor_thread',target_function=self.monitor)
-        self.thread_manager.start(name='download_thread')
-        self.thread_manager.start(name='monitor_thread')
-        self.thread_manager.join(name='download_thread')
-        self.thread_manager.join(name='monitor_thread')
-class VideoDownloaderSingleton():
-    _instance = None
-    @staticmethod
-    def get_instance(url_manager,request_manager,title=None,video_extention='mp4',download_directory=os.getcwd(),user_agent=None,download=True,get_info=False):
-        if VideoDownloaderSingleton._instance is None:
-            VideoDownloaderSingleton._instance = VideoDownloader(url=url,title=title,video_extention=video_extention,download_directory=download_directory,download=download,get_info=get_info,user_agent=user_agent)
-        elif VideoDownloaderSingleton._instance.title != title or video_extention != VideoDownloaderSingleton._instance.video_extention or url != VideoDownloaderSingleton._instance.url or download_directory != VideoDownloaderSingleton._instance.download_directory or user_agent != VideoDownloaderSingleton._instance.user_agent:
-            VideoDownloaderSingleton._instance = VideoDownloader(url=url,title=title,video_extention=video_extention,download_directory=download_directory,download=download,get_info=get_info,user_agent=user_agent)
-        return VideoDownloaderSingleton._instance
-
-class LinkManager:
-    """
-    LinkManager is a class for managing and extracting links and image links from a web page.
-
-    Args:
-        url (str): The URL of the web page (default is "https://example.com").
-        source_code (str or None): The source code of the web page (default is None).
-        url_manager (UrlManager or None): An instance of UrlManager (default is None).
-        request_manager (SafeRequest or None): An instance of SafeRequest (default is None).
-        soup_manager (SoupManager or None): An instance of SoupManager (default is None).
-        image_link_tags (str): HTML tags to identify image links (default is 'img').
-        img_link_attrs (str): HTML attributes to identify image link URLs (default is 'src').
-        link_tags (str): HTML tags to identify links (default is 'a').
-        link_attrs (str): HTML attributes to identify link URLs (default is 'href').
-        strict_order_tags (bool): Flag to indicate if tags and attributes should be matched strictly (default is False).
-        img_attr_value_desired (list or None): Desired attribute values for image links (default is None).
-        img_attr_value_undesired (list or None): Undesired attribute values for image links (default is None).
-        link_attr_value_desired (list or None): Desired attribute values for links (default is None).
-        link_attr_value_undesired (list or None): Undesired attribute values for links (default is None).
-        associated_data_attr (list): HTML attributes to associate with the extracted links (default is ["data-title", 'alt', 'title']).
-        get_img (list): HTML attributes used to identify associated images (default is ["data-title", 'alt', 'title']).
-
-    Methods:
-        re_initialize(): Reinitialize the LinkManager with the current settings.
-        update_url_manager(url_manager): Update the URL manager with a new instance.
-        update_url(url): Update the URL and reinitialize the LinkManager.
-        update_source_code(source_code): Update the source code and reinitialize the LinkManager.
-        update_soup_manager(soup_manager): Update the SoupManager and reinitialize the LinkManager.
-        update_desired(...): Update the desired settings and reinitialize the LinkManager.
-        find_all_desired(...): Find all desired links or image links based on the specified criteria.
-        find_all_domain(): Find all unique domain names in the extracted links.
-
-    Note:
-        - The LinkManager class helps manage and extract links and image links from web pages.
-        - The class provides flexibility in specifying criteria for link extraction.
-    """
-    def __init__(self,url="https://example.com",source_code=None,url_manager=None,request_manager=None,soup_manager=None,image_link_tags='img',img_link_attrs='src',link_tags='a',link_attrs='href',strict_order_tags=False,img_attr_value_desired=None,img_attr_value_undesired=None,link_attr_value_desired=None,link_attr_value_undesired=None,associated_data_attr=["data-title",'alt','title'],get_img=["data-title",'alt','title']):
-        if url_manager==None:
-            url_manager=UrlManager(url=url)
-        self.url_manager= url_manager
-        self.url=self.url_manager.url
-        if request_manager==None:
-            request_manager = SafeRequest(url_manager=self.url_manager)
-        self.request_manager=request_manager
-        if soup_manager == None:
-            soup_manager = SoupManager(url_manager=self.url_manager,request_manager=self.request_manager)
-        self.soup_manager = soup_manager
-        if source_code != None:
-            self.source_code=source_code
-        else:
-            self.source_code=self.request_manager.source_code_bytes
-        if self.source_code != self.soup_manager.source_code:
-            self.soup_manager.update_source_code(source_code=self.source_code)
-        self.strict_order_tags=strict_order_tags
-        self.image_link_tags=image_link_tags
-        self.img_link_attrs=img_link_attrs
-        self.link_tags=link_tags
-        self.link_attrs=link_attrs
-        self.img_attr_value_desired=img_attr_value_desired
-        self.img_attr_value_undesired=img_attr_value_undesired
-        self.link_attr_value_desired=link_attr_value_desired
-        self.link_attr_value_undesired=link_attr_value_undesired
-        self.associated_data_attr=associated_data_attr
-        self.get_img=get_img
-        self.all_desired_image_links=self.find_all_desired_links(tag=self.image_link_tags,
-                                                                 attr=self.img_link_attrs,
-                                                                 attr_value_desired=self.img_attr_value_desired,
-                                                                 attr_value_undesired=self.img_attr_value_undesired)
-        self.all_desired_links=self.find_all_desired_links(tag=self.link_tags,
-                                                           attr=self.link_attrs,
-                                                           attr_value_desired=self.link_attr_value_desired,
-                                                           attr_value_undesired=self.link_attr_value_undesired,
-                                                           associated_data_attr=self.associated_data_attr,
-                                                           get_img=get_img)
-    def re_initialize(self):
-        self.all_desired_image_links=self.find_all_desired_links(tag=self.image_link_tags,attr=self.img_link_attrs,strict_order_tags=self.strict_order_tags,attr_value_desired=self.img_attr_value_desired,attr_value_undesired=self.img_attr_value_undesired)
-        self.all_desired_links=self.find_all_desired_links(tag=self.link_tags,attr=self.link_attrs,strict_order_tags=self.strict_order_tags,attr_value_desired=self.link_attr_value_desired,attr_value_undesired=self.link_attr_value_undesired,associated_data_attr=self.associated_data_attr,get_img=self.get_img)
-    def update_url_manager(self,url_manager):
-        self.url_manager=url_manager
-        self.url=self.url_manager.url
-        self.request_manager.update_url_manager(url_manager=self.url_manager)
-        self.soup_manager.update_url_manager(url_manager=self.url_manager)
-        self.source_code=self.soup_manager.source_code
-        self.re_initialize()
-    def update_url(self,url):
-        self.url=url
-        self.url_manager.update_url(url=self.url)
-        self.url=self.url_manager.url
-        self.request_manager.update_url(url=self.url)
-        self.soup_manager.update_url(url=self.url)
-        self.source_code=self.soup_manager.source_code
-        self.re_initialize()
-    def update_source_code(self,source_code):
-        self.source_code=source_code
-        if self.source_code != self.soup_manager.source_code:
-            self.soup_manager.update_source_code(source_code=self.source_code)
-        self.re_initialize()
-    def update_soup_manager(self,soup_manager):
-        self.soup_manager=soup_manager
-        self.source_code=self.soup_manager.source_code
-        self.re_initialize()
-    def update_desired(self,img_attr_value_desired=None,img_attr_value_undesired=None,link_attr_value_desired=None,link_attr_value_undesired=None,image_link_tags=None,img_link_attrs=None,link_tags=None,link_attrs=None,strict_order_tags=None,associated_data_attr=None,get_img=None):
-       self.strict_order_tags = strict_order_tags or self.strict_order_tags
-       self.img_attr_value_desired=img_attr_value_desired or self.img_attr_value_desired
-       self.img_attr_value_undesired=img_attr_value_undesired or self.img_attr_value_undesired
-       self.link_attr_value_desired=link_attr_value_desired or self.link_attr_value_desired
-       self.link_attr_value_undesired=link_attr_value_undesired or self.link_attr_value_undesired
-       self.image_link_tags=image_link_tags or self.image_link_tags
-       self.img_link_attrs=img_link_attrs or self.img_link_attrs
-       self.link_tags=link_tags or self.link_tags
-       self.link_attrs=link_attrs or self.link_attrs
-       self.associated_data_attr=associated_data_attr or self.associated_data_attr
-       self.get_img=get_img or self.get_img
-       self.re_initialize()
-    def find_all_desired(self,tag='img',attr='src',strict_order_tags=False,attr_value_desired=None,attr_value_undesired=None,associated_data_attr=None,get_img=None):
-        def make_list(obj):
-            if isinstance(obj,list) or obj==None:
-                return obj
-            return [obj]
-        def get_desired_value(attr,attr_value_desired=None,attr_value_undesired=None):
-            if attr_value_desired:
-                for value in attr_value_desired:
-                    if value not in attr:
-                        return False
-            if attr_value_undesired:
-                for value in attr_value_undesired:
-                    if value in attr:
-                        return False
-            return True
-        attr_value_desired,attr_value_undesired,associated_data_attr,tags,attribs=make_list(attr_value_desired),make_list(attr_value_undesired),make_list(associated_data_attr),make_list(tag),make_list(attr)
-        desired_ls = []
-        assiciated_data=[]
-        for i,tag in enumerate(tags):
-            attribs_list=attribs
-            if strict_order_tags:
-                if len(attribs)<=i:
-                    attribs_list=[None]
-                else:
-                    attribs_list=make_list(attribs[i])
-            for attr in attribs_list:
-                for component in self.soup_manager.soup.find_all(tag):
-                    if attr in component.attrs and get_desired_value(attr=component[attr],attr_value_desired=attr_value_desired,attr_value_undesired=attr_value_undesired):
-                        if component[attr] not in desired_ls:
-                            desired_ls.append(component[attr])
-                            assiciated_data.append({"value":component[attr]})
-                            if associated_data_attr:
-                                for data in associated_data_attr:
-                                    if data in component.attrs:
-                                        assiciated_data[-1][data]=component.attrs[data]
-                                        if get_img and component.attrs[data]:
-                                            if data in get_img and len(component.attrs[data])!=0:
-                                                for each in self.soup_manager.soup.find_all('img'):
-                                                    if 'alt' in each.attrs:
-                                                        if each.attrs['alt'] == component.attrs[data] and 'src' in each.attrs:
-                                                            assiciated_data[-1]['image']=each.attrs['src']
-        desired_ls.append(assiciated_data)
-        return desired_ls
-    def find_all_domain(self):
-        domains_ls=[self.url_manager.protocol+'://'+self.url_manager.domain]
-        for desired in all_desired[:-1]:
-            if url_manager.is_valid_url(desired):
-                parse = urlparse(desired)
-                domain = parse.scheme+'://'+parse.netloc
-                if domain not in domains_ls:
-                    domains_ls.append(domain)
-    def find_all_desired_links(self,tag='img', attr='src',attr_value_desired=None,strict_order_tags=False,attr_value_undesired=None,associated_data_attr=None,all_desired=None,get_img=None):
-        all_desired = all_desired or self.find_all_desired(tag=tag,attr=attr,strict_order_tags=strict_order_tags,attr_value_desired=attr_value_desired,attr_value_undesired=attr_value_undesired,associated_data_attr=associated_data_attr,get_img=get_img)
-        assiciated_attrs = all_desired[-1]
-        valid_assiciated_attrs = []
-        desired_links=[]
-        for i,attr in enumerate(all_desired[:-1]):
-            valid_attr=self.url_manager.make_valid(attr,self.url_manager.protocol+'://'+self.url_manager.domain) 
-            if valid_attr:
-                desired_links.append(valid_attr)
-                valid_assiciated_attrs.append(assiciated_attrs[i])
-                valid_assiciated_attrs[-1]["link"]=valid_attr
-        desired_links.append(valid_assiciated_attrs)
-        return desired_links
-
-def CrawlManager():
-    def __init__(self,url=None,source_code=None,parse_type="html.parser"):
-        self.url=url
-        self.source_code=source_code
-        self.parse_type=parse_type
-        get_new_source_and_url(self,url)
-    def get_new_source_and_url(self,url=None):
-        if url == None:
-            url = self.url
-        self.response = self.response_manager.response
-        self.source_code=self.response_manager.source_code
-    def get_classes_and_meta_info():
-        class_name_1,class_name_2, class_value = 'meta','class','property','og:image'
-        attrs = 'href','src'
-        unique_classes, images=discover_classes_and_images(self,tag_name,class_name_1,class_name_2,class_value,attrs)
-        return unique_classes, images
-    def extract_links_from_url(self):
-        """
-        Extracts all href and src links from a given URL's source code.
-
-        Args:
-            base_url (str): The URL from which to extract links.
-
-        Returns:
-            dict: Dictionary containing image links and external links under the parent page.
-        """
-        agg_js = {'images':[],'external_links':[]}
-        
-        if self.response != None:
-            attrs = 'href','src'
-            href_links,src_links='',''
-            links = [href_links,src_links]
-            for i,each in enumerate(attrs):
-                 links[i]= [a[attr[i]] for a in get_find_all_with_attributes(self, attrs[i])]
-            # Convert all links to absolute links
-            absolute_links = [(url, link) for link in links[0] + links[1]]
-            # Separate images and external links
-            images = [link for link in absolute_links if link.lower().endswith(('.png', '.jpg', '.jpeg', '.gif', '.svg', '.webp'))]
-            external_links = [link for link in absolute_links if urlparse(link).netloc != urlparse(url).netloc]
-            agg_js['images']=images
-            agg_js['external_links']=external_links
-           
-        return agg_js
-
-
-    def correct_xml(xml_string):
-        # Parse the XML string
-        root = ET.fromstring(xml_string)
-
-        # Loop through each <image:loc> element and correct its text if needed
-        for image_loc in root.findall(".//image:loc", namespaces={'image': 'http://www.google.com/schemas/sitemap-image/1.1'}):
-            # Replace '&' with '&amp;' in the element's text
-            if '&' in image_loc.text:
-                image_loc.text = image_loc.text.replace('&', '&amp;')
-
-        # Convert the corrected XML back to string
-        corrected_xml = ET.tostring(root, encoding='utf-8').decode('utf-8')
-        return corrected_xml
-
-
-    def determine_values(self):
-        # This is just a mockup. In a real application, you'd analyze the URL or its content.
-
-        # Assuming a blog site
-        if 'blog' in self.url:
-            if '2023' in self.url:  # Assuming it's a current year article
-                return ('weekly', '0.8')
-            else:
-                return ('monthly', '0.6')
-        elif 'contact' in self.url:
-            return ('yearly', '0.3')
-        else:  # Homepage or main categories
-            return ('weekly', '1.0')
-    def crawl(url, max_depth=3, depth=1):
-        
-        if depth > max_depth:
-            return []
-
-        if url in visited:
-            return []
-
-        visited.add(url)
-
-        try:
-            
-            links = [a['href'] for a in self.soup.find_all('a', href=True)]
-            valid_links = []
-
-            for link in links:
-                parsed_link = urlparse(link)
-                base_url = "{}://{}".format(parsed_link.scheme, parsed_link.netloc)
-            
-                if base_url == url:  # Avoiding external URLs
-                    final_link = urljoin(url, parsed_link.path)
-                    if final_link not in valid_links:
-                        valid_links.append(final_link)
-
-            for link in valid_links:
-                crawl(link, max_depth, depth+1)
-
-            return valid_links
-
-        except Exception as e:
-            print(f"Error crawling {url}: {e}")
-            return []
-
-
-    # Define or import required functions here, like get_all_website_links, determine_values, 
-    # discover_classes_and_meta_images, and extract_links_from_url.
-    def get_meta_info(self):
-        
-        meta_info = {}
-        # Fetch the title if available
-        title_tag = parse_title()
-        if title_tag:
-            meta_info["title"] = title_tag
-        # Fetch meta tags
-        for meta_tag in soup.find_all('meta'):
-            name = meta_tag.get('name') or meta_tag.get('property')
-            if name:
-                content = meta_tag.get('content')
-                if content:
-                    meta_info[name] = content
-
-        return meta_info
-    def generate_sitemap(self,domain):
-        
-        with open('sitemap.xml', 'w', encoding='utf-8') as f:
-            string = '<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:image="http://www.google.com/schemas/sitemap-image/1.1">\n'
-            
-            for url in self.all_site_links:
-                string += f'  <url>\n    <loc>{url}</loc>\n'
-                preprocess=[]
-                self.get_new_source_and_url(url=url)
-                links = extract_links_from_url(url)
-                
-                for img in links['images']:
-                    if str(img).lower() not in preprocess:
-                        try:
-                            escaped_img = img.replace('&', '&amp;')
-
-                            str_write = f'    <image:image>\n      <image:loc>{escaped_img}</image:loc>\n    </image:image>\n'
-                            string += str_write
-                        except:
-                            pass
-                        preprocess.append(str(img).lower())
-                frequency, priority = determine_values(url)
-                string += f'    <changefreq>{frequency}</changefreq>\n'
-                string += f'    <priority>{priority}</priority>\n'
-                string += f'  </url>\n'
-                
-            string += '</urlset>\n'
-            f.write(string)            
-        # Output summary
-        print(f'Sitemap saved to sitemap.xml with {len(urls)} URLs.')
-        
-        # Output class and link details
-        for url in urls:
-            print(f"\nDetails for {url}:")
-            classes, meta_img_refs = discover_classes_and_meta_images(url)
-
-            print("\nClasses with href or src attributes:")
-            for class_name in classes:
-                print(f"\t{class_name}")
-            
-            print("\nMeta Image References:")
-            for img_ref in meta_img_refs:
-                print(f"\t{img_ref}")
-            
-            links = extract_links_from_url(url)
-
-            print("\nImages:")
-            for img in links['images']:
-                print(f"\t{img}")
-            
-            print("\nExternal Links:")
-            for ext_link in links['external_links']:
-                print(f"\t{ext_link}")
-class CrawlManagerSingleton():
-    _instance = None
-    @staticmethod
-    def get_instance(url=None,source_code=None,parse_type="html.parser"):
-        if CrawlManagerSingleton._instance is None:
-            CrawlManagerSingleton._instance = CrawlManager(url=url,parse_type=parse_type,source_code=source_code)
-        elif parse_type != CrawlManagerSingleton._instance.parse_type or url != CrawlManagerSingleton._instance.url  or source_code != CrawlManagerSingleton._instance.source_code:
-            CrawlManagerSingleton._instance = CrawlManager(url=url,parse_type=parse_type,source_code=source_code)
-        return CrawlManagerSingleton._instance
-import time
-import requests
-
-class DynamicRateLimiterManager:
-    def __init__(self, service_name='ethereum'):
-        self.services = {}
-        self.service_name = service_name
-        self.add_service(service_name)
-
-    def add_service(self, service_name="default", low_limit=10, high_limit=30, limit_epoch=60, starting_tokens=10, epoch_cycle_adjustment=True):
-        if service_name in self.services:
-            print(f"Service {service_name} already exists!")
-            return
-        self.services[service_name] = DynamicRateLimiter(low_limit=low_limit, high_limit=high_limit, limit_epoch=limit_epoch, starting_tokens=starting_tokens, epoch_cycle_adjustment=epoch_cycle_adjustment)
-
-    def request(self, request_url, service_name=None):
-        service_name = service_name or self.service_name
-        if service_name not in self.services:
-            self.add_service(service_name)
-
-        limiter = self.services[service_name]
-
-        while True:
-            if limiter.request():
-                response = requests.get(request_url)  # Actual request
-                if response.status_code == 200:
-                    limiter.request_tracker(True)
-                    return response.json()
-                elif response.status_code == 429:
-                    limiter.request_tracker(False)
-                    print(f"Rate limited by {service_name}. Adjusting limit and retrying...")
-                    time.sleep(limiter.get_sleep()["current_sleep"])
-                else:
-                    print(f"Unexpected response: {response.status_code}. Message: {response.text}")
-                    return None
-            else:
-                print(f"Rate limit reached for {service_name}. Waiting for the next epoch...")
-                time.sleep(limiter.get_sleep()["current_sleep"])
-
-    def log_request(self, service_name, success):
-        print(f"[{service_name}] Request {'succeeded' if success else 'denied'}. Current tokens: {self.services[service_name].get_current_tokens()}")
-
-class DynamicRateLimiter:
-    def __init__(self, low_limit, high_limit, limit_epoch, starting_tokens=None,epoch_cycle_adjustment:int=None):
-        self.low_limit = low_limit
-        self.high_limit = high_limit
-        self.limit_epoch = limit_epoch  # in seconds
-        self.request_status_json = {"succesful":[],"unsuccesful":[],"last_requested":get_time_stamp(),"first_requested":get_time_stamp(),"epoch_left":self.limit_epoch,"last_fail":get_time_stamp(),"count_since_fail":0}
-        self.current_limit = starting_tokens or low_limit  # Default to high_limit if starting_tokens isn't provided
-        self.epoch_cycle_adjustment = epoch_cycle_adjustment
-        # Additional attributes for tracking adjustment logic
-        self.last_adjusted_time = get_time_stamp()
-        self.successful_epochs_since_last_adjustment = 0
-        self.request_count_in_current_epoch = 0
-
-    def _refill_tokens(self):
-        time_since_last_request = get_time_stamp() - self.request_status_json["last_requested"]
-        new_tokens = (time_since_last_request / self.limit_epoch) * self.current_limit
-        self.tokens = min(self.current_limit, self.get_current_tokens())
-    def request_tracker(self,success):
-        if success:
-            self.request_status_json["succesful"].append(get_time_stamp())
-        else:
-            self.request_status_json["unsuccesful"].append(get_time_stamp())
-            self.request_status_json["last_fail"]=get_time_stamp()
-            self.request_status_json["count_since_fail"]=0
-            self.adjust_limit()
-        self.request_status_json["last_requested"]=get_time_stamp()
-    def calculate_tokens(self):
-        successful = []
-        for each in self.request_status_json["succesful"]:
-            if (get_time_stamp() - each)<self.limit_epoch:
-                successful.append(each)
-        self.request_status_json["succesful"]=successful
-        unsuccessful = []
-        for each in self.request_status_json["unsuccesful"]:
-            if (get_time_stamp() - each)<self.limit_epoch:
-                unsuccessful.append(each)
-        self.request_status_json["unsuccesful"]=unsuccessful
-        if len(successful)==0 and len(unsuccessful)==0:
-            pass
-        elif len(successful)!=0 and len(unsuccessful)==0:
-            self.request_status_json["first_requested"] = successful[0]
-        elif len(successful)==0 and len(unsuccessful)!=0:
-            self.request_status_json["first_requested"] = unsuccessful[0]
-        else:
-            self.request_status_json["first_requested"] = min(unsuccessful[0],successful[0])
-        self.request_status_json["epoch_left"]=self.limit_epoch-(self.request_status_json["last_requested"]-self.request_status_json["first_requested"])
-        
-        return self.request_status_json
-    def get_current_tokens(self):
-        self.request_status_json = self.calculate_tokens()
-        total_requests = len(self.request_status_json["succesful"])+len(self.request_status_json["unsuccesful"])
-        return max(0,self.current_limit-total_requests)
-    def get_sleep(self):
-        self.request_status_json = self.calculate_tokens()
-        self.request_status_json["current_sleep"]=self.request_status_json["epoch_left"]/max(1,self.get_current_tokens())
-        return self.request_status_json
-    def request(self):
-        self._refill_tokens()
-        if self.tokens > 0:
-            return True  # The request can be made
-        else:
-            if self.tokens == 0:
-                self.request_status_json["count_since_fail"]+=1
-                if self.epoch_cycle_adjustment != None:
-                    if self.request_status_json["count_since_fail"] >=self.epoch_cycle_adjustment:
-                        self.current_limit=min(self.current_limit+1,self.high_limit)
-            return False  # The request cannot be made
-    def _adjust_limit(self):
-        current_time = get_time_stamp()
-        if current_time - self.last_adjusted_time >= self.limit_epoch:
-            if len(self.clear_epoch()["succesful"]) >= self.tokens:
-                # We hit the rate limit this epoch, decrease our limit
-                self.tokens = max(1, self.tokens - 1)
-            else:
-                self.successful_epochs_since_last_adjustment += 1
-                if self.successful_epochs_since_last_adjustment >= 5:
-                    # We've had 5 successful epochs, increase our limit
-                    self.current_limit = min(self.high_limit, self.tokens + 1)
-                    self.successful_epochs_since_last_adjustment = 0
-            
-            # Reset our counters for the new epoch
-            self.last_adjusted_time = current_time
-            self.request_count_in_current_epoch = 0
-    def adjust_limit(self):
-        # Set the tokens to succesful requests_made - 1
-        self.tokens = len(self.calculate_tokens()["succesful"])
-
-        # Adjust the high_limit
-        self.current_limit = self.tokens
-
-        # Log the adjustment
-        print(f"Adjusted tokens to: {self.tokens} and high_limit to: {self.current_limit}")
-class DynamicRateLimiterManagerSingleton:
-    _instance = None
-    @staticmethod
-    def get_instance(service_name="default", low_limit=10, high_limit=30, limit_epoch=60,starting_tokens=10,epoch_cycle_adjustment=True):
-        if DynamicRateLimiterManagerSingleton._instance is None:
-            DynamicRateLimiterManagerSingleton._instance = DynamicRateLimiterManager(service_name=service_name, low_limit=low_limit, high_limit=limit_epoch, limit_epoch=60,starting_tokens=starting_tokens,epoch_cycle_adjustment=epoch_cycle_adjustment)
-        return DynamicRateLimiterManagerSingleton._instance
-
-
-
-