abstract-webtools 0.1.6.38__tar.gz → 0.1.6.40__tar.gz

{abstract_webtools-0.1.6.38/src/abstract_webtools.egg-info → abstract_webtools-0.1.6.40}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: abstract_webtools
-Version: 0.1.6.38
+Version: 0.1.6.40
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff

{abstract_webtools-0.1.6.38 → abstract_webtools-0.1.6.40}/setup.py

@@ -4,7 +4,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 setuptools.setup(
     name='abstract_webtools',
-    version='0.1.6.38',
+    version='0.1.6.40',
     author='putkoff',
     author_email='partners@abstractendeavors.com',
     description='Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.',

abstract_webtools-0.1.6.40/src/abstract_webtools/managers/seleniumManager.py

@@ -0,0 +1,128 @@
+import os
+from urllib.parse import urlparse
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.chrome.service import Service
+from webdriver_manager.chrome import ChromeDriverManager  # For automatic ChromeDriver installation
+import logging
+import urllib3
+from ..abstract_webtools import *  # Assuming this is a valid import
+from .urlManager import *
+
+# Suppress urllib3 warnings and debug logs
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+logging.getLogger("urllib3").setLevel(logging.WARNING)
+logging.getLogger("selenium").setLevel(logging.WARNING)
+
+# Default Chrome options (can be overridden)
+DEFAULT_CHROME_OPTIONS = [
+    "--headless",  # Run in headless mode
+    "--no-sandbox",
+    "--disable-dev-shm-usage",  # Avoid memory issues on servers
+    "--disable-gpu",
+    "--disable-software-rasterizer",
+    "--disable-extensions",
+    "--remote-debugging-port=9222"
+]
+
+class SingletonMeta(type):
+    _instances = {}
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            instance = super().__call__(*args, **kwargs)
+            cls._instances[cls] = instance
+        return cls._instances[cls]
+
+class SeleniumManager(metaclass=SingletonMeta):
+    def __init__(self, url):
+        if not hasattr(self, 'initialized'):  # Prevent reinitialization
+            self.initialized = True
+            parsed_url = urlparse(url)
+            self.domain = parsed_url.netloc
+            self.scheme = parsed_url.scheme or "https"  # Default to https if scheme is missing
+            self.base_url = f"{self.scheme}://{self.domain}"
+            self.site_dir = os.path.join(os.getcwd(), self.domain)
+            os.makedirs(self.site_dir, exist_ok=True)
+            self.drivers = {}
+            self.page_type = []
+
+    def get_url_to_path(self, url):
+        url = eatAll(str(url), ['', ' ', '\n', '\t', '\\', '/'])  # Assuming eatAll is defined elsewhere
+        parsed_url = urlparse(url)
+        if parsed_url.netloc == self.domain:
+            paths = parsed_url.path.split('/')
+            dir_path = self.site_dir
+            for path in paths[:-1]:
+                dir_path = os.path.join(dir_path, path)
+                os.makedirs(dir_path, exist_ok=True)
+            self.page_type.append(os.path.splitext(paths[-1])[-1] or 'html' if not self.page_type else self.page_type[-1])
+            dir_path = os.path.join(dir_path, paths[-1])
+            return dir_path
+
+    def saved_url_check(self, url):
+        return self.get_url_to_path(url)
+
+    def get_with_netloc(self, url):
+        parsed_url = urlparse(url)
+        if not parsed_url.netloc:
+            url = f"{self.scheme}://{self.domain}/{url.strip('/')}"
+        return url
+
+    def get_driver(self, url):
+        if url and url not in self.drivers:
+            # Set up Chrome options
+            chrome_options = Options()
+            for option in DEFAULT_CHROME_OPTIONS:
+                chrome_options.add_argument(option)
+            
+            # Specify Chrome binary location if needed (optional, comment out if not applicable)
+            # chrome_options.binary_location = "/home/profiles/solcatcher/.cache/selenium/chrome/linux64/130.0.6723.58/chrome"
+
+            # Automatically install and use ChromeDriver
+            service = Service(ChromeDriverManager().install())
+            driver = webdriver.Chrome(service=service, options=chrome_options)
+            
+            self.drivers[url] = driver
+            driver.get(url)
+        return self.drivers[url]
+
+    def quit_driver(self, url):
+        """Clean up a specific driver instance."""
+        if url in self.drivers:
+            self.drivers[url].quit()
+            del self.drivers[url]
+
+    def quit_all_drivers(self):
+        """Clean up all driver instances."""
+        for driver in self.drivers.values():
+            driver.quit()
+        self.drivers.clear()
+
+def normalize_url(url, base_url=None):
+    """Normalize and resolve relative URLs."""
+    manager = SeleniumManager(url)
+    base_url = manager.base_url if base_url is None else base_url
+    if url.startswith(base_url):
+        url = url[len(base_url):]
+    normalized_url = urljoin(base_url, url.split('#')[0])
+    if not normalized_url.startswith(base_url):
+        return None
+    return normalized_url
+
+def get_selenium_source(url):
+    """Fetch page source using Selenium."""
+    url_mgr = urlManager(url)  # Assuming urlManager is defined elsewhere
+    if url_mgr.url:
+        url = str(url_mgr.url)
+        manager = SeleniumManager(url)
+        driver = manager.get_driver(url)
+        try:
+            return driver.page_source
+        except Exception as e:
+            logging.error(f"Error fetching page source for {url}: {e}")
+            return None
+        # Note: Driver is not quit here to maintain Singleton behavior
+
+# Ensure cleanup on program exit (optional)
+import atexit
+atexit.register(lambda: SeleniumManager(url="").quit_all_drivers())  # Cleanup all drivers on exit

abstract_webtools-0.1.6.40/src/abstract_webtools/managers/soupManager/asoueces.py

@@ -0,0 +1,135 @@
+text = """{"title": "NoviSoul
+        novissbm@gmail.com", "href": "http://www.youtube.com/signin?authuser=0&next=%2Fwatch%3Fv%3DEaIYRM1yrM4&action_handle_signin=true", "description": ""},
+  {"title": "Sign in", "href": "https://accounts.google.com/ServiceLogin?continue=http%3A%2F%2Fwww.youtube.com%2Fsignin%3Faction_handle_signin%3Dtrue%26hl%3Den_GB%26next%3D%252Fwatch%253Fv%253DEaIYRM1yrM4%26nomobiletemp%3D1&uilel=3&service=youtube&passive=true&hl=en_GB", "description": ""},
+  {"title": "Sign up", "href": "http://www.youtube.com/signup?next=%2Fwatch%3Fv%3DEaIYRM1yrM4", "description": ""},
+  {"title": "9:58
+
+
+    
+ 
+
+Physics of Free Energy Deviceby Eugene Jeong        
+        
+336,881 views", "href": "http://www.youtube.com/watch?v=EB-jWfzkz_E", "description": ""},
+  {"title": "4:49
+
+
+    
+ 
+
+[www.witts.ws] Self-Running 40kW (40,000 Watt) Fuelless Generator (1 of 3)by wits2014        
+        
+488,638 views", "href": "http://www.youtube.com/watch?v=LFu-s6ZmGyE", "description": ""},
+  {"title": "2:33
+
+
+    
+ 
+
+Free Energy - Evidence of military antigravity technologyby DoubleMarkez        
+        
+390,020 views", "href": "http://www.youtube.com/watch?v=qljY-YfFaPc", "description": ""},
+  {"title": "15:01
+
+
+    
+ 
+
+APEX 2013   SSBM L10   Shroomed VS CT EMP Mew2Kingby Jason AxelrodRecommended for you", "href": "http://www.youtube.com/watch?v=pc7v49k5FhY", "description": ""},
+  {"title": "161
+            
+            
+              videos
+            
+          
+        
+      
+          
+              
+          
+          
+              
+          
+      
+    
+      
+        
+          
+Play all
+        
+      
+  
+washby dle3276", "href": "http://www.youtube.com/watch?v=AmcSt5hU4qA&list=PL4517CA6C6244A844", "description": ""},
+  {"title": "10:31
+
+
+    
+ 
+
+Pyramid Magnet - free energy - english subtitleby MrTermsof        
+        
+616,081 views", "href": "http://www.youtube.com/watch?v=pMbHswNoGWM", "description": ""},
+  {"title": "4:11
+
+
+    
+ 
+
+My all new newman motor 1.(TheDaftman)by theDaftman        
+        
+1,147,470 views", "href": "http://www.youtube.com/watch?v=dL4B_DNBtvc", "description": ""},
+  {"title": "2:18
+
+
+    
+ 
+
+Is there free energy in magnets?by aetherix01        
+        
+371,642 views", "href": "http://www.youtube.com/watch?v=vrn5B9a8aOk", "description": ""},
+  {"title": "3:00
+
+
+    
+ 
+
+The Most Dangerous Video On The Internet  - Trevor Paglenby killuminati63        
+        
+585,755 views", "href": "http://www.youtube.com/watch?v=9xEuhEHDJM8", "description": ""},
+  {"title": "2:18
+
+
+    
+ 
+
+Free Energy - Magnet Motorby ATBootstrap        
+        
+358,641 views", "href": "http://www.youtube.com/watch?v=hfkwCE3BeBs", "description": ""},
+  {"title": "2:38
+
+
+    
+ 
+
+100% free energy generator is easy to buildby LifeHack2012        
+        
+238,092 views", "href": "http://www.youtube.com/watch?v=GEUyhhMEs7U", "description": ""},
+  {"title": "3:41
+
+
+    
+ 
+
+5KW free energy –±–µ—Å—Ç–æ–ø–ª–∏–≤–Ω—ã–π –≥–µ–Ω–µ—Ä–∞—Ç–æ—Ä Kapanadze –ö–∞–ø–∞–Ω–∞–¥–∑–µby Alexander Frolov        
+        
+488,213 views", "href": "http://www.youtube.com/watch?v=uxQ99R4gOWY", "description": ""},""".split('\n')
+sources = ' '.join([te for te in text if te])
+while True:
+    if '  ' in sources:
+        sources = sources.replace('  ',' ').replace('\t',' ')
+    else:
+        break
+sources = sources.replace('}, {','},{').replace('},{','},\n{')
+input(sources)
+
+

abstract_webtools-0.1.6.40/src/abstract_webtools/managers/urlManager/urlManager.py

@@ -0,0 +1,220 @@
+import re
+import urllib.parse
+import requests
+from urllib.parse import urlparse, urlunparse, urljoin
+
+from ...abstract_webtools import *
+
+class urlManager:
+    """
+    Revised urlManager for managing and cleaning URLs.
+    
+    It splits URLs into their components, normalizes them (trimming spaces, lowercasing
+    scheme and domain, removing default ports, and cleaning up paths), and then creates
+    a list of potential variants (with/without www, http/https) so that a valid version
+    can be determined.
+    """
+    def __init__(self, url=None, session=None):
+        url = url or 'www.example.com'
+        self._url = url
+        self.session = session or requests
+        self.clean_urls = self.clean_url(url)
+        self.url = self.get_correct_url(clean_urls=self.clean_urls) or url
+        self.protocol, self.domain, self.path, self.query = self.url_to_pieces(self.url)
+        self.all_urls = []
+    
+    def url_to_pieces(self, url):
+        """
+        Split a URL into protocol, domain, path, and query components.
+        Uses urlparse for robustness.
+        """
+        try:
+            parsed = urlparse(url)
+            protocol = parsed.scheme if parsed.scheme else None
+            domain = parsed.netloc if parsed.netloc else None
+            path = parsed.path or ""
+            query = parsed.query or ""
+        except Exception as e:
+            print(f'The URL {url} was not reachable: {e}')
+            protocol, domain, path, query = None, None, "", ""
+        return protocol, domain, path, query
+
+    def clean_url(self, url=None) -> list:
+        """
+        Normalize and clean the URL, then return a list of potential URL variants.
+        
+        This method:
+          - Strips whitespace.
+          - Adds a scheme (defaults to https) if missing.
+          - Lowercases the scheme and domain.
+          - Removes default ports.
+          - Cleans up the path (removing duplicate slashes and trailing slash).
+          - Generates variants with and without 'www', and with both http and https.
+        """
+        url = url or self._url
+        url = url.strip()
+        # Ensure the URL has a scheme
+        if not re.match(r'https?://', url):
+            url = 'https://' + url
+
+        parsed = urlparse(url)
+        scheme = parsed.scheme.lower()
+        netloc = parsed.netloc.lower()
+        # Remove default port numbers if present
+        if ':' in netloc:
+            host, port = netloc.split(':', 1)
+            if (scheme == "http" and port == "80") or (scheme == "https" and port == "443"):
+                netloc = host
+        
+        # Normalize the path: remove duplicate slashes and a trailing slash
+        path = re.sub(r'//+', '/', parsed.path).rstrip('/')
+        
+        # Rebuild the cleaned URL without query or fragment
+        cleaned_url = urlunparse((scheme, netloc, path, '', '', ''))
+        
+        variants = []
+        # Add the primary variant
+        variants.append(cleaned_url)
+        # Generate a variant with/without 'www'
+        if netloc.startswith('www.'):
+            no_www = netloc[4:]
+            variants.append(urlunparse((scheme, no_www, path, '', '', '')))
+        else:
+            variants.append(urlunparse((scheme, f"www.{netloc}", path, '', '', '')))
+        
+        # Also generate variants with the alternate scheme
+        alt_scheme = 'http' if scheme == 'https' else 'https'
+        for variant in list(variants):
+            parsed_variant = urlparse(variant)
+            alt_variant = urlunparse((alt_scheme, parsed_variant.netloc, parsed_variant.path, '', '', ''))
+            variants.append(alt_variant)
+        
+        # Remove duplicates while preserving order
+        seen = set()
+        unique_variants = []
+        for v in variants:
+            if v not in seen:
+                unique_variants.append(v)
+                seen.add(v)
+        return unique_variants
+
+    def get_correct_url(self, url=None, clean_urls=None) -> str:
+        """
+        Attempts each URL variant by making an HTTP GET request.
+        Returns the first variant that returns a 200 OK response.
+        """
+        if url is None and clean_urls is None:
+            url = self._url
+            clean_urls = self.clean_urls
+        if url is not None and clean_urls is None:
+            clean_urls = self.clean_url(url)
+        elif url is None and clean_urls is not None:
+            url = self._url
+
+        for candidate in clean_urls:
+            try:
+                response = self.session.get(candidate, timeout=5)
+                if response.status_code == 200:
+                    return candidate
+            except requests.exceptions.RequestException as e:
+                print(f"Failed to reach {candidate}: {e}")
+        return None
+
+    def update_url(self, url):
+        """
+        Update the URL and refresh related attributes.
+        """
+        self._url = url
+        self.clean_urls = self.clean_url(url)
+        self.url = self.get_correct_url(clean_urls=self.clean_urls) or url
+        self.protocol, self.domain, self.path, self.query = self.url_to_pieces(self.url)
+        self.all_urls = []
+
+    def get_domain(self, url=None):
+        url = url or self.url
+        return urlparse(url).netloc
+
+    def url_join(self, base_url, path):
+        """
+        Joins a base URL with a relative path.
+        """
+        base_url = base_url.strip().rstrip('/')
+        path = path.strip().lstrip('/')
+        return f"{base_url}/{path}"
+
+    @property
+    def url(self):
+        return self._url
+
+    @url.setter
+    def url(self, new_url):
+        self._url = new_url
+
+    def is_valid_url(self, url=None):
+        """
+        Check if the given URL is valid.
+        """
+        url = url or self.url
+        parsed = urlparse(url)
+        return bool(parsed.scheme) and bool(parsed.netloc)
+    
+    def make_valid(self, href, url=None):
+        """
+        Validate a href. If it's not already valid, join it with the base URL.
+        """
+        if self.is_valid_url(href):
+            return href
+        base = url or self.url
+        new_link = urljoin(base, href)
+        if self.is_valid_url(new_link):
+            return new_link
+        return False
+
+    def get_relative_href(self, base, href):
+        """
+        For a relative href, join it with the base URL and strip any query or fragment.
+        """
+        joined = urljoin(base, href)
+        parsed = urlparse(joined)
+        clean_href = urlunparse((parsed.scheme, parsed.netloc, parsed.path, '', '', ''))
+        return clean_href
+
+    def url_basename(self, url=None):
+        url = url or self.url
+        path = urlparse(url).path
+        return path.strip('/').split('/')[-1]
+
+    def base_url(self, url=None):
+        url = url or self.url
+        match = re.match(r'https?://[^?#/]+/', url)
+        if match:
+            return match.group()
+        return None
+
+    def urljoin(self, base, path):
+        return urljoin(base, path)
+
+class urlManagerSingleton:
+    _instance = None
+
+    @staticmethod
+    def get_instance(url=None, session=requests):
+        if urlManagerSingleton._instance is None:
+            urlManagerSingleton._instance = urlManager(url, session=session)
+        elif urlManagerSingleton._instance.session != session or urlManagerSingleton._instance.url != url:
+            urlManagerSingleton._instance = urlManager(url, session=session)
+        return urlManagerSingleton._instance
+
+def get_url(url=None, url_mgr=None):
+    if not url and not url_mgr:
+        return None
+    if url:
+        url_mgr = urlManager(url)
+    return url_mgr.url
+
+def get_url_mgr(url=None, url_mgr=None):
+    if url_mgr is None and url:
+         url_mgr = urlManager(url=url)
+    if url_mgr and url is None:
+        url = url_mgr.url
+    return url_mgr

{abstract_webtools-0.1.6.38 → abstract_webtools-0.1.6.40/src/abstract_webtools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: abstract_webtools
-Version: 0.1.6.38
+Version: 0.1.6.40
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff

{abstract_webtools-0.1.6.38 → abstract_webtools-0.1.6.40}/src/abstract_webtools.egg-info/SOURCES.txt

@@ -36,6 +36,7 @@ src/abstract_webtools/managers/linkManager/linkManager.py
 src/abstract_webtools/managers/requestManager/__init__.py
 src/abstract_webtools/managers/requestManager/requestManager.py
 src/abstract_webtools/managers/soupManager/__init__.py
+src/abstract_webtools/managers/soupManager/asoueces.py
 src/abstract_webtools/managers/soupManager/soupManager.py
 src/abstract_webtools/managers/urlManager/__init__.py
 src/abstract_webtools/managers/urlManager/urlManager.py

abstract_webtools-0.1.6.38/src/abstract_webtools/managers/seleniumManager.py

@@ -1,119 +0,0 @@
-import os
-from ..abstract_webtools import *
-from .urlManager import *
-from urllib.parse import urlparse
-from abstract_utilities import *
-from selenium import webdriver
-from selenium.webdriver.chrome.options import Options
-import logging
-import urllib3
-
-# Suppress urllib3 warnings and debug logs
-urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-logging.getLogger("urllib3").setLevel(logging.WARNING)
-
-# Suppress Selenium logs
-logging.getLogger("selenium").setLevel(logging.WARNING)
-
-import os
-from selenium import webdriver
-from selenium.webdriver.chrome.options import Options
-
-# Setup Chrome options
-chrome_options = Options()
-chrome_options.binary_location = "/home/profiles/solcatcher/.cache/selenium/chrome/linux64/130.0.6723.58/chrome"
-chrome_options.add_argument("--headless")  # Run in headless mode
-chrome_options.add_argument("--no-sandbox")
-chrome_options.add_argument("--disable-dev-shm-usage")
-chrome_options.add_argument("--disable-gpu")
-chrome_options.add_argument("--disable-software-rasterizer")
-chrome_options.add_argument("--disable-extensions")
-chrome_options.add_argument("--remote-debugging-port=9222")
-
-
-class SingletonMeta(type):
-    _instances = {}
-    def __call__(cls, *args, **kwargs):
-        if cls not in cls._instances:
-            instance = super().__call__(*args, **kwargs)
-            cls._instances[cls] = instance
-        return cls._instances[cls]
-
-class seleniumManager(metaclass=SingletonMeta):
-    def __init__(self, url):
-        if not hasattr(self, 'initialized'):  # Prevent reinitialization
-            self.initialized = True
-            parsed_url = urlparse(url)
-            self.domain = parsed_url.netloc
-            self.scheme = parsed_url.scheme
-            self.base_url= f"{self.scheme}{self.domain}"
-            self.site_dir = os.path.join(os.getcwd(), self.domain)
-            os.makedirs(self.site_dir, exist_ok=True)
-            self.drivers = {}
-            self.page_type = []
-    
-    def get_url_to_path(self, url):
-        url = eatAll(str(url), ['',' ','\n','\t','\\','/'])
-        parsed_url = urlparse(url)
-        if parsed_url.netloc == self.domain:
-            paths = parsed_url.path.split('/')
-            dir_path = self.site_dir
-            for path in paths[:-1]:
-                dir_path = os.path.join(dir_path, path)
-                os.makedirs(dir_path, exist_ok=True)
-            self.page_type.append(os.path.splitext(paths[-1])[-1] or 'html' if len(self.page_type) == 0 else self.page_type[-1])
-            
-            dir_path = os.path.join(dir_path, paths[-1])
-            return dir_path
-
-    def saved_url_check(self, url):
-        path = self.get_url_to_path(url)
-        return path
-
-    def get_with_netloc(self, url):
-        parsed_url = urlparse(url)
-        if parsed_url.netloc == '':
-            url = f"{self.scheme}://{self.domain}/{url.strip()}"
-        return url
-
-    def get_driver(self, url):
-        if url and url not in self.drivers:
-            chrome_options = Options()
-            chrome_options.add_argument("--headless")
-            driver = webdriver.Chrome(options=chrome_options)
-            self.drivers[url] = driver
-            driver.get(url)
-        return self.drivers[url]
-def normalize_url(url, base_url=None):
-    """
-    Normalize and resolve relative URLs, ensuring proper domain and format.
-    """
-    # If URL starts with the base URL repeated, remove the extra part
-    manager = seleniumManager(url)
-    base_url = manager.base_url
-    if url.startswith(base_url):
-        url = url[len(base_url):]
-
-    # Resolve the URL against the base URL
-    normalized_url = urljoin(base_url, url.split('#')[0])
-
-    # Ensure only URLs belonging to the base domain are kept
-    if not normalized_url.startswith(base_url):
-        return None
-
-    return normalized_url
-# Function to get Selenium page source
-def get_selenium_source(url):
-    url_mgr = urlManager(url)
-    if url_mgr.url:
-        url = str(url_mgr.url)
-        manager = seleniumManager(url)
-        driver = manager.get_driver(url)
-        try:
-            # Get page source
-            page_source = driver.page_source
-            return page_source
-        finally:
-            # Don't quit the driver unless you're done with all interactions
-            pass
-

abstract_webtools-0.1.6.38/src/abstract_webtools/managers/urlManager/urlManager.py

@@ -1,230 +0,0 @@
-from ...abstract_webtools import *
-class urlManager:
-    """
-    urlManager is a class for managing URLs, including cleaning, validating, and finding the correct version.
-
-    Args:
-        url (str or None): The URL to manage (default is None).
-        session (requests.Session): A custom requests session (default is the requests module's session).
-
-    Attributes:
-        session (requests.Session): The requests session used for making HTTP requests.
-        clean_urls (list): List of cleaned URL variations.
-        url (str): The current URL.
-        protocol (str): The protocol part of the URL (e.g., "https").
-        domain (str): The domain part of the URL (e.g., "example.com").
-        path (str): The path part of the URL (e.g., "/path/to/resource").
-        query (str): The query part of the URL (e.g., "?param=value").
-        all_urls (list): List of all URLs (not used in the provided code).
-
-    Methods:
-        url_to_pieces(url): Split a URL into its protocol, domain, path, and query components.
-        clean_url(url): Return a list of potential URL versions with and without 'www' and 'http(s)'.
-        get_correct_url(url): Get the correct version of the URL from possible variations.
-        update_url(url): Update the URL and related attributes.
-        get_domain(url): Get the domain name from a URL.
-        url_join(url, path): Join a base URL with a path.
-        is_valid_url(url): Check if a URL is valid.
-        make_valid(href, url): Make a URL valid by joining it with a base URL.
-        get_relative_href(url, href): Get the relative href URL by joining it with a base URL.
-
-    Note:
-        - The urlManager class provides methods for managing URLs, including cleaning and validating them.
-        - It also includes methods for joining and validating relative URLs.
-    """
-
-    def __init__(self, url=None, session=None):
-        """
-        Initialize a urlManager instance.
-
-        Args:
-            url (str or None): The URL to manage (default is None).
-            session (requests.Session): A custom requests session (default is the requests module's session).
-        """
-        url = url or 'www.example.com'
-        self._url=url
-        self.url = url
-        self.session= session or requests
-        self.clean_urls = self.clean_url(url=url)
-        self.url = self.get_correct_url(clean_urls=self.clean_urls)
-        url_pieces = self.url_to_pieces(url=self.url)
-        self.protocol,self.domain,self.path,self.query=url_pieces
-        self.all_urls = []
-    def url_to_pieces(self, url):
-        
-        try:
-            match = re.match(r'^(https?)?://?([^/]+)(/[^?]+)?(\?.+)?', url)
-            if match:
-                protocol = match.group(1) if match.group(1) else None
-                domain = match.group(2) if match.group(1) else None
-                path = match.group(3) if match.group(3) else ""  # Handle None
-                query = match.group(4) if match.group(4) else ""  # Handle None
-        except:
-            print(f'the url {url} was not reachable')
-            protocol,domain,path,query=None,None,"",""
-        return protocol, domain, path, query
-
-    def clean_url(self,url=None) -> list:
-        """
-        Given a URL, return a list with potential URL versions including with and without 'www.', 
-        and with 'http://' and 'https://'.
-        """
-        url = url or self.url 
-        urls=[]
-        if url:
-            # Remove http:// or https:// prefix
-            cleaned = url.replace("http://", "").replace("https://", "")
-            no_subdomain = cleaned.replace("www.", "", 1)
-            
-            urls = [
-                f"https://{cleaned}",
-                f"http://{cleaned}",
-            ]
-
-            # Add variants without 'www' if it was present
-            if cleaned != no_subdomain:
-                urls.extend([
-                    f"https://{no_subdomain}",
-                    f"http://{no_subdomain}",
-                ])
-
-            # Add variants with 'www' if it wasn't present
-            else:
-                urls.extend([
-                    f"https://www.{cleaned}",
-                    f"http://www.{cleaned}",
-                ])
-
-        return urls
-
-    def get_correct_url(self,url=None,clean_urls=None) -> (str or None):
-        """
-        Gets the correct URL from the possible variations by trying each one with an HTTP request.
-
-        Args:
-            url (str): The URL to find the correct version of.
-            session (type(requests.Session), optional): The requests session to use for making HTTP requests.
-                Defaults to requests.
-
-        Returns:
-            str: The correct version of the URL if found, or None if none of the variations are valid.
-        """
-        self.url = url
-        if url==None and clean_urls != None:
-            if self.url:
-                url=self.url or clean_urls[0]
-        if url!=None and clean_urls==None:
-            clean_urls=self.clean_url(url)
-        elif url==None and clean_urls==None:
-            url=self.url
-            clean_urls=self.clean_urls
-        # Get the correct URL from the possible variations
-        for url in clean_urls:
-            try:
-                source = self.session.get(url)
-                return url
-            except requests.exceptions.RequestException as e:
-                print(e)
-        return None
-    def update_url(self,url):
-        # These methods seem essential for setting up the urlManager object.
-        self.url = url
-        self.clean_urls = self.clean_url()
-        self.correct_url = self.get_correct_url()
-        self.url =self.correct_url
-        self.protocol,self.domain,self.path,self.query=self.url_to_pieces(url=self.url)
-        self.all_urls = []
-    def get_domain(self,url=None):
-        url = url or self.url 
-        return urlparse(url).netloc
-    def url_join(self,url,path):
-        url = eatOuter(url,['/'])
-        path = eatInner(path,['/'])
-        slash=''
-        if path[0] not in ['?','&']:
-            slash = '/'
-        url = url+slash+path
-        return url
-    @property
-    def url(self):
-        return self._url
-    @url.setter
-    def url(self, new_url):
-        self._url = new_url
-    def is_valid_url(self,url=None):
-        """
-        Check if the given URL is valid.
-        """
-        url = url or self.url 
-        parsed = urlparse(url)
-        return bool(parsed.netloc) and bool(parsed.scheme)
-    
-    def make_valid(self,href,url=None):
-        def is_valid_url(url):
-            url = url or self.url 
-            """
-            Check if the given URL is valid.
-            """
-            parsed = urlparse(url)
-            return bool(parsed.netloc) and bool(parsed.scheme)
-        if is_valid_url(href):
-            return href
-        new_link=urljoin(url,href)
-        if is_valid_url(new_link):
-            return new_link
-        return False
-    
-    def get_relative_href(self,url,href):
-        # join the URL if it's relative (not an absolute link)
-        url = url or self.url 
-        href = urljoin(url, href)
-        parsed_href = urlparse(href)
-        # remove URL GET parameters, URL fragments, etc.
-        href = parsed_href.scheme + "://" + parsed_href.netloc + parsed_href.path
-        return href
-    def url_basename(self,url=None):
-        url = url or self.url 
-        path = urllib.parse.urlparse(url).path
-        return path.strip('/').split('/')[-1]
-
-
-    def base_url(self,url=None):
-        url = url or self.url 
-        return re.match(r'https?://[^?#]+/', url).group()
-
-
-    def urljoin(self,base, path):
-        if isinstance(path, bytes):
-            path = path.decode()
-        if not isinstance(path, str) or not path:
-            return None
-        if re.match(r'^(?:[a-zA-Z][a-zA-Z0-9+-.]*:)?//', path):
-            return path
-        if isinstance(base, bytes):
-            base = base.decode()
-        if not isinstance(base, str) or not re.match(
-                r'^(?:https?:)?//', base):
-            return None
-        return urllib.parse.urljoin(base, path)
-class urlManagerSingleton:
-    _instance = None
-    @staticmethod
-    def get_instance(url=None,session=requests):
-        if urlManagerSingleton._instance is None:
-            urlManagerSingleton._instance = urlManager(url,session=session)
-        elif urlManagerSingleton._instance.session != session or urlManagerSingleton._instance.url != url:
-            urlManagerSingleton._instance = urlManager(url,session=session)
-        return urlManagerSingleton._instance
-
-def get_url(url=None,url_mgr=None):
-    if not url and not url_mgr:
-        return None
-    if url:
-        url_mgr = urlManager(url)
-    return url_mgr.url
-def get_url_mgr(url=None,url_mgr=None):
-    if url_mgr == None and url:
-         url_mgr = urlManager(url=url)
-    if url_mgr and url == None:
-        url = url_mgr.url
-    return url_mgr