abstract-webtools 0.1.6.38__py3-none-any.whl → 0.1.6.39__py3-none-any.whl

abstract_webtools/managers/soupManager/asoueces.py

@@ -0,0 +1,135 @@
+text = """{"title": "NoviSoul
+        novissbm@gmail.com", "href": "http://www.youtube.com/signin?authuser=0&next=%2Fwatch%3Fv%3DEaIYRM1yrM4&action_handle_signin=true", "description": ""},
+  {"title": "Sign in", "href": "https://accounts.google.com/ServiceLogin?continue=http%3A%2F%2Fwww.youtube.com%2Fsignin%3Faction_handle_signin%3Dtrue%26hl%3Den_GB%26next%3D%252Fwatch%253Fv%253DEaIYRM1yrM4%26nomobiletemp%3D1&uilel=3&service=youtube&passive=true&hl=en_GB", "description": ""},
+  {"title": "Sign up", "href": "http://www.youtube.com/signup?next=%2Fwatch%3Fv%3DEaIYRM1yrM4", "description": ""},
+  {"title": "9:58
+
+
+    
+ 
+
+Physics of Free Energy Deviceby Eugene Jeong        
+        
+336,881 views", "href": "http://www.youtube.com/watch?v=EB-jWfzkz_E", "description": ""},
+  {"title": "4:49
+
+
+    
+ 
+
+[www.witts.ws] Self-Running 40kW (40,000 Watt) Fuelless Generator (1 of 3)by wits2014        
+        
+488,638 views", "href": "http://www.youtube.com/watch?v=LFu-s6ZmGyE", "description": ""},
+  {"title": "2:33
+
+
+    
+ 
+
+Free Energy - Evidence of military antigravity technologyby DoubleMarkez        
+        
+390,020 views", "href": "http://www.youtube.com/watch?v=qljY-YfFaPc", "description": ""},
+  {"title": "15:01
+
+
+    
+ 
+
+APEX 2013   SSBM L10   Shroomed VS CT EMP Mew2Kingby Jason AxelrodRecommended for you", "href": "http://www.youtube.com/watch?v=pc7v49k5FhY", "description": ""},
+  {"title": "161
+            
+            
+              videos
+            
+          
+        
+      
+          
+              
+          
+          
+              
+          
+      
+    
+      
+        
+          
+Play all
+        
+      
+  
+washby dle3276", "href": "http://www.youtube.com/watch?v=AmcSt5hU4qA&list=PL4517CA6C6244A844", "description": ""},
+  {"title": "10:31
+
+
+    
+ 
+
+Pyramid Magnet - free energy - english subtitleby MrTermsof        
+        
+616,081 views", "href": "http://www.youtube.com/watch?v=pMbHswNoGWM", "description": ""},
+  {"title": "4:11
+
+
+    
+ 
+
+My all new newman motor 1.(TheDaftman)by theDaftman        
+        
+1,147,470 views", "href": "http://www.youtube.com/watch?v=dL4B_DNBtvc", "description": ""},
+  {"title": "2:18
+
+
+    
+ 
+
+Is there free energy in magnets?by aetherix01        
+        
+371,642 views", "href": "http://www.youtube.com/watch?v=vrn5B9a8aOk", "description": ""},
+  {"title": "3:00
+
+
+    
+ 
+
+The Most Dangerous Video On The Internet  - Trevor Paglenby killuminati63        
+        
+585,755 views", "href": "http://www.youtube.com/watch?v=9xEuhEHDJM8", "description": ""},
+  {"title": "2:18
+
+
+    
+ 
+
+Free Energy - Magnet Motorby ATBootstrap        
+        
+358,641 views", "href": "http://www.youtube.com/watch?v=hfkwCE3BeBs", "description": ""},
+  {"title": "2:38
+
+
+    
+ 
+
+100% free energy generator is easy to buildby LifeHack2012        
+        
+238,092 views", "href": "http://www.youtube.com/watch?v=GEUyhhMEs7U", "description": ""},
+  {"title": "3:41
+
+
+    
+ 
+
+5KW free energy –±–µ—Å—Ç–æ–ø–ª–∏–≤–Ω—ã–π –≥–µ–Ω–µ—Ä–∞—Ç–æ—Ä Kapanadze –ö–∞–ø–∞–Ω–∞–¥–∑–µby Alexander Frolov        
+        
+488,213 views", "href": "http://www.youtube.com/watch?v=uxQ99R4gOWY", "description": ""},""".split('\n')
+sources = ' '.join([te for te in text if te])
+while True:
+    if '  ' in sources:
+        sources = sources.replace('  ',' ').replace('\t',' ')
+    else:
+        break
+sources = sources.replace('}, {','},{').replace('},{','},\n{')
+input(sources)
+
+

abstract_webtools/managers/urlManager/urlManager.py

@@ -1,230 +1,220 @@
+import re
+import urllib.parse
+import requests
+from urllib.parse import urlparse, urlunparse, urljoin
+
 from ...abstract_webtools import *
+
 class urlManager:
     """
-    urlManager is a class for managing URLs, including cleaning, validating, and finding the correct version.
-
-    Args:
-        url (str or None): The URL to manage (default is None).
-        session (requests.Session): A custom requests session (default is the requests module's session).
-
-    Attributes:
-        session (requests.Session): The requests session used for making HTTP requests.
-        clean_urls (list): List of cleaned URL variations.
-        url (str): The current URL.
-        protocol (str): The protocol part of the URL (e.g., "https").
-        domain (str): The domain part of the URL (e.g., "example.com").
-        path (str): The path part of the URL (e.g., "/path/to/resource").
-        query (str): The query part of the URL (e.g., "?param=value").
-        all_urls (list): List of all URLs (not used in the provided code).
-
-    Methods:
-        url_to_pieces(url): Split a URL into its protocol, domain, path, and query components.
-        clean_url(url): Return a list of potential URL versions with and without 'www' and 'http(s)'.
-        get_correct_url(url): Get the correct version of the URL from possible variations.
-        update_url(url): Update the URL and related attributes.
-        get_domain(url): Get the domain name from a URL.
-        url_join(url, path): Join a base URL with a path.
-        is_valid_url(url): Check if a URL is valid.
-        make_valid(href, url): Make a URL valid by joining it with a base URL.
-        get_relative_href(url, href): Get the relative href URL by joining it with a base URL.
-
-    Note:
-        - The urlManager class provides methods for managing URLs, including cleaning and validating them.
-        - It also includes methods for joining and validating relative URLs.
+    Revised urlManager for managing and cleaning URLs.
+    
+    It splits URLs into their components, normalizes them (trimming spaces, lowercasing
+    scheme and domain, removing default ports, and cleaning up paths), and then creates
+    a list of potential variants (with/without www, http/https) so that a valid version
+    can be determined.
     """
-
     def __init__(self, url=None, session=None):
-        """
-        Initialize a urlManager instance.
-
-        Args:
-            url (str or None): The URL to manage (default is None).
-            session (requests.Session): A custom requests session (default is the requests module's session).
-        """
         url = url or 'www.example.com'
-        self._url=url
-        self.url = url
-        self.session= session or requests
-        self.clean_urls = self.clean_url(url=url)
-        self.url = self.get_correct_url(clean_urls=self.clean_urls)
-        url_pieces = self.url_to_pieces(url=self.url)
-        self.protocol,self.domain,self.path,self.query=url_pieces
+        self._url = url
+        self.session = session or requests
+        self.clean_urls = self.clean_url(url)
+        self.url = self.get_correct_url(clean_urls=self.clean_urls) or url
+        self.protocol, self.domain, self.path, self.query = self.url_to_pieces(self.url)
         self.all_urls = []
+    
     def url_to_pieces(self, url):
-        
+        """
+        Split a URL into protocol, domain, path, and query components.
+        Uses urlparse for robustness.
+        """
         try:
-            match = re.match(r'^(https?)?://?([^/]+)(/[^?]+)?(\?.+)?', url)
-            if match:
-                protocol = match.group(1) if match.group(1) else None
-                domain = match.group(2) if match.group(1) else None
-                path = match.group(3) if match.group(3) else ""  # Handle None
-                query = match.group(4) if match.group(4) else ""  # Handle None
-        except:
-            print(f'the url {url} was not reachable')
-            protocol,domain,path,query=None,None,"",""
+            parsed = urlparse(url)
+            protocol = parsed.scheme if parsed.scheme else None
+            domain = parsed.netloc if parsed.netloc else None
+            path = parsed.path or ""
+            query = parsed.query or ""
+        except Exception as e:
+            print(f'The URL {url} was not reachable: {e}')
+            protocol, domain, path, query = None, None, "", ""
         return protocol, domain, path, query
 
-    def clean_url(self,url=None) -> list:
-        """
-        Given a URL, return a list with potential URL versions including with and without 'www.', 
-        and with 'http://' and 'https://'.
-        """
-        url = url or self.url 
-        urls=[]
-        if url:
-            # Remove http:// or https:// prefix
-            cleaned = url.replace("http://", "").replace("https://", "")
-            no_subdomain = cleaned.replace("www.", "", 1)
-            
-            urls = [
-                f"https://{cleaned}",
-                f"http://{cleaned}",
-            ]
-
-            # Add variants without 'www' if it was present
-            if cleaned != no_subdomain:
-                urls.extend([
-                    f"https://{no_subdomain}",
-                    f"http://{no_subdomain}",
-                ])
-
-            # Add variants with 'www' if it wasn't present
-            else:
-                urls.extend([
-                    f"https://www.{cleaned}",
-                    f"http://www.{cleaned}",
-                ])
-
-        return urls
-
-    def get_correct_url(self,url=None,clean_urls=None) -> (str or None):
-        """
-        Gets the correct URL from the possible variations by trying each one with an HTTP request.
-
-        Args:
-            url (str): The URL to find the correct version of.
-            session (type(requests.Session), optional): The requests session to use for making HTTP requests.
-                Defaults to requests.
-
-        Returns:
-            str: The correct version of the URL if found, or None if none of the variations are valid.
-        """
-        self.url = url
-        if url==None and clean_urls != None:
-            if self.url:
-                url=self.url or clean_urls[0]
-        if url!=None and clean_urls==None:
-            clean_urls=self.clean_url(url)
-        elif url==None and clean_urls==None:
-            url=self.url
-            clean_urls=self.clean_urls
-        # Get the correct URL from the possible variations
-        for url in clean_urls:
+    def clean_url(self, url=None) -> list:
+        """
+        Normalize and clean the URL, then return a list of potential URL variants.
+        
+        This method:
+          - Strips whitespace.
+          - Adds a scheme (defaults to https) if missing.
+          - Lowercases the scheme and domain.
+          - Removes default ports.
+          - Cleans up the path (removing duplicate slashes and trailing slash).
+          - Generates variants with and without 'www', and with both http and https.
+        """
+        url = url or self._url
+        url = url.strip()
+        # Ensure the URL has a scheme
+        if not re.match(r'https?://', url):
+            url = 'https://' + url
+
+        parsed = urlparse(url)
+        scheme = parsed.scheme.lower()
+        netloc = parsed.netloc.lower()
+        # Remove default port numbers if present
+        if ':' in netloc:
+            host, port = netloc.split(':', 1)
+            if (scheme == "http" and port == "80") or (scheme == "https" and port == "443"):
+                netloc = host
+        
+        # Normalize the path: remove duplicate slashes and a trailing slash
+        path = re.sub(r'//+', '/', parsed.path).rstrip('/')
+        
+        # Rebuild the cleaned URL without query or fragment
+        cleaned_url = urlunparse((scheme, netloc, path, '', '', ''))
+        
+        variants = []
+        # Add the primary variant
+        variants.append(cleaned_url)
+        # Generate a variant with/without 'www'
+        if netloc.startswith('www.'):
+            no_www = netloc[4:]
+            variants.append(urlunparse((scheme, no_www, path, '', '', '')))
+        else:
+            variants.append(urlunparse((scheme, f"www.{netloc}", path, '', '', '')))
+        
+        # Also generate variants with the alternate scheme
+        alt_scheme = 'http' if scheme == 'https' else 'https'
+        for variant in list(variants):
+            parsed_variant = urlparse(variant)
+            alt_variant = urlunparse((alt_scheme, parsed_variant.netloc, parsed_variant.path, '', '', ''))
+            variants.append(alt_variant)
+        
+        # Remove duplicates while preserving order
+        seen = set()
+        unique_variants = []
+        for v in variants:
+            if v not in seen:
+                unique_variants.append(v)
+                seen.add(v)
+        return unique_variants
+
+    def get_correct_url(self, url=None, clean_urls=None) -> str:
+        """
+        Attempts each URL variant by making an HTTP GET request.
+        Returns the first variant that returns a 200 OK response.
+        """
+        if url is None and clean_urls is None:
+            url = self._url
+            clean_urls = self.clean_urls
+        if url is not None and clean_urls is None:
+            clean_urls = self.clean_url(url)
+        elif url is None and clean_urls is not None:
+            url = self._url
+
+        for candidate in clean_urls:
             try:
-                source = self.session.get(url)
-                return url
+                response = self.session.get(candidate, timeout=5)
+                if response.status_code == 200:
+                    return candidate
             except requests.exceptions.RequestException as e:
-                print(e)
+                print(f"Failed to reach {candidate}: {e}")
         return None
-    def update_url(self,url):
-        # These methods seem essential for setting up the urlManager object.
-        self.url = url
-        self.clean_urls = self.clean_url()
-        self.correct_url = self.get_correct_url()
-        self.url =self.correct_url
-        self.protocol,self.domain,self.path,self.query=self.url_to_pieces(url=self.url)
+
+    def update_url(self, url):
+        """
+        Update the URL and refresh related attributes.
+        """
+        self._url = url
+        self.clean_urls = self.clean_url(url)
+        self.url = self.get_correct_url(clean_urls=self.clean_urls) or url
+        self.protocol, self.domain, self.path, self.query = self.url_to_pieces(self.url)
         self.all_urls = []
-    def get_domain(self,url=None):
-        url = url or self.url 
+
+    def get_domain(self, url=None):
+        url = url or self.url
         return urlparse(url).netloc
-    def url_join(self,url,path):
-        url = eatOuter(url,['/'])
-        path = eatInner(path,['/'])
-        slash=''
-        if path[0] not in ['?','&']:
-            slash = '/'
-        url = url+slash+path
-        return url
+
+    def url_join(self, base_url, path):
+        """
+        Joins a base URL with a relative path.
+        """
+        base_url = base_url.strip().rstrip('/')
+        path = path.strip().lstrip('/')
+        return f"{base_url}/{path}"
+
     @property
     def url(self):
         return self._url
+
     @url.setter
     def url(self, new_url):
         self._url = new_url
-    def is_valid_url(self,url=None):
+
+    def is_valid_url(self, url=None):
         """
         Check if the given URL is valid.
         """
-        url = url or self.url 
+        url = url or self.url
         parsed = urlparse(url)
-        return bool(parsed.netloc) and bool(parsed.scheme)
+        return bool(parsed.scheme) and bool(parsed.netloc)
     
-    def make_valid(self,href,url=None):
-        def is_valid_url(url):
-            url = url or self.url 
-            """
-            Check if the given URL is valid.
-            """
-            parsed = urlparse(url)
-            return bool(parsed.netloc) and bool(parsed.scheme)
-        if is_valid_url(href):
+    def make_valid(self, href, url=None):
+        """
+        Validate a href. If it's not already valid, join it with the base URL.
+        """
+        if self.is_valid_url(href):
             return href
-        new_link=urljoin(url,href)
-        if is_valid_url(new_link):
+        base = url or self.url
+        new_link = urljoin(base, href)
+        if self.is_valid_url(new_link):
             return new_link
         return False
-    
-    def get_relative_href(self,url,href):
-        # join the URL if it's relative (not an absolute link)
-        url = url or self.url 
-        href = urljoin(url, href)
-        parsed_href = urlparse(href)
-        # remove URL GET parameters, URL fragments, etc.
-        href = parsed_href.scheme + "://" + parsed_href.netloc + parsed_href.path
-        return href
-    def url_basename(self,url=None):
-        url = url or self.url 
-        path = urllib.parse.urlparse(url).path
+
+    def get_relative_href(self, base, href):
+        """
+        For a relative href, join it with the base URL and strip any query or fragment.
+        """
+        joined = urljoin(base, href)
+        parsed = urlparse(joined)
+        clean_href = urlunparse((parsed.scheme, parsed.netloc, parsed.path, '', '', ''))
+        return clean_href
+
+    def url_basename(self, url=None):
+        url = url or self.url
+        path = urlparse(url).path
         return path.strip('/').split('/')[-1]
 
+    def base_url(self, url=None):
+        url = url or self.url
+        match = re.match(r'https?://[^?#/]+/', url)
+        if match:
+            return match.group()
+        return None
+
+    def urljoin(self, base, path):
+        return urljoin(base, path)
 
-    def base_url(self,url=None):
-        url = url or self.url 
-        return re.match(r'https?://[^?#]+/', url).group()
-
-
-    def urljoin(self,base, path):
-        if isinstance(path, bytes):
-            path = path.decode()
-        if not isinstance(path, str) or not path:
-            return None
-        if re.match(r'^(?:[a-zA-Z][a-zA-Z0-9+-.]*:)?//', path):
-            return path
-        if isinstance(base, bytes):
-            base = base.decode()
-        if not isinstance(base, str) or not re.match(
-                r'^(?:https?:)?//', base):
-            return None
-        return urllib.parse.urljoin(base, path)
 class urlManagerSingleton:
     _instance = None
+
     @staticmethod
-    def get_instance(url=None,session=requests):
+    def get_instance(url=None, session=requests):
         if urlManagerSingleton._instance is None:
-            urlManagerSingleton._instance = urlManager(url,session=session)
+            urlManagerSingleton._instance = urlManager(url, session=session)
         elif urlManagerSingleton._instance.session != session or urlManagerSingleton._instance.url != url:
-            urlManagerSingleton._instance = urlManager(url,session=session)
+            urlManagerSingleton._instance = urlManager(url, session=session)
         return urlManagerSingleton._instance
 
-def get_url(url=None,url_mgr=None):
+def get_url(url=None, url_mgr=None):
     if not url and not url_mgr:
         return None
     if url:
         url_mgr = urlManager(url)
     return url_mgr.url
-def get_url_mgr(url=None,url_mgr=None):
-    if url_mgr == None and url:
+
+def get_url_mgr(url=None, url_mgr=None):
+    if url_mgr is None and url:
          url_mgr = urlManager(url=url)
-    if url_mgr and url == None:
+    if url_mgr and url is None:
         url = url_mgr.url
-    return url_mgr 
+    return url_mgr

{abstract_webtools-0.1.6.38.dist-info → abstract_webtools-0.1.6.39.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: abstract_webtools
-Version: 0.1.6.38
+Version: 0.1.6.39
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff

{abstract_webtools-0.1.6.38.dist-info → abstract_webtools-0.1.6.39.dist-info}/RECORD

@@ -30,12 +30,13 @@ abstract_webtools/managers/linkManager/linkManager.py,sha256=roxOzOELca0rOlcMaJk
 abstract_webtools/managers/requestManager/__init__.py,sha256=z2qGtweEoO_OKr959LGxVXEMu1hu7PIkmh89BEh5TI8,30
 abstract_webtools/managers/requestManager/requestManager.py,sha256=MrPJAXRNDXjwE_BzJF3xwraT54IxVNmPU4eHhDgNmbE,17351
 abstract_webtools/managers/soupManager/__init__.py,sha256=mqfXfqM9sWlYpOkoXUqtBoVvk2KQx1862NnmRVJwGtY,27
+abstract_webtools/managers/soupManager/asoueces.py,sha256=OaXqolZl0dI7b09NYwJ3Wnhuxf89ahZ1GjsOqy0GXfk,3506
 abstract_webtools/managers/soupManager/soupManager.py,sha256=U3_o189-OWoBRaSCe2sIkg-bHxBt2mKpYMyZd-nJjLQ,17201
 abstract_webtools/managers/urlManager/__init__.py,sha256=gaJCHeK91Z-eYsBnxgdhbIUten1-gbx-zqx70R6ag-Y,26
-abstract_webtools/managers/urlManager/urlManager.py,sha256=-HH6TEBXvCU2CfBdy3mwqn0eSqEZyWlrbF8B0XOj1LU,8859
+abstract_webtools/managers/urlManager/urlManager.py,sha256=vCFuLADmv3h7icaaoAsImGqb_49VizPY_ZvMl-C7PYk,7756
 abstract_webtools/managers/videos/Heather brooke swallo from condom.mp4,sha256=h-bKFLAHt7pGLGu4EcMvSSox7BPRK0Nga3u813iMVKQ,8335544
-abstract_webtools-0.1.6.38.dist-info/LICENSE,sha256=g3WEJFiVS27HyCGRTwKSsMLyciMaGFdWcZGOe1QalZk,3877
-abstract_webtools-0.1.6.38.dist-info/METADATA,sha256=wtJTD9Lcdho-o-q_2fW6Ds1QfyIbZNbeWgCQSvLRMZo,16051
-abstract_webtools-0.1.6.38.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-abstract_webtools-0.1.6.38.dist-info/top_level.txt,sha256=2DMJ7RmjTcjCsa-uwAV0K6eXXlIIkFDEjBLg_uyCmCI,18
-abstract_webtools-0.1.6.38.dist-info/RECORD,,
+abstract_webtools-0.1.6.39.dist-info/LICENSE,sha256=g3WEJFiVS27HyCGRTwKSsMLyciMaGFdWcZGOe1QalZk,3877
+abstract_webtools-0.1.6.39.dist-info/METADATA,sha256=8sK_2myKFHmzx2eW1vsPGUvlyGWK19DX_tCI2PTP6gc,16051
+abstract_webtools-0.1.6.39.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
+abstract_webtools-0.1.6.39.dist-info/top_level.txt,sha256=2DMJ7RmjTcjCsa-uwAV0K6eXXlIIkFDEjBLg_uyCmCI,18
+abstract_webtools-0.1.6.39.dist-info/RECORD,,

{abstract_webtools-0.1.6.38.dist-info → abstract_webtools-0.1.6.39.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.2)
+Generator: setuptools (76.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any