abstract-webtools 0.1.6.136__py3-none-any.whl → 0.1.6.137__py3-none-any.whl

abstract_webtools/managers/soupManager.py

@@ -1,362 +0,0 @@
-from ..abstract_webtools import *
-from .urlManager import *
-from .requestManager import *
-class soupManager:
-    """
-    SoupManager is a class for managing and parsing HTML source code using BeautifulSoup.
-
-    Args:
-        url (str or None): The URL to be parsed (default is None).
-        source_code (str or None): The HTML source code (default is None).
-        url_mgr (UrlManager or None): An instance of UrlManager (default is None).
-        requestManager (SafeRequest or None): An instance of SafeRequest (default is None).
-        parse_type (str): The type of parser to be used by BeautifulSoup (default is "html.parser").
-
-    Methods:
-        re_initialize(): Reinitialize the SoupManager with the current settings.
-        update_url(url): Update the URL and reinitialize the SoupManager.
-        update_source_code(source_code): Update the source code and reinitialize the SoupManager.
-        update_requestManager(requestManager): Update the request manager and reinitialize the SoupManager.
-        update_url_mgr(url_mgr): Update the URL manager and reinitialize the SoupManager.
-        update_parse_type(parse_type): Update the parsing type and reinitialize the SoupManager.
-        all_links: A property that provides access to all discovered links.
-        _all_links_get(): A method to load all discovered links.
-        get_all_website_links(tag="a", attr="href"): Get all URLs belonging to the same website.
-        meta_tags: A property that provides access to all discovered meta tags.
-        _meta_tags_get(): A method to load all discovered meta tags.
-        get_meta_tags(): Get all meta tags in the source code.
-        find_all(element, soup=None): Find all instances of an HTML element in the source code.
-        get_class(class_name, soup=None): Get the specified class from the HTML source code.
-        has_attributes(tag, *attrs): Check if an HTML tag has the specified attributes.
-        get_find_all_with_attributes(*attrs): Find all HTML tags with specified attributes.
-        get_all_desired_soup(tag=None, attr=None, attr_value=None): Get HTML tags based on specified criteria.
-        extract_elements(url, tag=None, class_name=None, class_value=None): Extract portions of source code based on filters.
-        find_all_with_attributes(class_name=None, *attrs): Find classes with associated href or src attributes.
-        get_images(tag_name, class_name, class_value): Get images with specific class and attribute values.
-        discover_classes_and_meta_images(tag_name, class_name_1, class_name_2, class_value, attrs): Discover classes and meta images.
-
-    Note:
-        - The SoupManager class is designed for parsing HTML source code using BeautifulSoup.
-        - It provides various methods to extract data and discover elements within the source code.
-    """
-    def __init__(self,url=None,source_code=None,url_mgr=None,req_mgr=None, parse_type="html.parser"):
-        self.soup=[]
-        url = get_url(url=url,url_mgr=url_mgr)
-        self.url_mgr = get_url_mgr(url=url,url_mgr=url_mgr)
-        self.url=self.url_mgr.url
-        self.req_mgr = get_req_mgr(req_mgr=req_mgr,url=self.url,url_mgr=self.url_mgr,source_code=source_code)
-        self.parse_type = parse_type
-        source_code = source_code or self.req_mgr.source_code or self.req_mgr.source_code_bytes
-        if source_code:
-            source_code = str(source_code)
-        self.source_code = source_code
-        self.soup= BeautifulSoup(self.source_code, self.parse_type)
-        self.all_tags_and_attribute_names = self.get_all_tags_and_attribute_names()
-        self.all_tags = self.all_tags_and_attribute_names.get('tags')
-        self.all_attribute_names = self.all_tags_and_attribute_names.get('attributes')
-        self.all_tags_and_attributes = self.all_tags + self.all_attribute_names
-        
-        self._all_links_data = None
-        self._meta_tags_data = None
-    def re_initialize(self):
-        self.soup= BeautifulSoup(self.source_code, self.parse_type)
-        self._all_links_data = None
-        self._meta_tags_data = None
-    def update_url(self,url):
-        self.url_mgr.update_url(url=url)
-        self.url=self.url_mgr.url
-        self.req_mgr.update_url(url=url)
-        self.source_code = self.req_mgr.source_code_bytes
-        self.re_initialize()
-    def update_source_code(self,source_code):
-        if source_code:
-            source_code = str(source_code)
-        self.source_code = source_code
-        self.re_initialize()
-    def update_requestManager(self,requestManager):
-        self.req_mgr = requestManager
-        self.url_mgr=self.req_mgr.url_mgr
-        self.url=self.url_mgr.url
-        self.source_code = self.req_mgr.source_code_bytes
-        self.re_initialize()
-    def update_url_mgr(self,url_mgr):
-        self.url_mgr=url_mgr
-        self.url=self.url_mgr.url
-        self.req_mgr.update_url_mgr(url_mgr=self.url_mgr)
-        self.source_code = self.req_mgr.source_code_bytes
-        self.re_initialize()
-    def update_parse_type(self,parse_type):
-        self.parse_type=parse_type
-        self.re_initialize()
-    @property
-    def all_links(self):
-        """This is a property that provides access to the _all_links_data attribute.
-        The first time it's accessed, it will load the data."""
-        if self._all_links_data is None:
-            print("Loading all links for the first time...")
-            self._all_links_data = self._all_links_get()
-        return self._all_links_data
-    def _all_links_get(self):
-        """A method that loads the data (can be replaced with whatever data loading logic you have)."""
-        return self.get_all_website_links()
-    def get_all_website_links(self,tag="a",attr="href") -> list:
-        """
-        Returns all URLs that are found on the specified URL and belong to the same website.
-
-        Args:
-            url (str): The URL to search for links.
-
-        Returns:
-            list: A list of URLs that belong to the same website as the specified URL.
-        """
-        all_urls=[self.url_mgr.url]
-        domain = self.url_mgr.domain
-        all_desired=self.get_all_website_links(tag=tag,attr=attr)
-        for tag in all_desired:
-            href = tag.attrs.get(attr)
-            if href == "" or href is None:
-                # href empty tag
-                continue
-            href=self.url_mgr.get_relative_href(self.url_mgr.url,href)
-            if not self.url_mgr.is_valid_url(href):
-                # not a valid URL
-                continue
-            if href in all_urls:
-                # already in the set
-                continue
-            if domain not in href:
-                # external link
-                continue
-            all_urls.append(href)
-                
-        return all_urls
-
-
-    @property
-    def meta_tags(self):
-        """This is a property that provides access to the _all_links_data attribute.
-        The first time it's accessed, it will load the data."""
-        if self._meta_tags_data is None:
-            print("Loading all links for the first time...")
-            self._meta_tags_data = self._all_links_get()
-        return self._meta_tags_data
-    def _meta_tags_get(self):
-        """A method that loads the data (can be replaced with whatever data loading logic you have)."""
-        return self.get_meta_tags()
-    def get_meta_tags(self):
-        tags = self.find_all("meta")
-        for meta_tag in tags:
-            for attr, values in meta_tag.attrs.items():
-                if attr not in self.meta_tags:
-                    self.meta_tags[attr] = []
-                if values not in self.meta_tags[attr]:
-                    self.meta_tags[attr].append(values)
-
-                    
-    def find_all(self,element,soup=None):
-        soup = self.soup if soup == None else soup
-        return soup.find_all(element)
-    def get_class(self,class_name,soup=None):
-        soup = self.soup if soup == None else soup
-        return soup.get(class_name)
-    @staticmethod
-    def has_attributes(tag, *attrs):
-        return any(tag.has_attr(attr) for attr in attrs)
-    def get_find_all_with_attributes(self, *attrs):
-        return self.soup.find_all(lambda t: self.has_attributes(t, *attrs))
-    def find_tags_by_attributes(self, tag: str = None, attr: str = None, attr_values: List[str] = None) ->List:
-        if not tag:
-            tags = self.soup.find_all(True)  # get all tags
-        else:
-            tags = self.soup.find_all(tag)  # get specific tags
-            
-        extracted_tags = []
-        for t in tags:
-            if attr:
-                attribute_value = t.get(attr)
-                if not attribute_value:  # skip tags without the desired attribute
-                    continue
-                if attr_values and not any(value in attribute_value for value in attr_values):  # skip tags without any of the desired attribute values
-                    continue
-            extracted_tags.append(t)
-        return extracted_tags
-
-
-    def extract_elements(self,url:str=None, tag:str=None, class_name:str=None, class_value:str=None) -> list:
-        """
-        Extracts portions of the source code from the specified URL based on provided filters.
-
-        Args:
-            url (str): The URL to fetch the source code from.
-            element_type (str, optional): The HTML element type to filter by. Defaults to None.
-            attribute_name (str, optional): The attribute name to filter by. Defaults to None.
-            class_name (str, optional): The class name to filter by. Defaults to None.
-
-        Returns:
-            list: A list of strings containing portions of the source code that match the provided filters.
-        """
-        elements = []
-        # If no filters are provided, return the entire source code
-        if not tag and not class_name and not class_value:
-            elements.append(str(self.soup))
-            return elements
-        # Find elements based on the filters provided
-        if tag:
-            elements.extend([str(tags) for tags in self.get_all_desired(tag)])
-        if class_name:
-            elements.extend([str(tags) for tags in self.get_all_desired(tag={class_name: True})])
-        if class_value:
-            elements.extend([str(tags) for tags in self.get_all_desired(class_name=class_name)])
-        return elements
-    def find_all_with_attributes(self, class_name=None, *attrs):
-        """
-        Discovers classes in the HTML content of the provided URL 
-        that have associated href or src attributes.
-
-        Args:
-            base_url (str): The URL from which to discover classes.
-
-        Returns:
-            set: A set of unique class names.
-        """
-
-    
-        unique_classes = set()
-        for tag in self.get_find_all_with_attributes(*attrs):
-            class_list = self.get_class(class_name=class_name, soup=tag)
-            unique_classes.update(class_list)
-        return unique_classes
-    def get_images(self, tag_name, class_name, class_value):
-        images = []
-        for tag in self.soup.find_all(tag_name):
-            if class_name in tag.attrs and tag.attrs[class_name] == class_value:
-                content = tag.attrs.get('content', '')
-                if content:
-                    images.append(content)
-        return images
-    def extract_text_sections(self) -> list:
-        """
-        Extract all sections of text from an HTML content using BeautifulSoup.
-
-        Args:
-            html_content (str): The HTML content to be parsed.
-
-        Returns:
-            list: A list containing all sections of text.
-        """
-        # Remove any script or style elements to avoid extracting JavaScript or CSS code
-        for script in self.soup(['script', 'style']):
-            script.decompose()
-
-        # Extract text from the remaining elements
-        text_sections = self.soup.stripped_strings
-        return [text for text in text_sections if text]
-    def discover_classes_and_meta_images(self, tag_name, class_name_1, class_name_2, class_value, attrs):
-        """
-        Discovers classes in the HTML content of the provided URL 
-        that have associated href or src attributes. Also, fetches 
-        image references from meta tags.
-
-        Args:
-            base_url (str): The URL from which to discover classes and meta images.
-
-        Returns:
-            tuple: A set of unique class names and a list of meta images.
-        """
-    
-        unique_classes = self.find_all_with_attributes(class_name=class_name_1, *attrs)
-        images = self.get_images(tag_name=tag_name, class_name=class_name_2, class_value=class_value)
-        return unique_classes, images
-    def get_all_tags_and_attribute_names(self):
-        tag_names = set()  # Using a set to ensure uniqueness
-        attribute_names = set()
-        get_all = self.find_tags_by_attributes()
-        for tag in get_all:  # True matches all tags
-            tag_names.add(tag.name)
-            for attr in tag.attrs:
-                attribute_names.add(attr)
-        tag_names_list = list(tag_names)
-        attribute_names_list = list(attribute_names)
-        return {"tags":tag_names_list,"attributes":attribute_names_list}
-
-    def get_all_attribute_values(self, tags_list=None):
-        """
-        Collects all attribute values for each specified tag or all tags if none are specified.
-        
-        Parameters:
-        - tags_list: List of specific tags to retrieve attributes from, e.g., ['script', 'img'].
-                    If None, retrieves attributes for all tags.
-        
-        Returns:
-        - attribute_values: Dictionary where each key is an attribute and the value is a list of unique values for that attribute.
-        """
-        attribute_values = {}
-        tags_list = tags_list or self.all_tags_and_attributes
-        # Get all tags matching tags_list criteria
-        for tag_name in tags_list:
-            for tag in self.soup.find_all(tag_name):
-                for attr, value in tag.attrs.items():
-                    if attr not in attribute_values:
-                        attribute_values[attr] = set()
-                    
-                    # Add attribute values
-                    if isinstance(value, list):
-                        attribute_values[attr].update(value)
-                    else:
-                        attribute_values[attr].add(value)
-        
-        # Convert each set to a list for consistency
-        for attr, values in attribute_values.items():
-            attribute_values[attr] = list(values)
-
-        # Capture JavaScript URLs inside <script> tags
-        attribute_values['script_links'] = self.get_js_links()
-
-        return attribute_values
-
-    def get_js_links(self):
-        """Extract URLs embedded in JavaScript within <script> tags."""
-        js_links = []
-        script_tags = self.soup.find_all('script')
-        for script in script_tags:
-            # Find URLs in the JavaScript code
-            urls_in_js = re.findall(r'["\'](https?://[^"\']+|/[^"\']+)["\']', script.get_text())
-            js_links.extend(urls_in_js)
-        return list(set(js_links))  # Remove duplicates
-    
-    @property
-    def url(self):
-        return self._url
-    @url.setter
-    def url(self, new_url):
-        self._url = new_url
-
-class SoupManagerSingleton():
-    _instance = None
-    @staticmethod
-    def get_instance(url_mgr,requestManager,parse_type="html.parser",source_code=None):
-        if SoupManagerSingleton._instance is None:
-            SoupManagerSingleton._instance = SoupManager(url_mgr,requestManager,parse_type=parse_type,source_code=source_code)
-        elif parse_type != SoupManagerSingleton._instance.parse_type  or source_code != SoupManagerSingleton._instance.source_code:
-            SoupManagerSingleton._instance = SoupManager(url_mgr,requestManager,parse_type=parse_type,source_code=source_code)
-        return SoupManagerSingleton._instance
-def get_soup_mgr(url=None,url_mgr=None,source_code=None,req_mgr=None,soup_mgr=None,parse_type="html.parser"):
-    url_mgr = get_url_mgr(url=url,url_mgr=url_mgr)
-    url = get_url(url=url,url_mgr=url_mgr)
-    req_mgr = get_req_mgr(url_mgr=url_mgr,url=url,source_code=source_code)
-    soup_mgr = soup_mgr or soupManager(url_mgr=url_mgr,req_mgr=req_mgr,url=url,source_code=source_code)
-    return soup_mgr
-def get_all_attribute_values(url=None,url_mgr=None,source_code=None,req_mgr=None,soup_mgr=None,tags_list = None,parse_type="html.parser"):
-    soup_mgr = get_soup_mgr(url=url,url_mgr=url_mgr,source_code=source_code,req_mgr=req_mgr,soup_mgr=soup_mgr)
-    return soup_mgr.get_all_attribute_values(tags_list=tags_list)
-def get_soup(url=None,url_mgr=None,req_mgr=None,source_code=None,soup_mgr=None,parse_type="html.parser"):
-    if source_code or soup_mgr:
-        if soup_mgr:
-            return soup_mgr.soup
-        return BeautifulSoup(source_code, parse_type)
-    url_mgr = get_url_mgr(url=url,url_mgr=url_mgr)
-    url = get_url(url=url,url_mgr=url_mgr)
-    req_mgr = req_mgr or get_req_mgr(url_mgr=url_mgr,url=url,source_code=source_code)
-    source_code = req_mgr.source_code
-    soup_mgr = get_soup_mgr(url=url,url_mgr=url_mgr,source_code=source_code,req_mgr=req_mgr,soup_mgr=soup_mgr)
-    return soup_mgr.soup

abstract_webtools/managers/urlManager.py

@@ -1,230 +0,0 @@
-from ..abstract_webtools import *
-class urlManager:
-    """
-    urlManager is a class for managing URLs, including cleaning, validating, and finding the correct version.
-
-    Args:
-        url (str or None): The URL to manage (default is None).
-        session (requests.Session): A custom requests session (default is the requests module's session).
-
-    Attributes:
-        session (requests.Session): The requests session used for making HTTP requests.
-        clean_urls (list): List of cleaned URL variations.
-        url (str): The current URL.
-        protocol (str): The protocol part of the URL (e.g., "https").
-        domain (str): The domain part of the URL (e.g., "example.com").
-        path (str): The path part of the URL (e.g., "/path/to/resource").
-        query (str): The query part of the URL (e.g., "?param=value").
-        all_urls (list): List of all URLs (not used in the provided code).
-
-    Methods:
-        url_to_pieces(url): Split a URL into its protocol, domain, path, and query components.
-        clean_url(url): Return a list of potential URL versions with and without 'www' and 'http(s)'.
-        get_correct_url(url): Get the correct version of the URL from possible variations.
-        update_url(url): Update the URL and related attributes.
-        get_domain(url): Get the domain name from a URL.
-        url_join(url, path): Join a base URL with a path.
-        is_valid_url(url): Check if a URL is valid.
-        make_valid(href, url): Make a URL valid by joining it with a base URL.
-        get_relative_href(url, href): Get the relative href URL by joining it with a base URL.
-
-    Note:
-        - The urlManager class provides methods for managing URLs, including cleaning and validating them.
-        - It also includes methods for joining and validating relative URLs.
-    """
-
-    def __init__(self, url=None, session=None):
-        """
-        Initialize a urlManager instance.
-
-        Args:
-            url (str or None): The URL to manage (default is None).
-            session (requests.Session): A custom requests session (default is the requests module's session).
-        """
-        url = url or 'www.example.com'
-        self._url=url
-        self.url = url
-        self.session= session or requests
-        self.clean_urls = self.clean_url(url=url)
-        self.url = self.get_correct_url(clean_urls=self.clean_urls)
-        url_pieces = self.url_to_pieces(url=self.url)
-        self.protocol,self.domain,self.path,self.query=url_pieces
-        self.all_urls = []
-    def url_to_pieces(self, url):
-        
-        try:
-            match = re.match(r'^(https?)?://?([^/]+)(/[^?]+)?(\?.+)?', url)
-            if match:
-                protocol = match.group(1) if match.group(1) else None
-                domain = match.group(2) if match.group(1) else None
-                path = match.group(3) if match.group(3) else ""  # Handle None
-                query = match.group(4) if match.group(4) else ""  # Handle None
-        except:
-            print(f'the url {url} was not reachable')
-            protocol,domain,path,query=None,None,"",""
-        return protocol, domain, path, query
-
-    def clean_url(self,url=None) -> list:
-        """
-        Given a URL, return a list with potential URL versions including with and without 'www.', 
-        and with 'http://' and 'https://'.
-        """
-        url = url or self.url 
-        urls=[]
-        if url:
-            # Remove http:// or https:// prefix
-            cleaned = url.replace("http://", "").replace("https://", "")
-            no_subdomain = cleaned.replace("www.", "", 1)
-            
-            urls = [
-                f"https://{cleaned}",
-                f"http://{cleaned}",
-            ]
-
-            # Add variants without 'www' if it was present
-            if cleaned != no_subdomain:
-                urls.extend([
-                    f"https://{no_subdomain}",
-                    f"http://{no_subdomain}",
-                ])
-
-            # Add variants with 'www' if it wasn't present
-            else:
-                urls.extend([
-                    f"https://www.{cleaned}",
-                    f"http://www.{cleaned}",
-                ])
-
-        return urls
-
-    def get_correct_url(self,url=None,clean_urls=None) -> (str or None):
-        """
-        Gets the correct URL from the possible variations by trying each one with an HTTP request.
-
-        Args:
-            url (str): The URL to find the correct version of.
-            session (type(requests.Session), optional): The requests session to use for making HTTP requests.
-                Defaults to requests.
-
-        Returns:
-            str: The correct version of the URL if found, or None if none of the variations are valid.
-        """
-        self.url = url
-        if url==None and clean_urls != None:
-            if self.url:
-                url=self.url or clean_urls[0]
-        if url!=None and clean_urls==None:
-            clean_urls=self.clean_url(url)
-        elif url==None and clean_urls==None:
-            url=self.url
-            clean_urls=self.clean_urls
-        # Get the correct URL from the possible variations
-        for url in clean_urls:
-            try:
-                source = self.session.get(url)
-                return url
-            except requests.exceptions.RequestException as e:
-                print(e)
-        return None
-    def update_url(self,url):
-        # These methods seem essential for setting up the urlManager object.
-        self.url = url
-        self.clean_urls = self.clean_url()
-        self.correct_url = self.get_correct_url()
-        self.url =self.correct_url
-        self.protocol,self.domain,self.path,self.query=self.url_to_pieces(url=self.url)
-        self.all_urls = []
-    def get_domain(self,url=None):
-        url = url or self.url 
-        return urlparse(url).netloc
-    def url_join(self,url,path):
-        url = eatOuter(url,['/'])
-        path = eatInner(path,['/'])
-        slash=''
-        if path[0] not in ['?','&']:
-            slash = '/'
-        url = url+slash+path
-        return url
-    @property
-    def url(self):
-        return self._url
-    @url.setter
-    def url(self, new_url):
-        self._url = new_url
-    def is_valid_url(self,url=None):
-        """
-        Check if the given URL is valid.
-        """
-        url = url or self.url 
-        parsed = urlparse(url)
-        return bool(parsed.netloc) and bool(parsed.scheme)
-    
-    def make_valid(self,href,url=None):
-        def is_valid_url(url):
-            url = url or self.url 
-            """
-            Check if the given URL is valid.
-            """
-            parsed = urlparse(url)
-            return bool(parsed.netloc) and bool(parsed.scheme)
-        if is_valid_url(href):
-            return href
-        new_link=urljoin(url,href)
-        if is_valid_url(new_link):
-            return new_link
-        return False
-    
-    def get_relative_href(self,url,href):
-        # join the URL if it's relative (not an absolute link)
-        url = url or self.url 
-        href = urljoin(url, href)
-        parsed_href = urlparse(href)
-        # remove URL GET parameters, URL fragments, etc.
-        href = parsed_href.scheme + "://" + parsed_href.netloc + parsed_href.path
-        return href
-    def url_basename(self,url=None):
-        url = url or self.url 
-        path = urllib.parse.urlparse(url).path
-        return path.strip('/').split('/')[-1]
-
-
-    def base_url(self,url=None):
-        url = url or self.url 
-        return re.match(r'https?://[^?#]+/', url).group()
-
-
-    def urljoin(self,base, path):
-        if isinstance(path, bytes):
-            path = path.decode()
-        if not isinstance(path, str) or not path:
-            return None
-        if re.match(r'^(?:[a-zA-Z][a-zA-Z0-9+-.]*:)?//', path):
-            return path
-        if isinstance(base, bytes):
-            base = base.decode()
-        if not isinstance(base, str) or not re.match(
-                r'^(?:https?:)?//', base):
-            return None
-        return urllib.parse.urljoin(base, path)
-class urlManagerSingleton:
-    _instance = None
-    @staticmethod
-    def get_instance(url=None,session=requests):
-        if urlManagerSingleton._instance is None:
-            urlManagerSingleton._instance = urlManager(url,session=session)
-        elif urlManagerSingleton._instance.session != session or urlManagerSingleton._instance.url != url:
-            urlManagerSingleton._instance = urlManager(url,session=session)
-        return urlManagerSingleton._instance
-
-def get_url(url=None,url_mgr=None):
-    if not url and not url_mgr:
-        return None
-    if url:
-        url_mgr = urlManager(url)
-    return url_mgr.url
-def get_url_mgr(url=None,url_mgr=None):
-    if url_mgr == None and url:
-         url_mgr = urlManager(url=url)
-    if url_mgr and url == None:
-        url = url_mgr.url
-    return url_mgr