abstract-webtools 0.1.6.136__py3-none-any.whl → 0.1.6.138__py3-none-any.whl

abstract_webtools/abstract_userpit.py

@@ -1,169 +0,0 @@
-import requests
-from bs4 import BeautifulSoup
-from urllib.parse import urljoin, urlparse
-import os
-import shutil
-import time
-from abstract_webtools import *
-
-
-# Import your custom classes/functions
-# from your_module import linkManager, get_soup_mgr
-
-# Configuration
-def normalize_url(url, base_url):
-    """
-    Normalize and resolve relative URLs, ensuring proper domain and format.
-    """
-    # If URL starts with the base URL repeated, remove the extra part
-    if url.startswith(base_url):
-        url = url[len(base_url):]
-
-    # Resolve the URL against the base URL
-    normalized_url = urljoin(base_url, url.split('#')[0])
-
-    # Ensure only URLs belonging to the base domain are kept
-    if not normalized_url.startswith(base_url):
-        return None
-
-    return normalized_url
-
-
-def is_valid_url(url, base_domain):
-    """
-    Check if the URL is valid and belongs to the same domain.
-    """
-    parsed = urlparse(url)
-    return parsed.scheme in ('http', 'https') and parsed.netloc == base_domain
-def save_page(url, content,output_dir):
-    """
-    Save HTML page to local directory.
-    """
-    parsed_url = urlparse(url)
-    page_path = parsed_url.path.lstrip('/')
-
-    if not page_path or page_path.endswith('/'):
-        page_path = os.path.join(page_path, 'index.html')
-    elif not os.path.splitext(page_path)[1]:
-        page_path += '.html'
-
-    page_full_path = os.path.join(output_dir, page_path)
-    os.makedirs(os.path.dirname(page_full_path), exist_ok=True)
-
-    with open(page_full_path, 'w', encoding='utf-8') as f:
-        f.write(content)
-    print(f"Saved page: {page_full_path}")
-def save_asset(asset_url, base_url,output_dir,session):
-    """
-    Download and save assets like images, CSS, JS files.
-    """
-    asset_url = normalize_url(asset_url, base_url)
-    if asset_url in downloaded_assets:
-        return
-    downloaded_assets.add(asset_url)
-
-    parsed_url = urlparse(asset_url)
-    asset_path = parsed_url.path.lstrip('/')
-    if not asset_path:
-        return  # Skip if asset path is empty
-
-    asset_full_path = os.path.join(output_dir, asset_path)
-    os.makedirs(os.path.dirname(asset_full_path), exist_ok=True)
-
-    try:
-        response = session.get(asset_url, stream=True)
-        response.raise_for_status()
-        with open(asset_full_path, 'wb') as f:
-            shutil.copyfileobj(response.raw, f)
-        print(f"Saved asset: {asset_full_path}")
-    except Exception as e:
-        print(f"Failed to save asset {asset_url}: {e}")
-class usurpManager():
-    def __init__(self,url,output_dir=None,max_depth=None,wait_between_requests=None,operating_system=None, browser=None, version=None,user_agent=None,website_bot=None):
-        self.url = url
-        website_bot = website_bot or 'http://yourwebsite.com/bot'
-        self.user_agent_mgr = UserAgentManager(operating_system=operating_system,browser=browser,version=version,user_agent=user_agent)
-        self.BASE_URL = urlManager(url=self.url).url  # Replace with your website's URL
-        self.OUTPUT_DIR = output_dir or 'download_site'
-        self.MAX_DEPTH = max_depth or 5  # Adjust as needed
-        self.WAIT_BETWEEN_REQUESTS = wait_between_requests or 1  # Seconds to wait between requests
-        USER_AGENT = self.user_agent_mgr.get_user_agent()
-        self.USER_AGENT = f"{USER_AGENT};{website_bot})"  # Customize as needed
-        # Initialize global sets
-        self.visited_pages = set()
-        self.downloaded_assets = set()
-
-        # Session with custom headers
-        self.session = requests.Session()
-        self.session.headers.update({
-            'User-Agent': USER_AGENT,
-            'Accept-Language': 'en-US,en;q=0.5',
-            "Access-Control-Allow-Origin": "*"})
-
-    def process_page(self,url, depth, base_domain):
-        """
-        Process a single page: download assets, save HTML, and crawl links.
-        """
-        print(url)
-        if url in self.visited_pages or depth > self.MAX_DEPTH:
-            return
-        self.visited_pages.add(url)
-        
-        try:
-            # Fetch the page content
-            response = self.session.get(url)
-            response.raise_for_status()
-            content = response.text
-
-            # Use your get_soup_mgr function to get the soup and attributes
-            soup_mgr = get_soup_mgr(url=url)
-            soup = soup_mgr.soup
-            all_attributes = soup_mgr.get_all_attribute_values()
-            # Now you can use all_attributes as needed
-
-            # Update asset links to local paths
-            for tag in soup.find_all(['img', 'script', 'link']):
-                attr = 'src' if tag.name != 'link' else 'href'
-                asset_url = tag.get(attr)
-                if asset_url:
-                    full_asset_url = normalize_url(asset_url, url)
-                    parsed_asset_url = urlparse(full_asset_url)
-
-                    if is_valid_url(full_asset_url, base_domain):
-                        save_asset(full_asset_url, self.url,self.session)
-                        # Update tag to point to the local asset
-                        local_asset_path = '/' + parsed_asset_url.path.lstrip('/')
-                        tag[attr] = local_asset_path
-
-            # Save the modified page
-            save_page(url, str(soup),self.OUTPUT_DIR)
-
-            # Use your linkManager to find all domain links
-            link_mgr = linkManager(url=url)
-            all_domains = link_mgr.find_all_domain()
-
-            # Process each domain link
-            for link_url in all_domains:
-                normalized_link = normalize_url(link_url, url)
-                if is_valid_url(normalized_link, base_domain):
-                    time.sleep(self.WAIT_BETWEEN_REQUESTS)
-                    self.process_page(normalized_link, depth + 1, base_domain)
-
-        except Exception as e:
-            print(f"Failed to process page {url}: {e}")
-
-    def main(self):
-        # Ensure output directory exists
-        os.makedirs(self.OUTPUT_DIR, exist_ok=True)
-
-        base_parsed = urlparse(self.BASE_URL)
-        base_domain = base_parsed.netloc
-
-        self.process_page(self.BASE_URL, 0, base_domain)
-        print("Website copying completed.")
-def test_download(url,directory):
-    url=url or 'https://algassert.com/quantum/2016/01/07/Delayed-Choice-Quantum-Erasure.html'
-    output_dir= directory or os.path.join(os.getcwd(),'testit')
-    os.makedirs(output_dir,exist_ok=True)
-    site_mgr = usurpManager(url,output_dir)
-    site_mgr.main()

abstract_webtools/managers/linkManager.py

@@ -1,189 +0,0 @@
-from ..abstract_webtools import *
-from .urlManager import *
-from .requestManager import *
-from .soupManager import *
-def get_url(url=None,url_mgr=None):
-    if not url and not url_mgr:
-        return None
-    if url:
-        url_mgr = urlManager(url)
-    return url_mgr.url
-def get_url_mgr(url=None,url_mgr=None):
-    if not url and not url_mgr:
-        return None
-    if url:
-        url_mgr = urlManager(url)
-    return url_mgr 
-class linkManager:
-    """
-    LinkManager is a class for managing and extracting links and image links from a web page.
-
-    Args:
-        url (str): The URL of the web page (default is "https://example.com").
-        source_code (str or None): The source code of the web page (default is None).
-        url_mgr (UrlManager or None): An instance of UrlManager (default is None).
-        request_manager (requestManager or None): An instance of requestManager (default is None).
-        soup_manager (SoupManager or None): An instance of SoupManager (default is None).
-        image_link_tags (str): HTML tags to identify image links (default is 'img').
-        img_link_attrs (str): HTML attributes to identify image link URLs (default is 'src').
-        link_tags (str): HTML tags to identify links (default is 'a').
-        link_attrs (str): HTML attributes to identify link URLs (default is 'href').
-        strict_order_tags (bool): Flag to indicate if tags and attributes should be matched strictly (default is False).
-        img_attr_value_desired (list or None): Desired attribute values for image links (default is None).
-        img_attr_value_undesired (list or None): Undesired attribute values for image links (default is None).
-        link_attr_value_desired (list or None): Desired attribute values for links (default is None).
-        link_attr_value_undesired (list or None): Undesired attribute values for links (default is None).
-        associated_data_attr (list): HTML attributes to associate with the extracted links (default is ["data-title", 'alt', 'title']).
-        get_img (list): HTML attributes used to identify associated images (default is ["data-title", 'alt', 'title']).
-
-    Methods:
-        re_initialize(): Reinitialize the LinkManager with the current settings.
-        update_url_mgr(url_mgr): Update the URL manager with a new instance.
-        update_url(url): Update the URL and reinitialize the LinkManager.
-        update_source_code(source_code): Update the source code and reinitialize the LinkManager.
-        update_soup_manager(soup_manager): Update the SoupManager and reinitialize the LinkManager.
-        update_desired(...): Update the desired settings and reinitialize the LinkManager.
-        find_all_desired(...): Find all desired links or image links based on the specified criteria.
-        find_all_domain(): Find all unique domain names in the extracted links.
-
-    Note:
-        - The LinkManager class helps manage and extract links and image links from web pages.
-        - The class provides flexibility in specifying criteria for link extraction.
-    """
-    def __init__(self,url=None,source_code=None,url_mgr=None,req_mgr=None,soup_manager=None,image_link_tags='img',img_link_attrs='src',link_tags='a',link_attrs='href',strict_order_tags=False,img_attr_value_desired=None,img_attr_value_undesired=None,link_attr_value_desired=None,link_attr_value_undesired=None,associated_data_attr=["data-title",'alt','title'],get_img=["data-title",'alt','title']):
-        self.url_mgr = get_url_mgr(url=url,url_mgr=url_mgr)
-        self.url=self.url_mgr.url
-        self.req_mgr = get_req_mgr(req_mgr=req_mgr,url=self.url,url_mgr=self.url_mgr,source_code=source_code)
-        self.source_code = source_code or self.req_mgr.source_code or self.req_mgr.source_code_bytes
-        self.soup_mgr = get_soup_mgr(req_mgr=self.req_mgr,url_mgr=self.url_mgr,source_code = source_code)
-
-        self.strict_order_tags=strict_order_tags
-        self.image_link_tags=image_link_tags
-        self.img_link_attrs=img_link_attrs
-        self.link_tags=link_tags
-        self.link_attrs=link_attrs
-        self.img_attr_value_desired=img_attr_value_desired
-        self.img_attr_value_undesired=img_attr_value_undesired
-        self.link_attr_value_desired=link_attr_value_desired
-        self.link_attr_value_undesired=link_attr_value_undesired
-        self.associated_data_attr=associated_data_attr
-        self.get_img=get_img
-        self.all_desired_image_links=self.find_all_desired_links(tag=self.image_link_tags,
-                                                                 attr=self.img_link_attrs,
-                                                                 attr_value_desired=self.img_attr_value_desired,
-                                                                 attr_value_undesired=self.img_attr_value_undesired)
-        self.all_desired_links=self.find_all_desired_links(tag=self.link_tags,
-                                                           attr=self.link_attrs,
-                                                           attr_value_desired=self.link_attr_value_desired,
-                                                           attr_value_undesired=self.link_attr_value_undesired,
-                                                           associated_data_attr=self.associated_data_attr,
-                                                           get_img=get_img)
-    def re_initialize(self):
-        self.all_desired_image_links=self.find_all_desired_links(tag=self.image_link_tags,attr=self.img_link_attrs,strict_order_tags=self.strict_order_tags,attr_value_desired=self.img_attr_value_desired,attr_value_undesired=self.img_attr_value_undesired)
-        self.all_desired_links=self.find_all_desired_links(tag=self.link_tags,attr=self.link_attrs,strict_order_tags=self.strict_order_tags,attr_value_desired=self.link_attr_value_desired,attr_value_undesired=self.link_attr_value_undesired,associated_data_attr=self.associated_data_attr,get_img=self.get_img)
-    def update_url_mgr(self,url_mgr):
-        self.url_mgr=url_mgr
-        self.url=self.url_mgr.url
-        self.req_mgr.update_url_mgr(url_mgr=self.url_mgr)
-        self.soup_mgr.update_url_mgr(url_mgr=self.url_mgr)
-        self.source_code=self.soup_mgr.source_code
-        self.re_initialize()
-    def update_url(self,url):
-        self.url=url
-        self.url_mgr.update_url(url=self.url)
-        self.url=self.url_mgr.url
-        self.req_mgr.update_url(url=self.url)
-        self.soup_mgr.update_url(url=self.url)
-        self.source_code=self.soup_mgr.source_code
-        self.re_initialize()
-    def update_source_code(self,source_code):
-        self.source_code=source_code
-        if self.source_code != self.soup_mgr.source_code:
-            self.soup_mgr.update_source_code(source_code=self.source_code)
-        self.re_initialize()
-    def update_soup_manager(self,soup_manager):
-        self.soup_mgr=soup_manager
-        self.source_code=self.soup_mgr.source_code
-        self.re_initialize()
-    def update_desired(self,img_attr_value_desired=None,img_attr_value_undesired=None,link_attr_value_desired=None,link_attr_value_undesired=None,image_link_tags=None,img_link_attrs=None,link_tags=None,link_attrs=None,strict_order_tags=None,associated_data_attr=None,get_img=None):
-       self.strict_order_tags = strict_order_tags or self.strict_order_tags
-       self.img_attr_value_desired=img_attr_value_desired or self.img_attr_value_desired
-       self.img_attr_value_undesired=img_attr_value_undesired or self.img_attr_value_undesired
-       self.link_attr_value_desired=link_attr_value_desired or self.link_attr_value_desired
-       self.link_attr_value_undesired=link_attr_value_undesired or self.link_attr_value_undesired
-       self.image_link_tags=image_link_tags or self.image_link_tags
-       self.img_link_attrs=img_link_attrs or self.img_link_attrs
-       self.link_tags=link_tags or self.link_tags
-       self.link_attrs=link_attrs or self.link_attrs
-       self.associated_data_attr=associated_data_attr or self.associated_data_attr
-       self.get_img=get_img or self.get_img
-       self.re_initialize()
-    def find_all_desired(self,tag='img',attr='src',strict_order_tags=False,attr_value_desired=None,attr_value_undesired=None,associated_data_attr=None,get_img=None):
-        def make_list(obj):
-            if isinstance(obj,list) or obj==None:
-                return obj
-            return [obj]
-        def get_desired_value(attr,attr_value_desired=None,attr_value_undesired=None):
-            if attr_value_desired:
-                for value in attr_value_desired:
-                    if value not in attr:
-                        return False
-            if attr_value_undesired:
-                for value in attr_value_undesired:
-                    if value in attr:
-                        return False
-            return True
-        attr_value_desired,attr_value_undesired,associated_data_attr,tags,attribs=make_list(attr_value_desired),make_list(attr_value_undesired),make_list(associated_data_attr),make_list(tag),make_list(attr)
-        desired_ls = []
-        assiciated_data=[]
-        for i,tag in enumerate(tags):
-            attribs_list=attribs
-            if strict_order_tags:
-                if len(attribs)<=i:
-                    attribs_list=[None]
-                else:
-                    attribs_list=make_list(attribs[i])
-            for attr in attribs_list:
-                for component in self.soup_mgr.soup.find_all(tag):
-                    if attr in component.attrs and get_desired_value(attr=component[attr],attr_value_desired=attr_value_desired,attr_value_undesired=attr_value_undesired):
-                        if component[attr] not in desired_ls:
-                            desired_ls.append(component[attr])
-                            assiciated_data.append({"value":component[attr]})
-                            if associated_data_attr:
-                                for data in associated_data_attr:
-                                    if data in component.attrs:
-                                        assiciated_data[-1][data]=component.attrs[data]
-                                        if get_img and component.attrs[data]:
-                                            if data in get_img and len(component.attrs[data])!=0:
-                                                for each in self.soup_mgr.soup.find_all('img'):
-                                                    if 'alt' in each.attrs:
-                                                        if each.attrs['alt'] == component.attrs[data] and 'src' in each.attrs:
-                                                            assiciated_data[-1]['image']=each.attrs['src']
-        desired_ls.append(assiciated_data)
-        return desired_ls
-    def find_all_domain(self):
-        domain = urlparse(self.url_mgr.url).netloc
-        domains_ls=[self.url_mgr.url]
-        for url in self.all_desired_links[:-1]:
-            if self.url_mgr.is_valid_url(url):
-                parse = urlparse(url)
-                comp_domain = parse.netloc
-                if url not in domains_ls and comp_domain == domain:
-                    domains_ls.append(url)
-        return domains_ls
-    def find_all_desired_links(self,tag='img', attr='src',attr_value_desired=None,strict_order_tags=False,attr_value_undesired=None,associated_data_attr=None,all_desired=None,get_img=None):
-        all_desired = all_desired or self.find_all_desired(tag=tag,attr=attr,strict_order_tags=strict_order_tags,attr_value_desired=attr_value_desired,attr_value_undesired=attr_value_undesired,associated_data_attr=associated_data_attr,get_img=get_img)
-        assiciated_attrs = all_desired[-1]
-        valid_assiciated_attrs = []
-        desired_links=[]
-        for i,attr in enumerate(all_desired[:-1]):
-            valid_attr=self.url_mgr.make_valid(attr,self.url_mgr.protocol+'://'+self.url_mgr.domain) 
-            if valid_attr:
-                desired_links.append(valid_attr)
-                valid_assiciated_attrs.append(assiciated_attrs[i])
-                valid_assiciated_attrs[-1]["link"]=valid_attr
-        desired_links.append(valid_assiciated_attrs)
-        return desired_links
-
-
-