abstract-webtools 0.1.6.9__py3-none-any.whl → 0.1.6.11__py3-none-any.whl

abstract_webtools/managers/crawlManager.py

@@ -1,230 +1,150 @@
-from .soupManager import *
+from abstract_webtools import * #.soupManager import *
 
 
-class crawlManager():
-    def __init__(self,url=None,req_mgr=None,url_mgr=None,source_code=None,parse_type="html.parser"):
-        self.url=url
-        self.source_code=source_code
-        self.parse_type=parse_type
+from urllib.parse import urlparse, urljoin
+import os
+import xml.etree.ElementTree as ET
+from bs4 import BeautifulSoup
+
+class CrawlManager:
+    def __init__(self, url=None, req_mgr=None, url_mgr=None, source_code=None, parse_type="html.parser"):
+        self.url = url
+        self.source_code = source_code
+        self.parse_type = parse_type
         self.url_mgr = url_mgr or urlManager(url=self.url)
-        self.req_mgr = requestManager(url_mgr=self.url_mgr)
+        self.req_mgr = req_mgr or requestManager(url_mgr=self.url_mgr)
         self.get_new_source_and_url(url)
-    def get_new_source_and_url(self,url=None):
-        if url == None:
+
+    def get_new_source_and_url(self, url=None):
+        """Fetches new source code and response for a given URL."""
+        if url is None:
             url = self.url
+        self.req_mgr.set_url(url)
+        self.source_code = self.req_mgr.source_code
         self.response = self.req_mgr.response
-        self.source_code=self.req_mgr.source_code
-    def get_classes_and_meta_info():
-        class_name_1,class_name_2, class_value = 'meta','class','property','og:image'
-        attrs = 'href','src'
-        unique_classes, images=discover_classes_and_images(self,tag_name,class_name_1,class_name_2,class_value,attrs)
-        return unique_classes, images
-    def extract_links_from_url(self,url=None):
-        """
-        Extracts all href and src links from a given URL's source code.
 
-        Args:
-            base_url (str): The URL from which to extract links.
+    def get_classes_and_meta_info(self):
+        """Returns unique classes and image links from meta tags."""
+        tag_name = 'meta'
+        class_name_1, class_name_2 = 'class', 'property'
+        class_value = 'og:image'
+        attrs = ['href', 'src']
+        unique_classes, images = discover_classes_and_images(self, tag_name, class_name_1, class_name_2, class_value, attrs)
+        return unique_classes, images
 
-        Returns:
-            dict: Dictionary containing image links and external links under the parent page.
-        """
-        url = url or self.url
-        agg_js = {'images':[],'external_links':[]}
+    def extract_links_from_url(self, url=None):
+        """Extracts all href and src links from a given URL's source code."""
+        url = url or self.url_mgr.url
+        soup = BeautifulSoup(self.source_code, self.parse_type)
+        links = {'images': [], 'external_links': []}
+
+        if self.response:
+            for attr in ['href', 'src']:
+                for tag in soup.find_all(attrs={attr: True}):
+                    link = tag.get(attr)
+                    if link:
+                        absolute_link = urljoin(url, link)
+                        if link.lower().endswith(('.png', '.jpg', '.jpeg', '.gif', '.svg', '.webp')):
+                            links['images'].append(absolute_link)
+                        elif urlparse(absolute_link).netloc != urlparse(url).netloc:
+                            links['external_links'].append(absolute_link)
         
-        if self.response != None:
-            attrs = 'href','src'
-            href_links,src_links='',''
-            links = [href_links,src_links]
-            for i,each in enumerate(attrs):
-                 links[i]= [a[attr[i]] for a in get_find_all_with_attributes(self, attrs[i])]
-            # Convert all links to absolute links
-            absolute_links = [(url, link) for link in links[0] + links[1]]
-            # Separate images and external links
-            images = [link for link in absolute_links if link.lower().endswith(('.png', '.jpg', '.jpeg', '.gif', '.svg', '.webp'))]
-            external_links = [link for link in absolute_links if urlparse(link).netloc != urlparse(url).netloc]
-            agg_js['images']=images
-            agg_js['external_links']=external_links
-           
-        return agg_js
-    def get_all_website_links(self,tag="a",attr="href") -> list:
-        """
-        Returns all URLs that are found on the specified URL and belong to the same website.
-
-        Args:
-            url (str): The URL to search for links.
-
-        Returns:
-            list: A list of URLs that belong to the same website as the specified URL.
-        """
-        all_urls=[self.url_mgr.url]
+        return links
+
+    def get_all_website_links(self):
+        """Finds all internal links on the website that belong to the same domain."""
+        all_urls = [self.url_mgr.url]
         domain = self.url_mgr.domain
         all_attribs = get_all_attribute_values(self.url_mgr.url)
-        for href in all_attribs.get('href',[]):
-            if href == "" or href is None:
-                # href empty tag
-                continue
-            href=self.url_mgr.get_relative_href(self.url_mgr.url,href)
-            if not self.url_mgr.is_valid_url(href):
-                # not a valid URL
-                continue
-            if href in all_urls:
-                # already in the set
-                continue
-            if domain not in href:
-                # external link
+        
+        for href in all_attribs.get('href', []):
+            if not href or not self.url_mgr.is_valid_url(href):
                 continue
-            all_urls.append(href)
-                  
+            full_url = urljoin(self.url_mgr.url, href)
+            if domain in full_url and full_url not in all_urls:
+                all_urls.append(full_url)
+        
         return all_urls
 
-    def correct_xml(xml_string):
-        # Parse the XML string
+    def correct_xml(self, xml_string):
+        """Corrects XML by encoding special characters in <image:loc> tags."""
         root = ET.fromstring(xml_string)
-
-        # Loop through each <image:loc> element and correct its text if needed
         for image_loc in root.findall(".//image:loc", namespaces={'image': 'http://www.google.com/schemas/sitemap-image/1.1'}):
-            # Replace '&' with '&amp;' in the element's text
             if '&' in image_loc.text:
                 image_loc.text = image_loc.text.replace('&', '&amp;')
+        return ET.tostring(root, encoding='utf-8').decode('utf-8')
 
-        # Convert the corrected XML back to string
-        corrected_xml = ET.tostring(root, encoding='utf-8').decode('utf-8')
-        return corrected_xml
-
-
-    def determine_values(self,url=None):
-        # This is just a mockup. In a real application, you'd analyze the URL or its content.
+    def determine_values(self, url=None):
+        """Determines frequency and priority based on URL type."""
         url = url or self.url
-        # Assuming a blog site
         if 'blog' in url:
-            if '2023' in url:  # Assuming it's a current year article
-                return ('weekly', '0.8')
-            else:
-                return ('monthly', '0.6')
+            return ('weekly', '0.8') if '2023' in url else ('monthly', '0.6')
         elif 'contact' in url:
             return ('yearly', '0.3')
-        else:  # Homepage or main categories
-            return ('weekly', '1.0')
-    def crawl(self,url, max_depth=3, depth=1):
-        visited=set()
-        if depth > max_depth:
-            return []
+        return ('weekly', '1.0')
 
-        if url in visited:
+    def crawl(self, url, max_depth=3, depth=1, visited=None):
+        """Recursively crawls the site up to max_depth and returns valid internal links."""
+        visited = visited or set()
+        if depth > max_depth or url in visited:
             return []
 
         visited.add(url)
-
         try:
             soup = get_soup(url)
-            links = [a['href'] for a in soup.find_all('a', href=True)]
-            valid_links = []
-
-            for link in links:
-                parsed_link = urlparse(link)
-                base_url = "{}://{}".format(parsed_link.scheme, parsed_link.netloc)
-            
-                if base_url == url:  # Avoiding external URLs
-                    final_link = urljoin(url, parsed_link.path)
-                    if final_link not in valid_links:
-                        valid_links.append(final_link)
-
-            for link in valid_links:
-                crawl(link, max_depth, depth+1)
-
-            return valid_links
-
+            links = []
+            for tag in soup.find_all('a', href=True):
+                link = urljoin(url, tag['href'])
+                if urlparse(link).netloc == urlparse(url).netloc and link not in visited:
+                    links.append(link)
+                    self.crawl(link, max_depth, depth + 1, visited)
+            return links
         except Exception as e:
             print(f"Error crawling {url}: {e}")
             return []
 
-
-    # Define or import required functions here, like get_all_website_links, determine_values, 
-    # discover_classes_and_meta_images, and extract_links_from_url.
-    def get_meta_info(self,url=None):
+    def get_meta_info(self, url=None):
+        """Fetches metadata, including title and meta tags, from the page."""
         url = url or self.url
-        soup_mgr = soupManager(url=url)
-        meta_info = {}
-        # Fetch the title if available
-        meta_tags = soup_mgr.find_all("meta")
-        url = eatAll(str(url),['',' ','\n','\t','\\','/'])
-        attribs = get_all_attribute_values(url)
-        soup = get_soup(url)
+        soup = BeautifulSoup(self.source_code, self.parse_type)
+        meta_info = {"title": None, "meta_tags": {}}
         
-        for meta_tag in meta_tags:
-            for attr, values in meta_tag.attrs.items():
-                
-                if attr not in meta_tag:
-                    meta_tag[attr] = []
-                if values not in meta_tag[attr]:
-                    meta_tag[attr].append(values)
-        title_tag = soup.find_all("title")
+        title_tag = soup.find("title")
         if title_tag:
-            meta_info["title"] = title_tag
-        # Fetch meta tags
-        for meta_tag in soup.find_all('meta'):
-            name = meta_tag.get('name') or meta_tag.get('property')
-            if name:
-                content = meta_tag.get('content')
-                if content:
-                    meta_info[name] = content
+            meta_info["title"] = title_tag.text
+
+        for meta in soup.find_all('meta'):
+            name = meta.get('name') or meta.get('property')
+            content = meta.get('content')
+            if name and content:
+                meta_info["meta_tags"][name] = content
 
         return meta_info
-    def generate_sitemap(self,domain):
-        
+
+    def generate_sitemap(self):
+        """Generates a sitemap.xml file with URLs, images, change frequency, and priority."""
+        urls = self.get_all_website_links()
         with open('sitemap.xml', 'w', encoding='utf-8') as f:
-            string = '<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:image="http://www.google.com/schemas/sitemap-image/1.1">\n'
-            
-            for url in self.get_all_website_links():
-                string += f'  <url>\n    <loc>{url}</loc>\n'
-                preprocess=[]
-                self.get_new_source_and_url(url=url)
-                links = get_all_attribute_values(url)
-                images = [link for link in links if link.lower().endswith(('.png', '.jpg', '.jpeg', '.gif', '.svg', '.webp'))]
+            f.write('<?xml version="1.0" encoding="UTF-8"?>\n')
+            f.write('<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" ')
+            f.write('xmlns:image="http://www.google.com/schemas/sitemap-image/1.1">\n')
 
-                for img in images:
-                    if str(img).lower() not in preprocess:
-                        try:
-                            escaped_img = img.replace('&', '&amp;')
-
-                            str_write = f'    <image:image>\n      <image:loc>{escaped_img}</image:loc>\n    </image:image>\n'
-                            string += str_write
-                        except:
-                            pass
-                        preprocess.append(str(img).lower())
+            for url in urls:
+                f.write(f'  <url>\n    <loc>{url}</loc>\n')
                 frequency, priority = self.determine_values(url)
-                string += f'    <changefreq>{frequency}</changefreq>\n'
-                string += f'    <priority>{priority}</priority>\n'
-                string += f'  </url>\n'
-                
-            string += '</urlset>\n'
-            f.write(string)            
-        # Output summary
-        print(f'Sitemap saved to sitemap.xml with {len(urls)} URLs.')
+                f.write(f'    <changefreq>{frequency}</changefreq>\n')
+                f.write(f'    <priority>{priority}</priority>\n')
+
+                images = [img for img in self.extract_links_from_url(url)['images']]
+                for img in images:
+                    escaped_img = img.replace('&', '&amp;')
+                    f.write(f'    <image:image>\n      <image:loc>{escaped_img}</image:loc>\n    </image:image>\n')
+
+                f.write('  </url>\n')
+
+            f.write('</urlset>\n')
         
-        # Output class and link details
-        for url in urls:
-            print(f"\nDetails for {url}:")
-            classes, meta_img_refs = discover_classes_and_meta_images(url)
-
-            print("\nClasses with href or src attributes:")
-            for class_name in classes:
-                print(f"\t{class_name}")
-            
-            print("\nMeta Image References:")
-            for img_ref in meta_img_refs:
-                print(f"\t{img_ref}")
-            
-            links = extract_links_from_url(url)
-
-            print("\nImages:")
-            for img in links['images']:
-                print(f"\t{img}")
-            
-            print("\nExternal Links:")
-            for ext_link in links['external_links']:
-                print(f"\t{ext_link}")
+        print(f'Sitemap saved to sitemap.xml with {len(urls)} URLs.')
 
 class crawlManagerSingleton():
     _instance = None
@@ -236,8 +156,9 @@ class crawlManagerSingleton():
             crawlManagerSingleton._instance = CrawlManager(url=url,parse_type=parse_type,source_code=source_code)
         return crawlManagerSingleton._instance
 def get_crawl_mgr(url=None,req_mgr=None,url_mgr=None,source_code=None,parse_type="html.parser"):
+    
+    url_mgr = get_url_mgr(url=url,url_mgr=url_mgr)
     url = get_url(url=url,url_mgr=url_mgr)
-    url_mgr = get_url(url=url,url_mgr=url_mgr)
     req_mgr=get_req_mgr(url=url,url_mgr=url_mgr,source_code=source_code)
     source_code = get_source(url=url,url_mgr=url_mgr,source_code=source_code,req_mgr=req_mgr)
     soup_mgr = get_soup_mgr(url=url,url_mgr=url_mgr,source_code=source_code,req_mgr=req_mgr,parse_type=parse_type)
@@ -248,3 +169,4 @@ def get_domain_crawl(url=None,req_mgr=None,url_mgr=None,source_code=None,parse_t
     url = get_url(url=url,url_mgr=url_mgr)
     all_domain_links = crawl_mgr.crawl(url=url, max_depth=max_depth, depth=depth)
     return all_domain_links
+get_domain_crawl(url='https://www.tradingview.com/symbols/SOLUSD/')

abstract_webtools/managers/requestManager.py

@@ -87,7 +87,8 @@ class requestManager:
                  timeout = None,
                  last_request_time=None,
                  max_retries=None,
-                 request_wait_limit=None):
+                 request_wait_limit=
+                 None):
         self.url_mgr = get_url_mgr(url=url,url_mgr=url_mgr)
         self.url=get_url(url=url,url_mgr=self.url_mgr)      
         self._url_mgr = self.url_mgr

{abstract_webtools-0.1.6.9.dist-info → abstract_webtools-0.1.6.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: abstract_webtools
-Version: 0.1.6.9
+Version: 0.1.6.11
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff

{abstract_webtools-0.1.6.9.dist-info → abstract_webtools-0.1.6.11.dist-info}/RECORD

@@ -7,13 +7,13 @@ abstract_webtools/url_grabber.py,sha256=pnCCev7ZIuM-6cAGTLmK5HfzZg_AX-fLcRpB6ZE7
 abstract_webtools/url_grabber_new.py,sha256=Oh2Kc0gBScCo0xpopNsg8JE5lIbPuzZVKM5f5GoZmw0,3454
 abstract_webtools/managers/__init__.py,sha256=5aIpbdUsDWTrhPUAjfIKnG54OULqOKan9LBL5EIUllo,407
 abstract_webtools/managers/cipherManager.py,sha256=NHQGdR11eNSm-1H-GezD5dyQgsPTJwY5kczt8Sher2s,1621
-abstract_webtools/managers/crawlManager.py,sha256=thImbK4ojRxEqF868Apox4-pJ85BuYR_u_iN51cxuEI,10684
+abstract_webtools/managers/crawlManager.py,sha256=ZcCDw_i8OEvfAyRvNu-Krx3kTigrFAFO4ZKMOhiXd9o,7967
 abstract_webtools/managers/domainManager.py,sha256=95znOBv05W77mW_fbZAfl4RmlENDlYqhEOMkL02L220,3610
 abstract_webtools/managers/dynamicRateLimiter.py,sha256=gopQcQo50JG2D0KcyepNCIQ_1uDQEBIHBzWf4R2Wgy0,7617
 abstract_webtools/managers/linkManager.py,sha256=m6y9s8jknrTX8RtOAFKeHd4yd23G7Rgf0T7Sp7wmHUw,12180
 abstract_webtools/managers/mySocketClient.py,sha256=-j1Q8Ds9RCSbjZdx3ZF9mVpgwxaO0BBssanUcpYVQoY,2045
 abstract_webtools/managers/networkManager.py,sha256=Op2QDXrP-gmm0tCToe-Ryt9xuOtMppcN2KLKP1WZiu0,952
-abstract_webtools/managers/requestManager.py,sha256=YksYgRivMMuZNOzyL5vaoXv9MLhgYeuLxO_UJiqPGWw,17312
+abstract_webtools/managers/requestManager.py,sha256=ko07C9igeTr_KtfeijaO126WRVyDw6jkvJhBHlFdwho,17330
 abstract_webtools/managers/seleniumManager.py,sha256=qSY8gH3N5YJIMwE_Alj9HNQRip_PziIo4_T9AZE_FQo,4273
 abstract_webtools/managers/soupManager.py,sha256=7nDB_QKneGjyTZUzchfbdHNvxxYiTyIn8AHon8ObTSY,17148
 abstract_webtools/managers/sslManager.py,sha256=C-QgQw9CW84uOE5kx2MPjC3RsLbE2JQqdwdTs0H4ecc,1370
@@ -21,8 +21,8 @@ abstract_webtools/managers/tlsAdapter.py,sha256=XZSMZz9EUOhv-h3_Waf6mjV1dA3oN_M_
 abstract_webtools/managers/urlManager.py,sha256=XqMrCM84BeWEfWtHc_8UFpT91ZtG-okzdKdCuC49vsA,8678
 abstract_webtools/managers/userAgentManager.py,sha256=33SB2p2FG7EYZl7l2iYm1U4gI9PcdkGTZHw5lg_Ogrw,1653
 abstract_webtools/managers/videoDownloader.py,sha256=6G_aLc05BTMUYUWc7iqYtHF_BaR7DnCNK_NJ-QnjsYY,10531
-abstract_webtools-0.1.6.9.dist-info/LICENSE,sha256=g3WEJFiVS27HyCGRTwKSsMLyciMaGFdWcZGOe1QalZk,3877
-abstract_webtools-0.1.6.9.dist-info/METADATA,sha256=zOphffSFpsOcKSPWQfx7PAs45qkCu_xgCpX8-1aO8P0,15857
-abstract_webtools-0.1.6.9.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
-abstract_webtools-0.1.6.9.dist-info/top_level.txt,sha256=2DMJ7RmjTcjCsa-uwAV0K6eXXlIIkFDEjBLg_uyCmCI,18
-abstract_webtools-0.1.6.9.dist-info/RECORD,,
+abstract_webtools-0.1.6.11.dist-info/LICENSE,sha256=g3WEJFiVS27HyCGRTwKSsMLyciMaGFdWcZGOe1QalZk,3877
+abstract_webtools-0.1.6.11.dist-info/METADATA,sha256=X43t8XSI9MPYjaEZQqgCrWgMpty6ZSyrjm317YE4VQ4,15858
+abstract_webtools-0.1.6.11.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
+abstract_webtools-0.1.6.11.dist-info/top_level.txt,sha256=2DMJ7RmjTcjCsa-uwAV0K6eXXlIIkFDEjBLg_uyCmCI,18
+abstract_webtools-0.1.6.11.dist-info/RECORD,,