abstract-webtools 0.1.5.86__tar.gz → 0.1.5.88__tar.gz

{abstract_webtools-0.1.5.86/src/abstract_webtools.egg-info → abstract_webtools-0.1.5.88}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: abstract_webtools
-Version: 0.1.5.86
+Version: 0.1.5.88
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff

{abstract_webtools-0.1.5.86 → abstract_webtools-0.1.5.88}/setup.py

@@ -4,7 +4,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 setuptools.setup(
     name='abstract_webtools',
-    version='0.1.5.86',
+    version='0.1.5.88',
     author='putkoff',
     author_email='partners@abstractendeavors.com',
     description='Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.',

{abstract_webtools-0.1.5.86 → abstract_webtools-0.1.5.88}/src/abstract_webtools/managers/crawlManager.py

@@ -1,14 +1,17 @@
-class CrawlManager:
-    def __init__(self,url=None,source_code=None,parse_type="html.parser"):
+from .soupManager import *
+class CrawlManager():
+    def __init__(self,url=None,req_mgr=None,url_mgr=None,source_code=None,parse_type="html.parser"):
         self.url=url
         self.source_code=source_code
         self.parse_type=parse_type
-        get_new_source_and_url(self,url)
+        self.url_mgr = url_mgr or urlManager(url=self.url)
+        self.req_mgr = requestManager(url_mgr=self.url_mgr)
+        self.get_new_source_and_url(url)
     def get_new_source_and_url(self,url=None):
         if url == None:
             url = self.url
-        self.response = self.request_mgr.response
-        self.source_code=self.request_mgr.source_code
+        self.response = self.req_mgr.response
+        self.source_code=self.req_mgr.source_code
     def get_classes_and_meta_info():
         class_name_1,class_name_2, class_value = 'meta','class','property','og:image'
         attrs = 'href','src'
@@ -41,7 +44,36 @@ class CrawlManager:
             agg_js['external_links']=external_links
            
         return agg_js
+    def get_all_website_links(self,tag="a",attr="href") -> list:
+        """
+        Returns all URLs that are found on the specified URL and belong to the same website.
 
+        Args:
+            url (str): The URL to search for links.
+
+        Returns:
+            list: A list of URLs that belong to the same website as the specified URL.
+        """
+        all_urls=[self.url_mgr.url]
+        domain = self.url_mgr.domain
+        all_attribs = get_attribs(self.url_mgr.url)
+        for href in all_attribs.get('href',[]):
+            if href == "" or href is None:
+                # href empty tag
+                continue
+            href=self.url_mgr.get_relative_href(self.url_mgr.url,href)
+            if not self.url_mgr.is_valid_url(href):
+                # not a valid URL
+                continue
+            if href in all_urls:
+                # already in the set
+                continue
+            if domain not in href:
+                # external link
+                continue
+            all_urls.append(href)
+                  
+        return all_urls
 
     def correct_xml(xml_string):
         # Parse the XML string
@@ -58,21 +90,21 @@ class CrawlManager:
         return corrected_xml
 
 
-    def determine_values(self):
+    def determine_values(self,url=None):
         # This is just a mockup. In a real application, you'd analyze the URL or its content.
-
+        url = url or self.url
         # Assuming a blog site
-        if 'blog' in self.url:
-            if '2023' in self.url:  # Assuming it's a current year article
+        if 'blog' in url:
+            if '2023' in url:  # Assuming it's a current year article
                 return ('weekly', '0.8')
             else:
                 return ('monthly', '0.6')
-        elif 'contact' in self.url:
+        elif 'contact' in url:
             return ('yearly', '0.3')
         else:  # Homepage or main categories
             return ('weekly', '1.0')
-    def crawl(url, max_depth=3, depth=1):
-        
+    def crawl(self,url, max_depth=3, depth=1):
+        visited=set()
         if depth > max_depth:
             return []
 
@@ -82,8 +114,8 @@ class CrawlManager:
         visited.add(url)
 
         try:
-            
-            links = [a['href'] for a in self.soup.find_all('a', href=True)]
+            soup = get_soup(url)
+            links = [a['href'] for a in soup.find_all('a', href=True)]
             valid_links = []
 
             for link in links:
@@ -107,11 +139,24 @@ class CrawlManager:
 
     # Define or import required functions here, like get_all_website_links, determine_values, 
     # discover_classes_and_meta_images, and extract_links_from_url.
-    def get_meta_info(self):
-        
+    def get_meta_info(self,url=None):
+        url = url or self.url
+        soup_mgr = soupManager(url=url)
         meta_info = {}
         # Fetch the title if available
-        title_tag = parse_title()
+        meta_tags = soup_mgr.find_all("meta")
+        url = eatAll(str(url),['',' ','\n','\t','\\','/'])
+        attribs = get_attribs(url)
+        soup = get_soup(url)
+        
+        for meta_tag in meta_tags:
+            for attr, values in meta_tag.attrs.items():
+                
+                if attr not in meta_tag:
+                    meta_tag[attr] = []
+                if values not in meta_tag[attr]:
+                    meta_tag[attr].append(values)
+        title_tag = soup.find_all("title")
         if title_tag:
             meta_info["title"] = title_tag
         # Fetch meta tags
@@ -128,13 +173,14 @@ class CrawlManager:
         with open('sitemap.xml', 'w', encoding='utf-8') as f:
             string = '<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:image="http://www.google.com/schemas/sitemap-image/1.1">\n'
             
-            for url in self.all_site_links:
+            for url in self.get_all_website_links():
                 string += f'  <url>\n    <loc>{url}</loc>\n'
                 preprocess=[]
                 self.get_new_source_and_url(url=url)
-                links = extract_links_from_url(url)
-                
-                for img in links['images']:
+                links = get_attribs(url)
+                images = [link for link in links if link.lower().endswith(('.png', '.jpg', '.jpeg', '.gif', '.svg', '.webp'))]
+
+                for img in images:
                     if str(img).lower() not in preprocess:
                         try:
                             escaped_img = img.replace('&', '&amp;')
@@ -144,7 +190,7 @@ class CrawlManager:
                         except:
                             pass
                         preprocess.append(str(img).lower())
-                frequency, priority = determine_values(url)
+                frequency, priority = self.determine_values(url)
                 string += f'    <changefreq>{frequency}</changefreq>\n'
                 string += f'    <priority>{priority}</priority>\n'
                 string += f'  </url>\n'
@@ -176,6 +222,7 @@ class CrawlManager:
             print("\nExternal Links:")
             for ext_link in links['external_links']:
                 print(f"\t{ext_link}")
+
 class CrawlManagerSingleton():
     _instance = None
     @staticmethod
@@ -185,3 +232,16 @@ class CrawlManagerSingleton():
         elif parse_type != CrawlManagerSingleton._instance.parse_type or url != CrawlManagerSingleton._instance.url  or source_code != CrawlManagerSingleton._instance.source_code:
             CrawlManagerSingleton._instance = CrawlManager(url=url,parse_type=parse_type,source_code=source_code)
         return CrawlManagerSingleton._instance
+def get_crawl_mgr(url=None,req_mgr=None,url_mgr=None,source_code=None,parse_type="html.parser"):
+    url = get_url(url=url,url_mgr=url_mgr)
+    url_mgr = get_url(url=url,url_mgr=url_mgr)
+    req_mgr=get_req_mgr(url=url,url_mgr=url_mgr,source_code=source_code)
+    source_code = get_source(url=url,url_mgr=url_mgr,source_code=source_code,req_mgr=req_mgr)
+    soup_mgr = get_soup_mgr(url=url,url_mgr=url_mgr,source_code=source_code,req_mgr=req_mgr,parse_type=parse_type)
+    crawl_mgr = CrawlManager(url=url,req_mgr=req_mgr,url_mgr=url_mgr,source_code=source_code,parse_type=parse_type)
+    return crawl_mgr
+def get_domain_crawl(url=None,req_mgr=None,url_mgr=None,source_code=None,parse_type="html.parser",max_depth=3, depth=1):
+    crawl_mgr = get_crawl_mgr(url=url,req_mgr=req_mgr,url_mgr=url_mgr,source_code=source_code,parse_type=parse_type)
+    url = get_url(url=url,url_mgr=url_mgr)
+    all_domain_links = crawl_mgr.crawl(url=url, max_depth=max_depth, depth=depth)
+    return all_domain_links

{abstract_webtools-0.1.5.86 → abstract_webtools-0.1.5.88}/src/abstract_webtools/managers/requestManager.py

@@ -343,6 +343,11 @@ class SafeRequestSingleton:
 def get_req_mgr(req_mgr=None,url=None,url_mgr=None,source_code=None):
     if req_mgr:
         url_mgr = req_mgr.url_mgr
+    url = get_url(url=url,url_mgr=url_mgr)
     url_mgr = get_url_mgr(url=url,url_mgr=url_mgr)
-    req_mgr = req_mgr or requestManager(url_mgr=url_mgr,source_code=source_code)
+    req_mgr = req_mgr or requestManager(url_mgr=url_mgr,url=url,source_code=source_code)
     return req_mgr
+def get_source(url=None,url_mgr=None,source_code=None):
+    # Placeholder for actual implementation.
+    req_mgr = get_req_mgr(req_mgr=req_mgr,url=url,url_mgr=url_mgr,source_code=source_code)
+    return req_mgr.source_code

{abstract_webtools-0.1.5.86 → abstract_webtools-0.1.5.88}/src/abstract_webtools/managers/seleniumManager.py

@@ -60,9 +60,10 @@ class domainManager(metaclass=SingletonMeta):
             for path in paths[:-1]:
                 dir_path = os.path.join(dir_path, path)
                 os.makedirs(dir_path, exist_ok=True)
-        self.page_type.append(os.path.splitext(paths[-1])[-1] or 'html' if len(self.page_type) == 0 else self.page_type[-1])
-        dir_path = os.path.join(dir_path, paths[-1])
-        return dir_path
+            self.page_type.append(os.path.splitext(paths[-1])[-1] or 'html' if len(self.page_type) == 0 else self.page_type[-1])
+            
+            dir_path = os.path.join(dir_path, paths[-1])
+            return dir_path
 
     def saved_url_check(self, url):
         path = self.get_url_to_path(url)

{abstract_webtools-0.1.5.86 → abstract_webtools-0.1.5.88}/src/abstract_webtools/managers/soupManager.py

@@ -51,6 +51,11 @@ class soupManager:
             source_code = str(source_code)
         self.source_code = source_code
         self.soup= BeautifulSoup(self.source_code, self.parse_type)
+        self.all_tags_and_attribute_names = self.get_all_tags_and_attribute_names()
+        self.all_tags = self.all_tags_and_attribute_names.get('tags')
+        self.all_attribute_names = self.all_tags_and_attribute_names.get('attributes')
+        self.all_tags_and_attributes = self.all_tags + self.all_attribute_names
+        
         self._all_links_data = None
         self._meta_tags_data = None
     def re_initialize(self):
@@ -106,7 +111,7 @@ class soupManager:
         """
         all_urls=[self.url_mgr.url]
         domain = self.url_mgr.domain
-        all_desired=self.get_all_desired_soup(tag=tag,attr=attr)
+        all_desired=self.get_all_website_links(tag=tag,attr=attr)
         for tag in all_desired:
             href = tag.attrs.get(attr)
             if href == "" or href is None:
@@ -274,22 +279,50 @@ class soupManager:
         attribute_names_list = list(attribute_names)
         return {"tags":tag_names_list,"attributes":attribute_names_list}
 
-    def get_all_attribute_values(self):
-        attribute_values={}
-        get_all = self.find_tags_by_attributes()
-        for tag in get_all:  # True matches all tags
-            for attr, value in tag.attrs.items():
-                # If attribute is not yet in the dictionary, add it with an empty set
-                if attr not in attribute_values:
-                    attribute_values[attr] = set()
-                # If the attribute value is a list (e.g., class), extend the set with the list
-                if isinstance(value, list):
-                    attribute_values[attr].update(value)
-                else:
-                    attribute_values[attr].add(value)
+    def get_all_attribute_values(self, tag_list=None):
+        """
+        Collects all attribute values for each specified tag or all tags if none are specified.
+        
+        Parameters:
+        - tag_list: List of specific tags to retrieve attributes from, e.g., ['script', 'img'].
+                    If None, retrieves attributes for all tags.
+        
+        Returns:
+        - attribute_values: Dictionary where each key is an attribute and the value is a list of unique values for that attribute.
+        """
+        attribute_values = {}
+        tag_list = tag_list or self.all_tags_and_attributes
+        # Get all tags matching tag_list criteria
+        for tag_name in tag_list:
+            for tag in self.soup.find_all(tag_name):
+                for attr, value in tag.attrs.items():
+                    if attr not in attribute_values:
+                        attribute_values[attr] = set()
+                    
+                    # Add attribute values
+                    if isinstance(value, list):
+                        attribute_values[attr].update(value)
+                    else:
+                        attribute_values[attr].add(value)
+        
+        # Convert each set to a list for consistency
         for attr, values in attribute_values.items():
             attribute_values[attr] = list(values)
+
+        # Capture JavaScript URLs inside <script> tags
+        attribute_values['script_links'] = self.get_js_links()
+
         return attribute_values
+
+    def get_js_links(self):
+        """Extract URLs embedded in JavaScript within <script> tags."""
+        js_links = []
+        script_tags = self.soup.find_all('script')
+        for script in script_tags:
+            # Find URLs in the JavaScript code
+            urls_in_js = re.findall(r'["\'](https?://[^"\']+|/[^"\']+)["\']', script.get_text())
+            js_links.extend(urls_in_js)
+        return list(set(js_links))  # Remove duplicates
     
     @property
     def url(self):
@@ -309,6 +342,10 @@ class SoupManagerSingleton():
         return SoupManagerSingleton._instance
 def get_soup_mgr(url=None,url_mgr=None,source_code=None,req_mgr=None,soup_mgr=None):
     url_mgr = get_url_mgr(url=url,url_mgr=url_mgr)
-    req_mgr = get_req_mgr(url_mgr=url_mgr,source_code=source_code)
-    soup_mgr = soup_mgr or soupManager(url_mgr=url_mgr,req_mgr=req_mgr)
+    url = get_url(url=url,url_mgr=url_mgr)
+    req_mgr = get_req_mgr(url_mgr=url_mgr,url=url,source_code=source_code)
+    soup_mgr = soup_mgr or soupManager(url_mgr=url_mgr,req_mgr=req_mgr,url=url,source_code=source_code)
     return soup_mgr
+def get_all_attribute_values(url=None,url_mgr=None,source_code=None,req_mgr=None,soup_mgr=None,tags_list = None):
+    soup_mgr = get_soup_mgr(url=url,url_mgr=url_mgr,source_code=source_code,req_mgr=req_mgr,soup_mgr=soup_mgr)
+    return soup_mgr.get_all_attribute_values(tags_list=tags_list)

{abstract_webtools-0.1.5.86 → abstract_webtools-0.1.5.88}/src/abstract_webtools/managers/urlManager.py

@@ -218,8 +218,8 @@ def get_url(url=None,url_mgr=None):
         url_mgr = urlManager(url)
     return url_mgr.url
 def get_url_mgr(url=None,url_mgr=None):
-    if not url and not url_mgr:
-        return None
-    if url:
-        url_mgr = urlManager(url)
+    if url_mgr == None and url:
+         url_mgr = urlManager(url=url)
+    if url_mgr and url == None:
+        url = url_mgr.url
     return url_mgr 

{abstract_webtools-0.1.5.86 → abstract_webtools-0.1.5.88}/src/abstract_webtools/url_grabber.py

@@ -25,7 +25,7 @@ def get_attrs(values):
   else:
     tags_js['attribute']=tags_js['attribute'][0]
   return tags_js
-def get_user_agent_manager(user_agent=None):
+def get_user_agent_mgr(user_agent=None):
   return UserAgentManager(user_agent=user_agent)
 def get_cipher_list():
   return CipherManager().get_default_ciphers()
@@ -128,78 +128,78 @@ def process_url(window,values):
     if warn_url=='' or warn_url == None:
       update_status(window,warn,warn_url,response_code,valid)
       return False
-    temp_url=UrlManager(url=warn_url).url
+    temp_url=urlManager(url=warn_url).url
     if temp_url:
       valid='valid'
-      response_code = SafeRequest(url=temp_mgr).response.status_code
+      response_code = requestManager(url=temp_mgr).response.status_code
       warn = 'success'
       warn_url = temp_mgr
       update_status(window,warn,warn_url,response_code,valid)
       return temp_mgr
     update_status(window,warn,warn_url,response_code,valid)
     return False
-def update_url(url_manager,request_manager,soup_manager,link_manager,values,cipher_list=get_cipher_list(),user_agent=get_user_agents()[0]):
+def update_url(url_mgr,request_mgr,soup_mgr,link_mgr,values,cipher_list=get_cipher_list(),user_agent=get_user_agents()[0]):
       ciphers = CipherManager(cipher_list=cipher_list).ciphers_string
-      request_manager = SafeRequest(url_manager=url_manager,ciphers=ciphers,user_agent=get_user_agents()[0])
-      if request_manager.source_code:
-        soup_manager= SoupManager(url_manager=url_manager,request_manager=request_manager)
-        link_manager= LinkManager(url_manager=url_manager,request_manager=request_manager,soup_manager=soup_manager)
-        window['-URL-'].update(value=url_manager.url)
-        window['-CIPHERS_OUTPUT-'].update(value=request_manager.ciphers)
-        return update_source_code(url_manager,request_manager,soup_manager,link_manager,values)
+      request_mgr = requestManager(url_mgr=url_mgr,ciphers=ciphers,user_agent=get_user_agents()[0])
+      if request_mgr.source_code:
+        soup_mgr= SoupManager(url_mgr=url_mgr,request_mgr=request_mgr)
+        link_mgr= LinkManager(url_mgr=url_mgr,request_mgr=request_mgr,soup_mgr=soup_mgr)
+        window['-URL-'].update(value=url_mgr.url)
+        window['-CIPHERS_OUTPUT-'].update(value=request_mgr.ciphers)
+        return update_source_code(url_mgr,request_mgr,soup_mgr,link_mgr,values)
       else:
-        return url_manager,request_manager,soup_manager,link_manager
-def update_source_code(url_manager,request_manager,soup_manager,link_manager,values):
+        return url_mgr,request_mgr,soup_mgr,link_mgr
+def update_source_code(url_mgr,request_mgr,soup_mgr,link_mgr,values):
     parse_type = values['-parse_type-']
-    if parse_type != soup_manager.parse_type:
-      soup_manager.update_parse_type(parse_type=parse_type)
-    all_tags=soup_manager.get_all_tags_and_attribute_names()
-    window['-SOURCECODE-'].update(value=soup_manager.soup)
+    if parse_type != soup_mgr.parse_type:
+      soup_mgr.update_parse_type(parse_type=parse_type)
+    all_tags=soup_mgr.get_all_tags_and_attribute_names()
+    window['-SOURCECODE-'].update(value=soup_mgr.soup)
     if values['-SOUP_TAG-'] != all_tags['tags']:
       window['-SOUP_TAG-'].update(values=all_tags['tags'],value=all_tags['tags'][0])
     if values['-SOUP_ATTRIBUTE-'] != all_tags['attributes']:
       window['-SOUP_ATTRIBUTE-'].update(values=all_tags['attributes'],value=all_tags['attributes'][0])
       window['-SOUP_ATTRIBUTE_1-'].update(values=all_tags['attributes'],value=all_tags['attributes'][0])
       window['-SOUP_ATTRIBUTE_2-'].update(values=all_tags['attributes'],value=all_tags['attributes'][0])
-      return url_manager,request_manager,soup_manager,link_manager
+      return url_mgr,request_mgr,soup_mgr,link_mgr
 def url_grabber_while(window,initial_url="www.example.com"):
     return_data=None
     url_grab = False
-    url_manager=UrlManager(url=initial_url)
-    request_manager = SafeRequest(url_manager=url_manager)
-    soup_manager= SoupManager(url_manager=url_manager,request_manager=request_manager)
-    link_manager= LinkManager(url_manager=url_manager,request_manager=request_manager,soup_manager=soup_manager)
+    url_mgr=urlManager(url=initial_url)
+    request_mgr = requestManager(url_mgr=url_mgr)
+    soup_mgr= SoupManager(url_mgr=url_mgr,request_mgr=request_mgr)
+    link_mgr= LinkManager(url_mgr=url_mgr,request_mgr=request_mgr,soup_mgr=soup_mgr)
     while True:
         event, values = window.read()
         if event == sg.WINDOW_CLOSED:
             break
         if event=='-GRAB_URL-' or not url_grab:
           url=values['-URL-']
-          if UrlManager(url=url).url:
-            if url != url_manager.url or url == initial_url:
-              url_manager = UrlManager(url=url)
+          if urlManager(url=url).url:
+            if url != url_mgr.url or url == initial_url:
+              url_mgr = urlManager(url=url)
               
-              url_manager,request_manager,soup_manager,link_manager=update_url(url_manager=url_manager,request_manager=request_manager,soup_manager=soup_manager,link_manager=link_manager,values=values)
-              window['-URL-'].update(value=url_manager.url)
+              url_mgr,request_mgr,soup_mgr,link_mgr=update_url(url_mgr=url_mgr,request_mgr=request_mgr,soup_mgr=soup_mgr,link_mgr=link_mgr,values=values)
+              window['-URL-'].update(value=url_mgr.url)
               url_grab=True
         if event == 'get soup':
             tags_js = get_attrs(values)
-            all_desired=soup_manager.find_tags_by_attributes(tag=tags_js['tag'], attr=tags_js['attribute'],attr_values=tags_js['input'])
+            all_desired=soup_mgr.find_tags_by_attributes(tag=tags_js['tag'], attr=tags_js['attribute'],attr_values=tags_js['input'])
             window['-FIND_ALL_OUTPUT-'].update(value=all_desired)
         if event == '-CUSTOMUA-':
             window['-SOURCECODE-'].update(disabled=values['-CUSTOMUA-'])
             if not values['-CUSTOMUA-']:
-                window['-USERAGENT-'].update(value=user_agent_manager.user_agent_header)
+                window['-USERAGENT-'].update(value=user_agent_mgr.user_agent_header)
                 window['-USERAGENT-'].update(disabled=True)
             else:
                 window['-USERAGENT-'].update(disabled=False)
         if event=='Get All Text':
-            window['-FIND_ALL_OUTPUT-'].update(value=soup_manager.extract_text_sections())
+            window['-FIND_ALL_OUTPUT-'].update(value=soup_mgr.extract_text_sections())
         if event == 'Action':
             parse_type = values['-parse_type-']
-            if parse_type != soup_manager.parse_type:
-              soup_manager.update_parse_type(parse_type=parse_type)
-            window['-SOURCECODE-'].update(value=soup_manager.soup)
+            if parse_type != soup_mgr.parse_type:
+              soup_mgr.update_parse_type(parse_type=parse_type)
+            window['-SOURCECODE-'].update(value=soup_mgr.soup)
         elif event == 'Send Soup':
           return_data = values['-FIND_ALL_OUTPUT-']
           break

{abstract_webtools-0.1.5.86 → abstract_webtools-0.1.5.88/src/abstract_webtools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: abstract_webtools
-Version: 0.1.5.86
+Version: 0.1.5.88
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff