abstract-webtools 0.1.6.119__tar.gz → 0.1.6.120__tar.gz

{abstract_webtools-0.1.6.119 → abstract_webtools-0.1.6.120}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: abstract_webtools
-Version: 0.1.6.119
+Version: 0.1.6.120
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff

{abstract_webtools-0.1.6.119 → abstract_webtools-0.1.6.120}/setup.py

@@ -4,7 +4,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 setuptools.setup(
     name='abstract_webtools',
-    version='0.1.6.119',
+    version='0.1.6.120',
     author='putkoff',
     author_email='partners@abstractendeavors.com',
     description='Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.',

{abstract_webtools-0.1.6.119 → abstract_webtools-0.1.6.120}/src/abstract_webtools/abstract_usurpit.py

@@ -53,7 +53,7 @@ def is_valid_url(url, base_domain):
     """
     parsed = urlparse(url)
     return parsed.scheme in ('http', 'https') and parsed.netloc == base_domain
-def save_page(url, content,output_dir):
+def get_save_page_path(url, output_dir):
     """
     Save HTML page to local directory.
     """
@@ -66,12 +66,22 @@ def save_page(url, content,output_dir):
         page_path += '.html'
 
     page_full_path = os.path.join(output_dir, page_path)
-    os.makedirs(os.path.dirname(page_full_path), exist_ok=True)
-
-    with open(page_full_path, 'w', encoding='utf-8') as f:
-        f.write(content)
-    print(f"Saved page: {page_full_path}")
-def save_asset(asset_url, base_url,output_dir,downloaded_assets=None,session=None):
+    return page_full_path
+def save_page(url, content,output_dir):
+    page_full_path = get_save_page_path(url=url,
+                                        output_dir=output_dir)
+    if page_full_path:
+        dirname = os.path.dirname(page_full_path)
+        os.makedirs(dirname, exist_ok=True)
+
+        with open(page_full_path, 'w', encoding='utf-8') as f:
+            f.write(content)
+        print(f"Saved page: {page_full_path}")
+def get_asset_path(asset_url,
+                   base_url,
+                   output_dir,
+                   downloaded_assets=None,
+                   session=None):
     """
     Download and save assets like images, CSS, JS files.
     """
@@ -88,17 +98,29 @@ def save_asset(asset_url, base_url,output_dir,downloaded_assets=None,session=Non
         return  # Skip if asset path is empty
 
     asset_full_path = os.path.join(output_dir, asset_path)
-    os.makedirs(os.path.dirname(asset_full_path), exist_ok=True)
-
-    try:
-        response = session.get(asset_url, stream=True)
-        response.raise_for_status()
-        with open(asset_full_path, 'wb') as f:
-            shutil.copyfileobj(response.raw, f)
-        print(f"Saved asset: {asset_full_path}")
-    except Exception as e:
-        print(f"Failed to save asset {asset_url}: {e}")
-    return downloaded_assets
+    return asset_full_path
+def save_asset(asset_url,
+               base_url,
+               output_dir,
+               downloaded_assets=None,
+               session=None):
+    asset_full_path = get_asset_path(asset_url=asset_url,
+                                     base_url=base_url,
+                                     output_dir=output_dir,
+                                     downloaded_assets=downloaded_assets,
+                                     session=session)
+    if asset_full_path:
+        os.makedirs(os.path.dirname(asset_full_path), exist_ok=True)
+
+        try:
+            response = session.get(asset_url, stream=True)
+            response.raise_for_status()
+            with open(asset_full_path, 'wb') as f:
+                shutil.copyfileobj(response.raw, f)
+            print(f"Saved asset: {asset_full_path}")
+        except Exception as e:
+            print(f"Failed to save asset {asset_url}: {e}")
+        return downloaded_assets
 class usurpManager():
     def __init__(self,url,output_dir=None,max_depth=None,wait_between_requests=None,operating_system=None, browser=None, version=None,user_agent=None,website_bot=None):
         self.url = url
@@ -135,34 +157,40 @@ class usurpManager():
             response = self.session.get(url)
             #response.raise_for_status()
             content = response.text
-
-            # Use your get_soup_mgr function to get the soup and attributes
-            soup_mgr = get_soup_mgr(url=url)
-            soup = soup_mgr.soup
-            all_attributes = soup_mgr.get_all_attribute_values()
-            # Now you can use all_attributes as needed
-
-            # Update asset links to local paths
-            for tag in soup.find_all(['img', 'script', 'link']):
-                attr = 'src' if tag.name != 'link' else 'href'
-                asset_url = tag.get(attr)
-                if asset_url:
-                    full_asset_url = normalize_url(asset_url, url)
-                    parsed_asset_url = urlparse(full_asset_url)
-
-                    if is_valid_url(full_asset_url, base_domain):
-                        self.downloaded_assets = save_asset(full_asset_url, self.url,self.OUTPUT_DIR,self.downloaded_assets,self.session)
-                        # Update tag to point to the local asset
-                        local_asset_path = '/' + parsed_asset_url.path.lstrip('/')
-                        tag[attr] = local_asset_path
-
-            # Save the modified page
-            save_page(url, str(soup),self.OUTPUT_DIR)
+            page_full_path = get_save_page_path(url=url,
+                                        output_dir=self.OUTPUT_DIR)
+            if not os.path.exists(page_full_path):
+                # Use your get_soup_mgr function to get the soup and attributes
+                soup_mgr = get_soup_mgr(url=url)
+                soup = soup_mgr.soup
+                all_attributes = soup_mgr.get_all_attribute_values()
+                # Now you can use all_attributes as needed
+                get_asset_path(asset_url=full_asset_url,
+                               base_url=self.url,
+                               output_dir=self.OUTPUT_DIR,
+                               downloaded_assets=self.downloaded_assets,
+                               session=self.session)
+                # Update asset links to local paths
+                for tag in soup.find_all(['img', 'script', 'link']):
+                    attr = 'src' if tag.name != 'link' else 'href'
+                    asset_url = tag.get(attr)
+                    if asset_url:
+                        full_asset_url = normalize_url(asset_url, url)
+                        parsed_asset_url = urlparse(full_asset_url)
+
+                        if is_valid_url(full_asset_url, base_domain):
+                            self.downloaded_assets = save_asset(full_asset_url, self.url,self.OUTPUT_DIR,self.downloaded_assets,self.session)
+                            # Update tag to point to the local asset
+                            local_asset_path = '/' + parsed_asset_url.path.lstrip('/')
+                            tag[attr] = local_asset_path
+
+                # Save the modified page
+                save_page(url, str(soup),self.OUTPUT_DIR)
 
             # Use your linkManager to find all domain links
             link_mgr = linkManager(url=url)
             all_domains = link_mgr.find_all_domain()
-
+            
             # Process each domain link
             for link_url in make_list(all_domains):
                 normalized_link = normalize_url(link_url, url)

{abstract_webtools-0.1.6.119 → abstract_webtools-0.1.6.120}/src/abstract_webtools.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: abstract_webtools
-Version: 0.1.6.119
+Version: 0.1.6.120
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff