abstract-webtools 0.1.6.117__tar.gz → 0.1.6.119__tar.gz

{abstract_webtools-0.1.6.117 → abstract_webtools-0.1.6.119}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: abstract_webtools
-Version: 0.1.6.117
+Version: 0.1.6.119
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff

{abstract_webtools-0.1.6.117 → abstract_webtools-0.1.6.119}/setup.py

@@ -4,7 +4,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 setuptools.setup(
     name='abstract_webtools',
-    version='0.1.6.117',
+    version='0.1.6.119',
     author='putkoff',
     author_email='partners@abstractendeavors.com',
     description='Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.',

{abstract_webtools-0.1.6.117 → abstract_webtools-0.1.6.119}/src/abstract_webtools/abstract_usurpit.py

@@ -9,13 +9,25 @@ from abstract_utilities import *
 
 # Import your custom classes/functions
 # from your_module import linkManager, get_soup_mgr
-def get_directory_from_url(url):
+def make_directory(directory=None,path=None):
+    if directory==None:
+        directory=os.getcwd()
+    if path:
+        directory = os.path.join(base_dir,path)
+    os.makedirs(directory,exist_ok=True)
+    return directory
+def get_domain_name_from_url(url):
     parsed_url = urlparse(url)
-    netloc = parsed_url['netloc']
-    directory_name = netloc.split('.')
+    netloc = parsed_url.netloc
+    parsed_spl = netloc.split('.')
+    directory_name = '.'.join(parsed_spl[:-1])
     if directory_name.startswith('www.'):
         directory_name = directory_name[len('www.'):]
     return directory_name
+def get_domain_directory_from_url(url,base_dir=None):
+    base_dir =base_dir or os.getcwd()
+    domain_name = get_domain_name_from_url(url)
+    return make_directory(directory,domain_name)
 # Configuration
 def normalize_url(url, base_url):
     """
@@ -176,7 +188,6 @@ def test_download(url=None,directory=None):
     site_mgr = usurpManager(url,output_dir)
     
 def usurpit(url,output_dir=None,max_depth=None,wait_between_requests=None,operating_system=None, browser=None, version=None,user_agent=None,website_bot=None):
-    output_dir= output_dir or get_directory_from_url(url) or os.path.join(os.getcwd(),'usurped')
-    os.makedirs(output_dir,exist_ok=True)
+    output_dir = get_domain_name_from_url(url) or  make_directory(path='usurped')
     site_mgr = usurpManager(url,output_dir=output_dir,max_depth=max_depth,wait_between_requests=wait_between_requests,operating_system=operating_system, browser=browser, version=version,user_agent=user_agent,website_bot=website_bot)
     site_mgr.main()

{abstract_webtools-0.1.6.117 → abstract_webtools-0.1.6.119}/src/abstract_webtools.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: abstract_webtools
-Version: 0.1.6.117
+Version: 0.1.6.119
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff