abstract-webtools 0.1.6.3__tar.gz → 0.1.6.5__tar.gz

{abstract_webtools-0.1.6.3/src/abstract_webtools.egg-info → abstract_webtools-0.1.6.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: abstract_webtools
-Version: 0.1.6.3
+Version: 0.1.6.5
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff

{abstract_webtools-0.1.6.3 → abstract_webtools-0.1.6.5}/setup.py

@@ -4,7 +4,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 setuptools.setup(
     name='abstract_webtools',
-    version='0.1.6.03',
+    version='0.1.6.05',
     author='putkoff',
     author_email='partners@abstractendeavors.com',
     description='Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.',

{abstract_webtools-0.1.6.3 → abstract_webtools-0.1.6.5}/src/abstract_webtools/managers/crawlManager.py

@@ -1,21 +1,6 @@
 from .soupManager import *
 
-def normalize_url(url, base_url):
-    """
-    Normalize and resolve relative URLs, ensuring proper domain and format.
-    """
-    # If URL starts with the base URL repeated, remove the extra part
-    if url.startswith(base_url):
-        url = url[len(base_url):]
 
-    # Resolve the URL against the base URL
-    normalized_url = urljoin(base_url, url.split('#')[0])
-
-    # Ensure only URLs belonging to the base domain are kept
-    if not normalized_url.startswith(base_url):
-        return None
-
-    return normalized_url
 class crawlManager():
     def __init__(self,url=None,req_mgr=None,url_mgr=None,source_code=None,parse_type="html.parser"):
         self.url=url

{abstract_webtools-0.1.6.3 → abstract_webtools-0.1.6.5}/src/abstract_webtools/managers/seleniumManager.py

@@ -46,6 +46,7 @@ class seleniumManager(metaclass=SingletonMeta):
             parsed_url = urlparse(url)
             self.domain = parsed_url.netloc
             self.scheme = parsed_url.scheme
+            self.base_url= f"{self.scheme}{self.domain}"
             self.site_dir = os.path.join(os.getcwd(), self.domain)
             os.makedirs(self.site_dir, exist_ok=True)
             self.drivers = {}
@@ -83,7 +84,24 @@ class seleniumManager(metaclass=SingletonMeta):
             self.drivers[url] = driver
             driver.get(url)
         return self.drivers[url]
+def normalize_url(url, base_url=None):
+    """
+    Normalize and resolve relative URLs, ensuring proper domain and format.
+    """
+    # If URL starts with the base URL repeated, remove the extra part
+    manager = seleniumManager(url)
+    base_url = manager.base_url
+    if url.startswith(base_url):
+        url = url[len(base_url):]
 
+    # Resolve the URL against the base URL
+    normalized_url = urljoin(base_url, url.split('#')[0])
+
+    # Ensure only URLs belonging to the base domain are kept
+    if not normalized_url.startswith(base_url):
+        return None
+
+    return normalized_url
 # Function to get Selenium page source
 def get_selenium_source(url):
     url_mgr = urlManager(url)

{abstract_webtools-0.1.6.3 → abstract_webtools-0.1.6.5}/src/abstract_webtools/managers/soupManager.py

@@ -349,3 +349,14 @@ def get_soup_mgr(url=None,url_mgr=None,source_code=None,req_mgr=None,soup_mgr=No
 def get_all_attribute_values(url=None,url_mgr=None,source_code=None,req_mgr=None,soup_mgr=None,tags_list = None):
     soup_mgr = get_soup_mgr(url=url,url_mgr=url_mgr,source_code=source_code,req_mgr=req_mgr,soup_mgr=soup_mgr)
     return soup_mgr.get_all_attribute_values(tags_list=tags_list)
+def get_soup(url=None,url_mgr=None,req_mgr=None,source_code=None,parse_type="html.parser"):
+    if source_code or soup_mgr:
+        if soup_mgr:
+            return soup_mgr.soup
+        return BeautifulSoup(source_code, parse_type)
+    url_mgr = get_url_mgr(url=url,url_mgr=url_mgr)
+    url = get_url(url=url,url_mgr=url_mgr)
+    req_mgr = req_mgr or get_req_mgr(url_mgr=url_mgr,url=url,source_code=source)
+    source_code = req_mgr.source_code
+    soup_mgr = get_soup_mgr(url=url,url_mgr=url_mgr,source_code=source_code,req_mgr=req_mgr,soup_mgr=soup_mgr)
+    return soup_mgr.soup

{abstract_webtools-0.1.6.3 → abstract_webtools-0.1.6.5/src/abstract_webtools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: abstract_webtools
-Version: 0.1.6.3
+Version: 0.1.6.5
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff