abstract-webtools 0.1.6.3__py3-none-any.whl → 0.1.6.4__py3-none-any.whl

abstract_webtools/managers/crawlManager.py

@@ -1,21 +1,6 @@
 from .soupManager import *
 
-def normalize_url(url, base_url):
-    """
-    Normalize and resolve relative URLs, ensuring proper domain and format.
-    """
-    # If URL starts with the base URL repeated, remove the extra part
-    if url.startswith(base_url):
-        url = url[len(base_url):]
 
-    # Resolve the URL against the base URL
-    normalized_url = urljoin(base_url, url.split('#')[0])
-
-    # Ensure only URLs belonging to the base domain are kept
-    if not normalized_url.startswith(base_url):
-        return None
-
-    return normalized_url
 class crawlManager():
     def __init__(self,url=None,req_mgr=None,url_mgr=None,source_code=None,parse_type="html.parser"):
         self.url=url

abstract_webtools/managers/seleniumManager.py

@@ -46,6 +46,7 @@ class seleniumManager(metaclass=SingletonMeta):
             parsed_url = urlparse(url)
             self.domain = parsed_url.netloc
             self.scheme = parsed_url.scheme
+            self.base_url= f"{self.scheme}{self.domain}"
             self.site_dir = os.path.join(os.getcwd(), self.domain)
             os.makedirs(self.site_dir, exist_ok=True)
             self.drivers = {}
@@ -83,7 +84,24 @@ class seleniumManager(metaclass=SingletonMeta):
             self.drivers[url] = driver
             driver.get(url)
         return self.drivers[url]
+def normalize_url(url, base_url=None):
+    """
+    Normalize and resolve relative URLs, ensuring proper domain and format.
+    """
+    # If URL starts with the base URL repeated, remove the extra part
+    manager = seleniumManager(url)
+    base_url = manager.base_url
+    if url.startswith(base_url):
+        url = url[len(base_url):]
 
+    # Resolve the URL against the base URL
+    normalized_url = urljoin(base_url, url.split('#')[0])
+
+    # Ensure only URLs belonging to the base domain are kept
+    if not normalized_url.startswith(base_url):
+        return None
+
+    return normalized_url
 # Function to get Selenium page source
 def get_selenium_source(url):
     url_mgr = urlManager(url)

{abstract_webtools-0.1.6.3.dist-info → abstract_webtools-0.1.6.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: abstract_webtools
-Version: 0.1.6.3
+Version: 0.1.6.4
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff

{abstract_webtools-0.1.6.3.dist-info → abstract_webtools-0.1.6.4.dist-info}/RECORD

@@ -7,22 +7,22 @@ abstract_webtools/url_grabber.py,sha256=pnCCev7ZIuM-6cAGTLmK5HfzZg_AX-fLcRpB6ZE7
 abstract_webtools/url_grabber_new.py,sha256=Oh2Kc0gBScCo0xpopNsg8JE5lIbPuzZVKM5f5GoZmw0,3454
 abstract_webtools/managers/__init__.py,sha256=5aIpbdUsDWTrhPUAjfIKnG54OULqOKan9LBL5EIUllo,407
 abstract_webtools/managers/cipherManager.py,sha256=NHQGdR11eNSm-1H-GezD5dyQgsPTJwY5kczt8Sher2s,1621
-abstract_webtools/managers/crawlManager.py,sha256=NBHyyoKOcy77-0yJGfwaeTD6UZhEJx-k47qIvnLdG-s,11166
+abstract_webtools/managers/crawlManager.py,sha256=vpNFw7ws-ce94XZc3yzvFOcAIknGX_LwAB5nTctxIQs,10645
 abstract_webtools/managers/domainManager.py,sha256=95znOBv05W77mW_fbZAfl4RmlENDlYqhEOMkL02L220,3610
 abstract_webtools/managers/dynamicRateLimiter.py,sha256=gopQcQo50JG2D0KcyepNCIQ_1uDQEBIHBzWf4R2Wgy0,7617
 abstract_webtools/managers/linkManager.py,sha256=m6y9s8jknrTX8RtOAFKeHd4yd23G7Rgf0T7Sp7wmHUw,12180
 abstract_webtools/managers/mySocketClient.py,sha256=-j1Q8Ds9RCSbjZdx3ZF9mVpgwxaO0BBssanUcpYVQoY,2045
 abstract_webtools/managers/networkManager.py,sha256=Op2QDXrP-gmm0tCToe-Ryt9xuOtMppcN2KLKP1WZiu0,952
 abstract_webtools/managers/requestManager.py,sha256=YksYgRivMMuZNOzyL5vaoXv9MLhgYeuLxO_UJiqPGWw,17312
-abstract_webtools/managers/seleniumManager.py,sha256=CtQQYtDrFfgp8ujC6i5SCe0b_hgIA1K68io0aO4igoM,3623
+abstract_webtools/managers/seleniumManager.py,sha256=qSY8gH3N5YJIMwE_Alj9HNQRip_PziIo4_T9AZE_FQo,4273
 abstract_webtools/managers/soupManager.py,sha256=7nSaq7OHaimc8602BihAXCA2ra0dbsB26d4yJzsRARc,16548
 abstract_webtools/managers/sslManager.py,sha256=C-QgQw9CW84uOE5kx2MPjC3RsLbE2JQqdwdTs0H4ecc,1370
 abstract_webtools/managers/tlsAdapter.py,sha256=XZSMZz9EUOhv-h3_Waf6mjV1dA3oN_M_oWuoo4VZ_HE,1454
 abstract_webtools/managers/urlManager.py,sha256=XqMrCM84BeWEfWtHc_8UFpT91ZtG-okzdKdCuC49vsA,8678
 abstract_webtools/managers/userAgentManager.py,sha256=33SB2p2FG7EYZl7l2iYm1U4gI9PcdkGTZHw5lg_Ogrw,1653
 abstract_webtools/managers/videoDownloader.py,sha256=6G_aLc05BTMUYUWc7iqYtHF_BaR7DnCNK_NJ-QnjsYY,10531
-abstract_webtools-0.1.6.3.dist-info/LICENSE,sha256=g3WEJFiVS27HyCGRTwKSsMLyciMaGFdWcZGOe1QalZk,3877
-abstract_webtools-0.1.6.3.dist-info/METADATA,sha256=i8w_enqqacZOgS8uS-l-Vyz5MhnzgJ68ZIrg3M9x0Xg,15857
-abstract_webtools-0.1.6.3.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
-abstract_webtools-0.1.6.3.dist-info/top_level.txt,sha256=2DMJ7RmjTcjCsa-uwAV0K6eXXlIIkFDEjBLg_uyCmCI,18
-abstract_webtools-0.1.6.3.dist-info/RECORD,,
+abstract_webtools-0.1.6.4.dist-info/LICENSE,sha256=g3WEJFiVS27HyCGRTwKSsMLyciMaGFdWcZGOe1QalZk,3877
+abstract_webtools-0.1.6.4.dist-info/METADATA,sha256=JdSW5nzlcxthd2T54jsAaXQ8LAcGpZYIg0Bz2J0ct_k,15857
+abstract_webtools-0.1.6.4.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
+abstract_webtools-0.1.6.4.dist-info/top_level.txt,sha256=2DMJ7RmjTcjCsa-uwAV0K6eXXlIIkFDEjBLg_uyCmCI,18
+abstract_webtools-0.1.6.4.dist-info/RECORD,,