abstract-webtools 0.1.6.138__py3-none-any.whl → 0.1.6.140__py3-none-any.whl

abstract_webtools/managers/__init__.py

@@ -12,3 +12,4 @@ from .urlManager import *
 from .userAgentManager import *
 from .seleniumManager import *
 from .videoDownloader import *
+from .middleManager import *

abstract_webtools/managers/linkManager/linkManager.py

@@ -2,18 +2,7 @@ from ...abstract_webtools import *
 from ..urlManager import *
 from ..requestManager import *
 from ..soupManager import *
-def get_url(url=None,url_mgr=None):
-    if not url and not url_mgr:
-        return None
-    if url:
-        url_mgr = urlManager(url)
-    return url_mgr.url
-def get_url_mgr(url=None,url_mgr=None):
-    if not url and not url_mgr:
-        return None
-    if url:
-        url_mgr = urlManager(url)
-    return url_mgr 
+from ..middleManager import *
 class linkManager:
     """
     LinkManager is a class for managing and extracting links and image links from a web page.
@@ -50,13 +39,40 @@ class linkManager:
         - The LinkManager class helps manage and extract links and image links from web pages.
         - The class provides flexibility in specifying criteria for link extraction.
     """
-    def __init__(self,url=None,source_code=None,url_mgr=None,req_mgr=None,soup_manager=None,image_link_tags='img',img_link_attrs='src',link_tags='a',link_attrs='href',strict_order_tags=False,img_attr_value_desired=None,img_attr_value_undesired=None,link_attr_value_desired=None,link_attr_value_undesired=None,associated_data_attr=["data-title",'alt','title'],get_img=["data-title",'alt','title']):
-        self.url_mgr = get_url_mgr(url=url,url_mgr=url_mgr)
-        self.url=self.url_mgr.url
-        self.req_mgr = get_req_mgr(req_mgr=req_mgr,url=self.url,url_mgr=self.url_mgr,source_code=source_code)
-        self.source_code = source_code or self.req_mgr.source_code or self.req_mgr.source_code_bytes
-        self.soup_mgr = get_soup_mgr(req_mgr=self.req_mgr,url_mgr=self.url_mgr,source_code = source_code)
+    def __init__(self,
+                 url=None,
+                 source_code=None,
+                 soup=None,
+                 url_mgr=None,
+                 req_mgr=None,
+                 soup_mgr=None,
+                 image_link_tags='img',
+                 img_link_attrs='src',
+                 link_tags='a',
+                 link_attrs='href',
+                 strict_order_tags=False,
+                 img_attr_value_desired=None,
+                 img_attr_value_undesired=None,
+                 link_attr_value_desired=None,
+                 link_attr_value_undesired=None,
+                 associated_data_attr=["data-title",'alt','title'],
+                 get_img=["data-title",'alt','title']
+                 ):
+     
+
+        all_tools = get_soup_tools(
+            url=url,
+            url_mgr=url_mgr,
+            source_code=source_code,
+            req_mgr=req_mgr,
+            soup=soup,
+            soup_mgr=soup_mgr,
+            target_manager = self
+            )
 
+                                                       
+                                                 
+        
         self.strict_order_tags=strict_order_tags
         self.image_link_tags=image_link_tags
         self.img_link_attrs=img_link_attrs
@@ -171,19 +187,25 @@ class linkManager:
                 if url not in domains_ls and comp_domain == domain:
                     domains_ls.append(url)
         return domains_ls
+
     def find_all_desired_links(self,tag='img', attr='src',attr_value_desired=None,strict_order_tags=False,attr_value_undesired=None,associated_data_attr=None,all_desired=None,get_img=None):
         all_desired = all_desired or self.find_all_desired(tag=tag,attr=attr,strict_order_tags=strict_order_tags,attr_value_desired=attr_value_desired,attr_value_undesired=attr_value_undesired,associated_data_attr=associated_data_attr,get_img=get_img)
         assiciated_attrs = all_desired[-1]
         valid_assiciated_attrs = []
         desired_links=[]
         for i,attr in enumerate(all_desired[:-1]):
-            valid_attr=self.url_mgr.make_valid(attr,self.url_mgr.protocol+'://'+self.url_mgr.domain) 
-            if valid_attr:
-                desired_links.append(valid_attr)
-                valid_assiciated_attrs.append(assiciated_attrs[i])
-                valid_assiciated_attrs[-1]["link"]=valid_attr
+
+            self.url_mgr.domain = self.url_mgr.domain or ''
+
+            self.url_mgr.protocol = self.url_mgr.protocol or 'https'
+ 
+            if attr:
+                valid_attr=self.url_mgr.make_valid(attr,self.url_mgr.protocol+'://'+self.url_mgr.domain) 
+                if valid_attr:
+                    desired_links.append(valid_attr)
+                    valid_assiciated_attrs.append(assiciated_attrs[i])
+                    valid_assiciated_attrs[-1]["link"]=valid_attr
         desired_links.append(valid_assiciated_attrs)
         return desired_links
 
-
     

abstract_webtools/managers/middleManager/__init__.py

@@ -0,0 +1 @@
+from .src import *

abstract_webtools/managers/middleManager/imports.py

@@ -0,0 +1,18 @@
+import logging
+from ..urlManager import (
+    urlManager,
+    get_url,
+    get_url_mgr
+    )
+from ..requestManager import (
+    requestManager,
+    get_source,
+    get_req_mgr
+    )
+from ..soupManager import (
+    soupManager,
+    get_soup,
+    get_soup_mgr
+    )
+from bs4 import BeautifulSoup
+logging.basicConfig(level=logging.INFO)

abstract_webtools/managers/middleManager/src/UnifiedWebManager.py

@@ -0,0 +1,135 @@
+from ..imports import *
+
+class UnifiedWebManager:
+    """
+    Unified middleware that ties together URL, request, and soup managers.
+    Lazily initializes components based on provided inputs.
+    
+    Args:
+        url (str or None): The base URL.
+        source_code (str or bytes or None): Pre-fetched source code.
+        url_mgr (urlManager or None): Existing URL manager.
+        req_mgr (requestManager or None): Existing request manager.
+        soup_mgr (soupManager or None): Existing soup manager.
+        parse_type (str): Parser type for BeautifulSoup (default: "html.parser").
+    """
+    def __init__(self, url=None, source_code=None, url_mgr=None, req_mgr=None, soup_mgr=None,soup=None, parse_type="html.parser"):
+        self._url = url
+        self._source_code = source_code
+        self._url_mgr = url_mgr
+        self._req_mgr = req_mgr
+        self._soup_mgr = soup_mgr
+        self._parse_type = parse_type
+        self._soup = None  # Lazy
+
+    @property
+    def url_mgr(self):
+        if self._url_mgr is None:
+            if self._url is None:
+                logging.warning("No URL provided; URL manager cannot be created.")
+                return None
+            self._url_mgr = urlManager(url=self._url)
+        return self._url_mgr
+
+    @property
+    def url(self):
+        if self._url is None and self.url_mgr:
+            self._url = self.url_mgr.url
+        return self._url
+
+    @property
+    def req_mgr(self):
+        if self._req_mgr is None:
+            self._req_mgr = requestManager(
+                url=self.url,
+                url_mgr=self.url_mgr,
+                source_code=self._source_code
+            )
+        return self._req_mgr
+
+    @property
+    def source_code(self):
+        if self._source_code is None and self.req_mgr:
+            self._source_code = self.req_mgr.source_code
+        return self._source_code
+
+    @property
+    def soup_mgr(self):
+        if self._soup_mgr is None:
+            self._soup_mgr = soupManager(
+                url=self.url,
+                url_mgr=self.url_mgr,
+                req_mgr=self.req_mgr,
+                source_code=self.source_code
+            )
+        return self._soup_mgr
+
+    @property
+    def soup(self):
+        if self._soup is None:
+            source = self.source_code
+            if source is None:
+                logging.warning("No source code available; Soup cannot be created.")
+                return None
+            if isinstance(source, bytes):
+                source = source.decode('utf-8', errors='ignore')
+            self._soup = BeautifulSoup(source, self._parse_type)
+        return self._soup
+
+    def update_url(self, url):
+        """Update the URL and reset dependent managers."""
+        self._url = url
+        self._url_mgr = None
+        self._req_mgr = None
+        self._soup_mgr = None
+        self._source_code = None
+        self._soup = None
+
+    def update_source_code(self, source_code):
+        """Update the source code and reset dependent managers."""
+        self._source_code = source_code
+        self._req_mgr = None
+        self._soup_mgr = None
+        self._soup = None
+
+    # Convenience methods for direct access
+    def get_all_tools(self):
+        """Return a dict with all components (similar to original getters)."""
+        return {
+            'url': self.url,
+            'url_mgr': self.url_mgr,
+            'source_code': self.source_code,
+            'req_mgr': self.req_mgr,
+            'soup': self.soup,
+            'soup_mgr': self.soup_mgr
+        }
+    def endow_to_manager(self, target_manager, all_tools=None):
+        """
+        Endow (assign) the attributes from all_tools to the target manager instance.
+        
+        Args:
+            target_manager: The instance (e.g., another manager class) to endow attributes to.
+            all_tools (dict or None): Optional dict of tools/attributes. If None, uses self.get_all_tools().
+        """
+        if all_tools is None:
+            all_tools = self.get_all_tools()
+        for key, value in all_tools.items():
+            setattr(target_manager, key, value)
+# Wrapper functions for backward compatibility
+def get_url_tools(url=None, url_mgr=None):
+    mgr = UnifiedWebManager(url=url, url_mgr=url_mgr)
+    return {'url': mgr.url, 'url_mgr': mgr.url_mgr}
+
+def get_req_tools(url=None, url_mgr=None, source_code=None, req_mgr=None):
+    mgr = UnifiedWebManager(url=url, url_mgr=url_mgr, source_code=source_code, req_mgr=req_mgr)
+    return {'url': mgr.url, 'url_mgr': mgr.url_mgr, 'source_code': mgr.source_code, 'req_mgr': mgr.req_mgr}
+
+def get_soup_tools(url=None, url_mgr=None, source_code=None, req_mgr=None, soup=None, soup_mgr=None,target_manager=None):
+    mgr = UnifiedWebManager(url=url, url_mgr=url_mgr, source_code=source_code, req_mgr=req_mgr, soup_mgr=soup_mgr)
+    if soup is not None:
+        mgr._soup = soup  # Allow overriding
+    if target_manager:
+        mgr.endow_to_manager(target_manager, all_tools=None)
+    return mgr.get_all_tools()
+
+

abstract_webtools/managers/middleManager/src/__init__.py

@@ -0,0 +1,2 @@
+from .UnifiedWebManager import *
+from .legacy_tools import *

abstract_webtools/managers/middleManager/src/legacy_tools.py

@@ -0,0 +1,8 @@
+from ..imports import (
+    get_url,
+    get_soup_mgr,
+    get_source,
+    get_req_mgr,
+    get_soup,
+    get_soup_mgr
+    )

abstract_webtools/managers/requestManager/requestManager.py

@@ -1,5 +1,4 @@
 from ...abstract_webtools import *
-
 from ..userAgentManager import *
 from ..cipherManager import *
 from ..sslManager import *
@@ -7,10 +6,14 @@ from ..tlsAdapter import *
 from ..networkManager import *
 from ..seleniumManager import *
 from ..urlManager import *
+logging.basicConfig(level=logging.INFO)
+
 class requestManager:
     """
-    SafeRequest is a class for making HTTP requests with error handling and retries.
-
+    requestManager is a class for making HTTP requests with error handling and retries.
+    It supports initializing with a provided source_code without requiring a URL.
+    If source_code is provided, it uses that as the response content and skips fetching.
+    Enhanced to parse source_code for URLs, PHP blocks, and React/JS data even if not HTML.
     Args:
         url (str or None): The URL to make requests to (default is None).
         url_mgr (urlManager or None): An instance of urlManager (default is None).
@@ -57,278 +60,320 @@ class requestManager:
         - It provides methods for authentication, response handling, and error management.
     """
     def __init__(self,
-                 url=None,
-                 source_code=None,
-                 url_mgr=None,
-                 network_manager=None,
-                 user_agent_manager=None,
-                 ssl_manager=None,
-                 ssl_options=None,
-                 tls_adapter=None,
-                 user_agent=None,
-                 proxies=None,
-                 headers=None,
-                 cookies=None,
-                 session=None,
-                 adapter=None,
-                 protocol=None,
-                 ciphers=None,
-                 spec_login=False,
-                 login_referer=None,
-                 login_user_agent=None,
-                 auth=None,
-                 login_url=None,
-                 email = None,
-                 password=None,
-                 checkbox=None,
-                 dropdown=None,
-                 certification=None,
-                 stream=False,
-                 timeout = None,
-                 last_request_time=None,
-                 max_retries=None,
-                 request_wait_limit=
-                 None):
-        self.url_mgr = get_url_mgr(url=url,url_mgr=url_mgr)
-        self.url=get_url(url=url,url_mgr=self.url_mgr)      
-        self._url_mgr = self.url_mgr
-        self._url=self.url       
-        self.user_agent = user_agent
-        self.user_agent_manager = user_agent_manager or UserAgentManager(user_agent=self.user_agent)
-        self.headers= headers or self.user_agent_manager.header or {'Accept': '*/*'}
-        self.user_agent= self.user_agent_manager.user_agent
-        self.ciphers=ciphers or CipherManager().ciphers_string
-        self.certification=certification
-        self.ssl_options=ssl_options
-        self.ssl_manager = ssl_manager or SSLManager(ciphers=self.ciphers, ssl_options=self.ssl_options, certification=self.certification)
-        self.tls_adapter=tls_adapter or  TLSAdapter(ssl_manager=self.ssl_manager,certification=self.certification,ssl_options=self.ssl_manager.ssl_options)
-        self.network_manager= network_manager or NetworkManager(user_agent_manager=self.user_agent_manager,ssl_manager=self.ssl_manager, tls_adapter=self.tls_adapter,user_agent=user_agent,proxies=proxies,cookies=cookies,ciphers=ciphers, certification=certification, ssl_options=ssl_options)
-        self.stream=stream
-        self.tls_adapter=self.network_manager.tls_adapter
-        self.ciphers=self.network_manager.ciphers
-        self.certification=self.network_manager.certification
-        self.ssl_options=self.network_manager.ssl_options
-        self.proxies=self.network_manager.proxies
-        self.timeout=timeout
-        self.cookies=self.network_manager.cookies
-        self.session = session or requests.session()
-        self.auth = auth
-        self.spec_login=spec_login
-        self.password=password
-        self.email = email
-        self.checkbox=checkbox
-        self.dropdown=dropdown
-        self.login_url=login_url
-        self.login_user_agent=login_user_agent
-        self.login_referer=login_referer
-        self.protocol=protocol or 'https://'
-        
-        self.stream=stream if isinstance(stream,bool) else False
-        self.initialize_session()
-        self.last_request_time=last_request_time
-        self.max_retries = max_retries or 3
-        self.request_wait_limit = request_wait_limit or 1.5
-        self._response=None
-        self.status_code=None
-        self.source_code = get_selenium_source(self.url)
-        self.source_code_bytes=None
-        self.source_code_json = {}
-        self.react_source_code=[]
-        self._response_data = None
-        self.process_response_data()
-    def update_url_mgr(self,url_mgr):
-        self.url_mgr=url_mgr
+                     url=None,
+                     source_code=None,
+                     url_mgr=None,
+                     network_manager=None,
+                     user_agent_manager=None,
+                     ssl_manager=None,
+                     ssl_options=None,
+                     tls_adapter=None,
+                     user_agent=None,
+                     proxies=None,
+                     headers=None,
+                     cookies=None,
+                     session=None,
+                     adapter=None,
+                     protocol=None,
+                     ciphers=None,
+                     spec_login=False,
+                     login_referer=None,
+                     login_user_agent=None,
+                     auth=None,
+                     login_url=None,
+                     email=None,
+                     password=None,
+                     checkbox=None,
+                     dropdown=None,
+                     certification=None,
+                     stream=False,
+                     timeout=None,
+                     last_request_time=None,
+                     max_retries=None,
+                     request_wait_limit=None):
+            self.url_mgr = get_url_mgr(url=url, url_mgr=url_mgr)
+            self.url = get_url(url=url, url_mgr=self.url_mgr)
+            self._url_mgr = self.url_mgr
+            self._url = self.url
+            self.user_agent = user_agent
+            self.user_agent_manager = user_agent_manager or UserAgentManager(user_agent=self.user_agent)
+            self.headers = headers or self.user_agent_manager.header or {'Accept': '*/*'}
+            self.user_agent = self.user_agent_manager.user_agent
+            self.ciphers = ciphers or CipherManager().ciphers_string
+            self.certification = certification
+            self.ssl_options = ssl_options
+            self.ssl_manager = ssl_manager or SSLManager(ciphers=self.ciphers, ssl_options=self.ssl_options, certification=self.certification)
+            self.tls_adapter = tls_adapter or TLSAdapter(ssl_manager=self.ssl_manager, certification=self.certification, ssl_options=self.ssl_manager.ssl_options)
+            self.network_manager = network_manager or NetworkManager(user_agent_manager=self.user_agent_manager, ssl_manager=self.ssl_manager, tls_adapter=self.tls_adapter, user_agent=user_agent, proxies=proxies, cookies=cookies, ciphers=ciphers, certification=certification, ssl_options=ssl_options)
+            self.stream = stream
+            self.tls_adapter = self.network_manager.tls_adapter
+            self.ciphers = self.network_manager.ciphers
+            self.certification = self.network_manager.certification
+            self.ssl_options = self.network_manager.ssl_options
+            self.proxies = self.network_manager.proxies
+            self.timeout = timeout
+            self.cookies = self.network_manager.cookies
+            self.session = session or requests.Session()
+            self.auth = auth
+            self.spec_login = spec_login
+            self.password = password
+            self.email = email
+            self.checkbox = checkbox
+            self.dropdown = dropdown
+            self.login_url = login_url
+            self.login_user_agent = login_user_agent
+            self.login_referer = login_referer
+            self.protocol = protocol or 'https://'
+            self.stream = stream if isinstance(stream, bool) else False
+            self.initialize_session()
+            self.last_request_time = last_request_time
+            self.max_retries = max_retries or 3
+            self.request_wait_limit = request_wait_limit or 1.5
+            self._response = None
+            self.status_code = None
+            self.source_code = None
+            self.source_code_bytes = None
+            self.source_code_json = {}
+            self.react_source_code = []
+            self.extracted_urls = []
+            self.php_blocks = []
+            self._response_data = None
+            if source_code is not None:
+                self._response = source_code
+                self.process_response_data()
+            else:
+                self.re_initialize()
+
+    def update_url_mgr(self, url_mgr):
+        self.url_mgr = url_mgr
         self.re_initialize()
-    def update_url(self,url):
+
+    def update_url(self, url):
         self.url_mgr.update_url(url=url)
         self.re_initialize()
+
     def re_initialize(self):
-        self._response=None
-        self.make_request()
+        self._response = None
+        if self.url_mgr.url is not None:
+            self.make_request()
         self.source_code = None
-        self.source_code_bytes=None
+        self.source_code_bytes = None
         self.source_code_json = {}
-        self.react_source_code=[]
+        self.react_source_code = []
+        self.extracted_urls = []
+        self.php_blocks = []
         self._response_data = None
         self.process_response_data()
+
     @property
     def response(self):
         """Lazy-loading of response."""
-        if self._response is None:
+        if self._response is None and self.url_mgr.url is not None:
             self._response = self.fetch_response()
-            
-            
         return self._response
-    def authenticate(self,session, login_url=None, email=None, password=None,checkbox=None,dropdown=None):
-        login_urls = login_url or [self.url_mgr.url,self.url_mgr.domain,self.url_mgr.url_join(url=self.url_mgr.domain,path='login'),self.url_mgr.url_join(url=self.url_mgr.domain,path='auth')]
+
+    def authenticate(self, session, login_url=None, email=None, password=None, checkbox=None, dropdown=None):
+        login_urls = login_url or [self.url_mgr.url, self.url_mgr.domain, self.url_mgr.url_join(url=self.url_mgr.domain, path='login'), self.url_mgr.url_join(url=self.url_mgr.domain, path='auth')]
         s = session
-        if not isinstance(login_urls,list):
-            login_urls=[login_urls]
+        if not isinstance(login_urls, list):
+            login_urls = [login_urls]
         for login_url in login_urls:
             login_url_mgr = urlManager(login_url)
             login_url = login_url_mgr.url
-            
             r = s.get(login_url)
             soup = BeautifulSoup(r.content, "html.parser")
             # Find the token or any CSRF protection token
             token = soup.find('input', {'name': 'token'}).get('value') if soup.find('input', {'name': 'token'}) else None
-            if token != None:
+            if token is not None:
                 break
         login_data = {}
-        if email != None:
-            login_data['email']=email
-        if password != None:
+        if email is not None:
+            login_data['email'] = email
+        if password is not None:
             login_data['password'] = password
-        if checkbox != None:
+        if checkbox is not None:
             login_data['checkbox'] = checkbox
-        if dropdown != None:
-            login_data['dropdown']=dropdown
-        if token != None:
+        if dropdown is not None:
+            login_data['dropdown'] = dropdown
+        if token is not None:
             login_data['token'] = token
         s.post(login_url, data=login_data)
         return s
 
-    def fetch_response(self) -> Union[requests.Response, None]:
+    def fetch_response(self) -> requests.Response | None | str | bytes:
         """Actually fetches the response from the server."""
-        # You can further adapt this method to use retries or other logic you had
-        # in your original code, but the main goal here is to fetch and return the response
         return self.try_request()
+
     def spec_auth(self, session=None, email=None, password=None, login_url=None, login_referer=None, login_user_agent=None):
-        s = session or requests.session()
-        
-        domain = self.url_mgr.url_join(self.url_mgr.get_correct_url(self.url_mgr.domain),'login') if login_url is None else login_url
+        s = session or requests.Session()
+        domain = self.url_mgr.url_join(self.url_mgr.get_correct_url(self.url_mgr.domain), 'login') if login_url is None else login_url
         login_url = self.url_mgr.get_correct_url(url=domain)
-        
         login_referer = login_referer or self.url_mgr.url_join(url=login_url, path='?role=fast&to=&s=1&m=1&email=YOUR_EMAIL')
         login_user_agent = login_user_agent or 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0'
-        
         headers = {"Referer": login_referer, 'User-Agent': login_user_agent}
         payload = {'email': email, 'pass': password}
-        
         page = s.get(login_url)
         soup = BeautifulSoup(page.content, 'lxml')
         action_url = soup.find('form')['action']
         s.post(action_url, data=payload, headers=headers)
         return s
+
     def initialize_session(self):
-        s = self.session  
+        s = self.session
         if self.auth:
-            s= self.auth
+            s = self.auth
         elif self.spec_login:
-            s=self.spec_auth(session=s,email=self.email, password=self.password, login_url=self.login_url, login_referer=self.login_referer, login_user_agent=self.login_user_agent)
+            s = self.spec_auth(session=s, email=self.email, password=self.password, login_url=self.login_url, login_referer=self.login_referer, login_user_agent=self.login_user_agent)
         elif any([self.password, self.email, self.login_url, self.checkbox, self.dropdown]):
-            s=self.authenticate(session=s, login_url=self.login_url, email=self.email, password=self.password, checkbox=self.checkbox, dropdown=self.dropdown)
+            s = self.authenticate(session=s, login_url=self.login_url, email=self.email, password=self.password, checkbox=self.checkbox, dropdown=self.dropdown)
         s.proxies = self.proxies
         s.cookies["cf_clearance"] = self.network_manager.cookies
         s.headers.update(self.headers)
         s.mount(self.protocol, self.network_manager.tls_adapter)
         return s
+
     def process_response_data(self):
         """Processes the fetched response data."""
         if not self.response:
             return  # No data to process
-        if  isinstance(self.response,str):
-            self.source_code = self.response
+        if isinstance(self.response, (str, bytes)):
+            if isinstance(self.response, str):
+                self.source_code = self.response
+                self.source_code_bytes = self.response.encode('utf-8')  # Assume UTF-8
+            else:
+                self.source_code_bytes = self.response
+                try:
+                    self.source_code = self.response.decode('utf-8')
+                except UnicodeDecodeError:
+                    self.source_code = self.response.decode('latin-1')  # Fallback
+            # Check if it's JSON
+            try:
+                data = json.loads(self.source_code)
+                self.source_code_json = data.get("response", data)
+            except json.JSONDecodeError:
+                pass
         else:
             self.source_code = self.response.text
             self.source_code_bytes = self.response.content
-            if self.response.headers.get('content-type') == 'application/json':
-                data = convert_to_json(self.source_code)
-                if data:
+            if self.response.headers.get('content-type', '').startswith('application/json'):
+                try:
+                    data = json.loads(self.source_code)
                     self.source_code_json = data.get("response", data)
-            
-            self.get_react_source_code()
-    def get_react_source_code(self) -> list:
-        """
-        Fetches the source code of the specified URL and extracts JavaScript and JSX source code (React components).
+                except json.JSONDecodeError:
+                    pass
+        self.extract_urls()
+        self.extract_php_blocks()
+        self.get_react_source_code()
+
+    def extract_urls(self):
+        """Extract URLs from source_code using regex."""
+        if not self.source_code:
+            return
+        url_pattern = r'https?://[^\s<>"\']+'
+        self.extracted_urls = re.findall(url_pattern, self.source_code)
 
-        Args:
-            url (str): The URL to fetch the source code from.
+    def extract_php_blocks(self):
+        """Extract PHP blocks from source_code if present."""
+        if not self.source_code:
+            return
+        php_pattern = r'<\?php(.*?)?\?>'
+        self.php_blocks = re.findall(php_pattern, self.source_code, re.DOTALL)
 
-        Returns:
-            list: A list of strings containing JavaScript and JSX source code found in <script> tags.
+    def get_react_source_code(self) -> list:
         """
-        if self.url_mgr.url is None:
+        Extracts JavaScript and JSX source code from <script> tags if HTML-like.
+        If not HTML and looks like JS/React code, appends the whole source_code.
+        """
+        if not self.source_code:
             return []
-        soup = BeautifulSoup(self.source_code_bytes,"html.parser")
-        script_tags = soup.find_all('script', type=lambda t: t and ('javascript' in t or 'jsx' in t))
+        # Check if likely JS code (e.g., contains 'import', 'function', 'React')
+        is_js_like = any(keyword in self.source_code.lower() for keyword in ['import ', 'function ', 'react', 'export ', 'const ', 'let ', 'var '])
+        # Check if HTML-like
+        is_html_like = self.source_code.strip().startswith('<') or '<html' in self.source_code.lower() or '<!doctype' in self.source_code.lower()
+        if not is_html_like and is_js_like:
+            self.react_source_code.append(self.source_code)
+            return self.react_source_code
+        content = self.source_code_bytes or self.source_code.encode('utf-8')
+        soup = BeautifulSoup(content, "html.parser")
+        script_tags = soup.find_all('script', type=lambda t: t and ('javascript' in t.lower() or 'jsx' in t.lower()))
         for script_tag in script_tags:
-            self.react_source_code.append(script_tag.string)
+            if script_tag.string:
+                self.react_source_code.append(script_tag.string)
+        # If no scripts found but JS-like, append whole
+        if not script_tags and is_js_like:
+            self.react_source_code.append(self.source_code)
+        return self.react_source_code
 
-
-    def get_status(url:str=None) -> int:
+    def get_status(self, url: str = None) -> int | None:
         """
         Gets the HTTP status code of the given URL.
-
-        Args:
-            url (str): The URL to check the status of.
-
-        Returns:
-            int: The HTTP status code of the URL, or None if the request fails.
         """
-        # Get the status code of the URL
-        return try_request(url=url).status_code
+        url = url or self.url_mgr.url
+        if url is None:
+            return None
+        try:
+            response = requests.head(url, timeout=5)
+            return response.status_code
+        except requests.RequestException:
+            return None
+
     def wait_between_requests(self):
         """
         Wait between requests based on the request_wait_limit.
         """
         if self.last_request_time:
-            sleep_time = self.request_wait_limit - (get_time_stamp() - self.last_request_time)
+            sleep_time = self.request_wait_limit - (time.time() - self.last_request_time)
             if sleep_time > 0:
                 logging.info(f"Sleeping for {sleep_time:.2f} seconds.")
-                get_sleep(sleep_time)
+                time.sleep(sleep_time)
 
     def make_request(self):
         """
         Make a request and handle potential errors.
         """
-        # Update the instance attributes if they are passed
-        
+        if self.url_mgr.url is None:
+            return None
         self.wait_between_requests()
         for _ in range(self.max_retries):
             try:
-                self.try_request()  # 10 seconds timeout
-                if self.response:
-                    self.status_code = self.response.status_code
-                    if self.response.status_code == 200:
-                        self.last_request_time = get_time_stamp()
-                        return self.response
-                    elif self.response.status_code == 429:
-                        logging.warning(f"Rate limited by {self.url_mgr.url}. Retrying...")
-                        get_sleep(5)  # adjust this based on the server's rate limit reset time
+                self._response = self.try_request()
+                if self._response:
+                    if not isinstance(self._response, (str, bytes)):
+                        self.status_code = self._response.status_code
+                        if self._response.status_code == 200:
+                            self.last_request_time = time.time()
+                            return self._response
+                        elif self._response.status_code == 429:
+                            logging.warning(f"Rate limited by {self.url_mgr.url}. Retrying...")
+                            time.sleep(5)
             except requests.Timeout as e:
-                logging.error(f"Request to {cleaned_url} timed out: {e}")
+                logging.error(f"Request to {self.url_mgr.url} timed out: {e}")
             except requests.ConnectionError:
                 logging.error(f"Connection error for URL {self.url_mgr.url}.")
-            except requests.Timeout:
-                logging.error(f"Request timeout for URL {self.url_mgr.url}.")
             except requests.RequestException as e:
                 logging.error(f"Request exception for URL {self.url_mgr.url}: {e}")
         try:
             response = get_selenium_source(self.url_mgr.url)
             if response:
-                self.response = response
-                return self.response
-        except:
-            logging.error(f"Failed to retrieve content from {self.url_mgr.url} after {self.max_retries} retries.")
+                self._response = response
+                self.status_code = 200  # Assume success
+                return self._response
+        except Exception as e:
+            logging.error(f"Failed to retrieve content from {self.url_mgr.url} after {self.max_retries} retries: {e}")
             return None
-    def try_request(self) -> Union[requests.Response, None]:
+
+    def try_request(self) -> requests.Response | str | bytes | None:
         """
         Tries to make an HTTP request to the given URL using the provided session.
-
-        Args:
-            timeout (int): Timeout for the request.
-
-        Returns:
-            requests.Response or None: The response object if the request is successful, or None if the request fails.
         """
+        if self.url_mgr.url is None:
+            return None
         try:
-            return get_selenium_source(self.url_mgr.url)#self.session.get(url=self.url_mgr.url, timeout=self.timeout,stream=self.stream)
-        except requests.exceptions.RequestException as e:
-            print(e)
+            return get_selenium_source(self.url_mgr.url)  # or self.session.get(self.url_mgr.url, timeout=self.timeout, stream=self.stream)
+        except requests.RequestException as e:
+            logging.error(f"Request failed: {e}")
             return None
 
-
     @property
     def url(self):
         return self.url_mgr.url
@@ -345,12 +390,11 @@ class SafeRequestSingleton:
         elif SafeRequestSingleton._instance.url != url or SafeRequestSingleton._instance.headers != headers or SafeRequestSingleton._instance.max_retries != max_retries or SafeRequestSingleton._instance.request_wait_limit != request_wait_limit:
             SafeRequestSingleton._instance = SafeRequest(url,url_mgr=urlManagerSingleton,headers=headers,max_retries=max_retries,last_request_time=last_request_time,request_wait_limit=request_wait_limit)
         return SafeRequestSingleton._instance
+def get_source(url=None,url_mgr=None,source_code=None,req_mgr=None):
+    req_mgr = get_req_mgr(req_mgr=req_mgr,url=url,url_mgr=url_mgr,source_code=source_code)
+    return req_mgr.source_code
 def get_req_mgr(url=None,url_mgr=None,source_code=None,req_mgr=None):
     url = get_url(url=url,url_mgr=url_mgr)
     url_mgr = get_url_mgr(url=url,url_mgr=url_mgr )
     req_mgr = req_mgr  or requestManager(url_mgr=url_mgr,url=url,source_code=source_code)
     return req_mgr
-def get_source(url=None,url_mgr=None,source_code=None,req_mgr=None):
-    # Placeholder for actual implementation.
-    req_mgr = get_req_mgr(req_mgr=req_mgr,url=url,url_mgr=url_mgr,source_code=source_code)
-    return req_mgr.source_code

abstract_webtools/managers/soupManager/soupManager.py

@@ -49,7 +49,7 @@ class soupManager:
         source_code = source_code or self.req_mgr.source_code or self.req_mgr.source_code_bytes
         if source_code:
             source_code = str(source_code)
-        self.source_code = source_code
+        self.source_code = source_code or ''
         self.soup= BeautifulSoup(self.source_code, self.parse_type)
         self.all_tags_and_attribute_names = self.get_all_tags_and_attribute_names()
         self.all_tags = self.all_tags_and_attribute_names.get('tags')
@@ -340,15 +340,6 @@ class SoupManagerSingleton():
         elif parse_type != SoupManagerSingleton._instance.parse_type  or source_code != SoupManagerSingleton._instance.source_code:
             SoupManagerSingleton._instance = SoupManager(url_mgr,requestManager,parse_type=parse_type,source_code=source_code)
         return SoupManagerSingleton._instance
-def get_soup_mgr(url=None,url_mgr=None,source_code=None,req_mgr=None,soup_mgr=None,parse_type="html.parser"):
-    url_mgr = get_url_mgr(url=url,url_mgr=url_mgr)
-    url = get_url(url=url,url_mgr=url_mgr)
-    req_mgr = get_req_mgr(url_mgr=url_mgr,url=url,source_code=source_code)
-    soup_mgr = soup_mgr or soupManager(url_mgr=url_mgr,req_mgr=req_mgr,url=url,source_code=source_code)
-    return soup_mgr
-def get_all_attribute_values(url=None,url_mgr=None,source_code=None,req_mgr=None,soup_mgr=None,tags_list = None,parse_type="html.parser"):
-    soup_mgr = get_soup_mgr(url=url,url_mgr=url_mgr,source_code=source_code,req_mgr=req_mgr,soup_mgr=soup_mgr)
-    return soup_mgr.get_all_attribute_values(tags_list=tags_list)
 def get_soup(url=None,url_mgr=None,req_mgr=None,source_code=None,soup_mgr=None,parse_type="html.parser"):
     if source_code or soup_mgr:
         if soup_mgr:
@@ -360,3 +351,12 @@ def get_soup(url=None,url_mgr=None,req_mgr=None,source_code=None,soup_mgr=None,p
     source_code = req_mgr.source_code
     soup_mgr = get_soup_mgr(url=url,url_mgr=url_mgr,source_code=source_code,req_mgr=req_mgr,soup_mgr=soup_mgr)
     return soup_mgr.soup
+def get_soup_mgr(url=None,url_mgr=None,source_code=None,req_mgr=None,soup_mgr=None,parse_type="html.parser"):
+    url_mgr = get_url_mgr(url=url,url_mgr=url_mgr)
+    url = get_url(url=url,url_mgr=url_mgr)
+    req_mgr = get_req_mgr(url_mgr=url_mgr,url=url,source_code=source_code)
+    soup_mgr = soup_mgr or soupManager(url_mgr=url_mgr,req_mgr=req_mgr,url=url,source_code=source_code)
+    return soup_mgr
+def get_all_attribute_values(url=None,url_mgr=None,source_code=None,req_mgr=None,soup_mgr=None,tags_list = None,parse_type="html.parser"):
+    soup_mgr = get_soup_mgr(url=url,url_mgr=url_mgr,source_code=source_code,req_mgr=req_mgr,soup_mgr=soup_mgr)
+    return soup_mgr.get_all_attribute_values(tags_list=tags_list)

abstract_webtools/managers/urlManager/urlManager.py

@@ -17,21 +17,21 @@ class urlManager:
     Now handles url=None gracefully: sets internals to None/empty and methods return None or empty values without errors.
     """
     def __init__(self, url=None, session=None):
-        self._url = url  # Allow None
-        self.session = session or requests.Session()
-        if self._url is None:
-            self.clean_urls = []
-            self.url = None
-            self.protocol = None
-            self.domain = None
-            self.path = ""
-            self.query = ""
-            self.all_urls = []
-        else:
-            self.clean_urls = self.clean_url()
-            self.url = self.get_correct_url() or self._url
-            self.protocol, self.domain, self.path, self.query = self.url_to_pieces(self.url)
-            self.all_urls = []
+            self._url = url  # Allow None
+            self.session = session or requests.Session()
+            if self._url is None:
+                self.clean_urls = []
+                self.url = None
+                self.protocol = None
+                self.domain = None
+                self.path = ""
+                self.query = ""
+                self.all_urls = []
+            else:
+                self.clean_urls = self.clean_url()
+                self.url = self.get_correct_url() or self._url
+                self.protocol, self.domain, self.path, self.query = self.url_to_pieces(self.url)
+                self.all_urls = []
 
     def url_to_pieces(self, url):
         """
@@ -233,17 +233,9 @@ class urlManagerSingleton:
         elif urlManagerSingleton._instance.session != session or urlManagerSingleton._instance.url != url:
             urlManagerSingleton._instance = urlManager(url, session=session)
         return urlManagerSingleton._instance
+def get_url(url=None,url_mgr=None):
+    url_mgr = get_url_mgr(url=url,url_mgr=url_mgr)
+    return url_mgr.url
+def get_url_mgr(url=None,url_mgr=None):
+    return url_mgr or urlManager(url)
 
-def get_url(url=None, url_mgr=None):
-    if not url and not url_mgr:
-        return None
-    if url_mgr is None and url is not None:
-        url_mgr = urlManager(url)
-    return url_mgr.url if url_mgr else None
-
-def get_url_mgr(url=None, url_mgr=None):
-    if url_mgr is None:
-        url_mgr = urlManager(url=url)  # Always create instance, even if url=None
-    if url_mgr and url is None:
-        url = url_mgr.url
-    return url_mgr

{abstract_webtools-0.1.6.138.dist-info → abstract_webtools-0.1.6.140.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: abstract_webtools
-Version: 0.1.6.138
+Version: 0.1.6.140
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff

{abstract_webtools-0.1.6.138.dist-info → abstract_webtools-0.1.6.140.dist-info}/RECORD

@@ -10,7 +10,7 @@ abstract_webtools/main.py,sha256=_I7pPXPkoLZOoYGLQDrSLGhGuQt6-PVyXEHZSmglk2g,132
 abstract_webtools/soup_gui.py,sha256=n95YAps1R6DpMwR4UbthSqQby0C5WHUa9tsW-f2qpLg,5184
 abstract_webtools/url_grabber.py,sha256=pnCCev7ZIuM-6cAGTLmK5HfzZg_AX-fLcRpB6ZE70B8,10441
 abstract_webtools/url_grabber_new.py,sha256=xb23qo4anOY0Ax3CAfaHJ8s5VEz61Sinh-XpEDFW7Is,3621
-abstract_webtools/managers/__init__.py,sha256=9pgy52NB-ONxLqoCRF52GZ6G7GM6Uc0-fgA1HvKcwxc,407
+abstract_webtools/managers/__init__.py,sha256=d7Q6_McRuKOHmKuna19s0l1wMgtM1JgUX8rHaSqJIcE,436
 abstract_webtools/managers/allss\.py,sha256=IBhlyRQHfK-BtwUnSEbIPqlI1MtZ8-XsdaHv0b91HQ0,269
 abstract_webtools/managers/cipherManager.py,sha256=NHQGdR11eNSm-1H-GezD5dyQgsPTJwY5kczt8Sher2s,1621
 abstract_webtools/managers/crawlManager.py,sha256=62Ej6AQC6-qXX_EWOmcJ2szNvEjmebFGugMz65HF1qI,12983
@@ -30,16 +30,21 @@ abstract_webtools/managers/videoDownloader2.py,sha256=v3H6akdhvVWGrB-r35m3cp_-aK
 abstract_webtools/managers/clownworld/__init__.py,sha256=eq25euhRbFqHLm1ibi_7FGz_oNWs-kkyAkETzK3r4_Q,35
 abstract_webtools/managers/clownworld/get_bolshevid_video.py,sha256=dNZdOxhXSA13DWFjdSOmvYrI3HybkrrvTBaMDbJfhfo,10140
 abstract_webtools/managers/linkManager/__init__.py,sha256=NpfWNzvTLSfsIWSeLYIxPzeLHADk_grSx5rfgCeWERw,27
-abstract_webtools/managers/linkManager/linkManager.py,sha256=roxOzOELca0rOlcMaJkTQHN3S0XF7dJihZmMq-uIXPQ,12184
+abstract_webtools/managers/linkManager/linkManager.py,sha256=KYGjAu2YYF7NzztfHPVXAk1X1zjU39-bnFpaSxErTDg,12368
+abstract_webtools/managers/middleManager/__init__.py,sha256=RLLS1CxPpixIiV50P6tFaJcQ9C2O3lz19I4EDMc_4rE,19
+abstract_webtools/managers/middleManager/imports.py,sha256=T0cdlABayG64RI4PnDRf7gwLvcQ5owobD0EdaD0Fcuc,334
+abstract_webtools/managers/middleManager/src/UnifiedWebManager.py,sha256=2jtr4ebfDjpw1TcaZ5D9A324jtKLNohx-Ol5JrCBon0,4996
+abstract_webtools/managers/middleManager/src/__init__.py,sha256=YaSAh7AG1EvFWFZBIe4pGvzmfr60rpR9ZDWoQKqAMd0,61
+abstract_webtools/managers/middleManager/src/legacy_tools.py,sha256=2cCnRaq8UO7HdtffNtAOsZFJm_mpZbpvBuX0pIIWGaM,125
 abstract_webtools/managers/requestManager/__init__.py,sha256=z2qGtweEoO_OKr959LGxVXEMu1hu7PIkmh89BEh5TI8,30
-abstract_webtools/managers/requestManager/requestManager.py,sha256=g3kqaGRXu5ZR8rj58xN32fBVPpDXI_CRuW3tJgnbznE,17459
+abstract_webtools/managers/requestManager/requestManager.py,sha256=JxJKz52DhyExrdpRUFIsG4GTY89AQ_ogeujGLDhefLI,20118
 abstract_webtools/managers/soupManager/__init__.py,sha256=mqfXfqM9sWlYpOkoXUqtBoVvk2KQx1862NnmRVJwGtY,27
 abstract_webtools/managers/soupManager/asoueces.py,sha256=OaXqolZl0dI7b09NYwJ3Wnhuxf89ahZ1GjsOqy0GXfk,3506
-abstract_webtools/managers/soupManager/soupManager.py,sha256=U3_o189-OWoBRaSCe2sIkg-bHxBt2mKpYMyZd-nJjLQ,17201
+abstract_webtools/managers/soupManager/soupManager.py,sha256=T76uvnR7rsbQk2DiwMNRprxiMO4rGSi32f3TZfft_pQ,17207
 abstract_webtools/managers/urlManager/__init__.py,sha256=gaJCHeK91Z-eYsBnxgdhbIUten1-gbx-zqx70R6ag-Y,26
 abstract_webtools/managers/urlManager/urlManager (Copy).py,sha256=vCFuLADmv3h7icaaoAsImGqb_49VizPY_ZvMl-C7PYk,7756
-abstract_webtools/managers/urlManager/urlManager.py,sha256=LG8WiEsf16vMzum48D5rmbRNK6VzYG4FyOTr2FJiOEc,9133
-abstract_webtools-0.1.6.138.dist-info/METADATA,sha256=wBxaIqQkImZwBOPVoOzJYc2UAp3jxtuL7y9SWULLiUY,7289
-abstract_webtools-0.1.6.138.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-abstract_webtools-0.1.6.138.dist-info/top_level.txt,sha256=2DMJ7RmjTcjCsa-uwAV0K6eXXlIIkFDEjBLg_uyCmCI,18
-abstract_webtools-0.1.6.138.dist-info/RECORD,,
+abstract_webtools/managers/urlManager/urlManager.py,sha256=vY4KQXtcrlC2YtlultxQpVe581l5kAuT5VGA0WrI16g,8945
+abstract_webtools-0.1.6.140.dist-info/METADATA,sha256=kGrkruUx1tWeifzVXhxWsTX8a8jGNgeHxiDH6FCbv_o,7289
+abstract_webtools-0.1.6.140.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+abstract_webtools-0.1.6.140.dist-info/top_level.txt,sha256=2DMJ7RmjTcjCsa-uwAV0K6eXXlIIkFDEjBLg_uyCmCI,18
+abstract_webtools-0.1.6.140.dist-info/RECORD,,