abstract-webtools 0.1.6.145__py3-none-any.whl → 0.1.6.147__py3-none-any.whl

abstract_webtools/managers/soupManager/soupManager.py

@@ -1,6 +1,9 @@
 from ...abstract_webtools import *
 from ..urlManager import *
 from ..requestManager import *
+from bs4 import BeautifulSoup
+import re, json
+
 class soupManager:
     """
     SoupManager is a class for managing and parsing HTML source code using BeautifulSoup.
@@ -39,25 +42,49 @@ class soupManager:
         - The SoupManager class is designed for parsing HTML source code using BeautifulSoup.
         - It provides various methods to extract data and discover elements within the source code.
     """
-    def __init__(self,url=None,source_code=None,url_mgr=None,req_mgr=None,soup=None, parse_type="html.parser"):
-        self.soup=[]
-        url = get_url(url=url,url_mgr=url_mgr)
-        self.url_mgr = get_url_mgr(url=url,url_mgr=url_mgr)
-        self.url=self.url_mgr.url
-        self.req_mgr = get_req_mgr(req_mgr=req_mgr,url=self.url,url_mgr=self.url_mgr,source_code=source_code)
-        self.parse_type = parse_type
-        source_code = source_code or self.req_mgr.source_code or self.req_mgr.source_code_bytes
-        if source_code:
-            source_code = str(source_code)
-        self.source_code = source_code or ''
-        self.soup= soup or BeautifulSoup(self.source_code, self.parse_type)
-        self.all_tags_and_attribute_names = self.get_all_tags_and_attribute_names()
-        self.all_tags = self.all_tags_and_attribute_names.get('tags')
-        self.all_attribute_names = self.all_tags_and_attribute_names.get('attributes')
-        self.all_tags_and_attributes = self.all_tags + self.all_attribute_names
-        
-        self._all_links_data = None
-        self._meta_tags_data = None
+
+    def __init__(self, url=None, source_code=None, req_mgr=None, parse_type="html.parser"):
+        self.url = url
+        self.req_mgr = req_mgr
+        self.source_code = (source_code or (req_mgr.source_code if req_mgr else "")) or ""
+        self.soup = BeautifulSoup(self.source_code, parse_type)
+
+    def all_meta(self):
+        out = []
+        for m in self.soup.find_all("meta"):
+            row = {}
+            for k in ("name","property","http-equiv","itemprop","charset","content"):
+                v = m.get(k)
+                if v: row[k] = v
+            if row: out.append(row)
+        return out
+
+    def citation_dict(self):
+        out = {}
+        for m in self.soup.find_all("meta"):
+            k = (m.get("name") or m.get("property") or "").lower()
+            if k.startswith("citation_") and m.get("content"):
+                out.setdefault(k, []).append(m["content"])
+        return out
+
+    def all_links(self):
+        res = []
+        for l in self.soup.find_all("link"):
+            rel = l.get("rel")
+            if isinstance(rel, list): rel = " ".join(rel)
+            res.append({
+                "rel": rel, "href": l.get("href"),
+                "type": l.get("type"), "title": l.get("title"), "hreflang": l.get("hreflang")
+            })
+        return res
+
+    def all_jsonld(self):
+        blocks = []
+        for s in self.soup.find_all("script", type=re.compile("application/ld\\+json", re.I)):
+            txt = s.get_text(strip=True)
+            try: blocks.append(json.loads(txt))
+            except Exception: blocks.append({"raw": txt})
+        return blocks
     def re_initialize(self):
         self.soup= BeautifulSoup(self.source_code, self.parse_type)
         self._all_links_data = None

abstract_webtools/managers/sslManager.py

@@ -1,12 +1,21 @@
 from ..abstract_webtools import *
+# sslManager.py
+from ..abstract_webtools import *  # must expose ssl, ssl_
+from .cipherManager import CipherManager  # be explicit, safer
+
 class SSLManager:
     def __init__(self, ciphers=None, ssl_options=None, certification=None):
         self.ciphers = ciphers or CipherManager().ciphers_string
         self.ssl_options = ssl_options or self.get_default_ssl_settings()
         self.certification = certification or ssl.CERT_REQUIRED
         self.ssl_context = self.get_context()
+
     def get_default_ssl_settings(self):
         return ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1 | ssl.OP_NO_COMPRESSION
-    def get_context(self):
-        return ssl_.create_urllib3_context(ciphers=self.ciphers, cert_reqs=self.certification, options=self.ssl_options)
 
+    def get_context(self):
+        return ssl_.create_urllib3_context(
+            ciphers=self.ciphers,
+            cert_reqs=self.certification,
+            options=self.ssl_options
+        )

abstract_webtools/managers/userAgentManager.py

@@ -1,29 +1,27 @@
+# userAgentManager.py
 from ..abstract_webtools import *
 import random
+
 operating_systems = ['Macintosh','Windows','Linux']
 browsers = ['Firefox','Chrome','IceDragon','Waterfox','Gecko','Safari','MetaSr']
-def get_itter(iter_input,itter_list):
-    if not iter_input:
-        return itter_list[0]
-    if iter_input in itter_list:
-        return iter_input
-    iter_input_lower = iter_input.lower()
-    for itter in itter_list:
-        itter_lower = itter.lower()
-        if iter_input_lower in itter_lower:
-            return itter
-    return itter_list[0]
-def get_browser(browser=None):
-    return get_itter(browser,browsers)
-def get_operating_system(operating_system=None):
-    return get_itter(operating_system,operating_systems)
+
+def _pick(val, options):
+    if not val: return options[0]
+    if val in options: return val
+    l = val.lower()
+    for o in options:
+        if l in o.lower():
+            return o
+    return options[0]
+
 class UserAgentManager:
-    def __init__(self, operating_system=None, browser=None, version=None,user_agent=None):
-        self.operating_system = get_operating_system(operating_system=operating_system)
-        self.browser = get_browser(browser=browser)
+    def __init__(self, operating_system=None, browser=None, version=None, user_agent=None):
+        self.operating_system = _pick(operating_system, operating_systems)
+        self.browser = _pick(browser, browsers)
         self.version = version or '42.0'
         self.user_agent = user_agent or self.get_user_agent()
-        self.header = self.user_agent_header()
+        self.header = {"user-agent": self.user_agent}
+
     @staticmethod
     def user_agent_db():
         from ..big_user_agent_list import big_user_agent_dict
@@ -31,30 +29,23 @@ class UserAgentManager:
 
     def get_user_agent(self):
         ua_db = self.user_agent_db()
+        os_db = ua_db.get(self.operating_system) or random.choice(list(ua_db.values()))
+        br_db = os_db.get(self.browser) or random.choice(list(os_db.values()))
+        if self.version in br_db:
+            return br_db[self.version]
+        return random.choice(list(br_db.values()))
 
-        if self.operating_system and self.operating_system in ua_db:
-            operating_system_db = ua_db[self.operating_system]
-        else:
-            operating_system_db = random.choice(list(ua_db.values()))
-
-        if self.browser and self.browser in operating_system_db:
-            browser_db = operating_system_db[self.browser]
-        else:
-            browser_db = random.choice(list(operating_system_db.values()))
-
-        if self.version and self.version in browser_db:
-            return browser_db[self.version]
-        else:
-            return random.choice(list(browser_db.values()))
-
-    def user_agent_header(self):
-        return {"user-agent": self.user_agent}
 class UserAgentManagerSingleton:
     _instance = None
+
     @staticmethod
-    def get_instance(user_agent=UserAgentManager().get_user_agent()[0]):
+    def get_instance(**kwargs):
+        ua = kwargs.get("user_agent")
         if UserAgentManagerSingleton._instance is None:
-            UserAgentManagerSingleton._instance = UserAgentManager(user_agent=user_agent)
-        elif UserAgentManagerSingleton._instance.user_agent != user_agent:
-            UserAgentManagerSingleton._instance = UserAgentManager(user_agent=user_agent)
+            UserAgentManagerSingleton._instance = UserAgentManager(**kwargs)
+        else:
+            # rebuild if user_agent explicitly changed
+            inst = UserAgentManagerSingleton._instance
+            if ua and ua != inst.user_agent:
+                UserAgentManagerSingleton._instance = UserAgentManager(**kwargs)
         return UserAgentManagerSingleton._instance

abstract_webtools/url_grabber.py

@@ -1,7 +1,11 @@
 from abstract_gui import make_component,sg
 import inspect
 import re
+from . import UserAgentManager,UrlManager,SafeRequest,SoupManager,LinkManager,CipherManager,requests,ssl,BeautifulSoup,HTTPAdapter,PoolManager,ssl_
+<<<<<<< HEAD
+=======
 from .managers import *
+>>>>>>> ba4baf2 (Deploy version 0.1.6.147 at 2025-09-07 09:40:38 UTC)
 window = None
 
 def get_attrs(values):
@@ -25,7 +29,11 @@ def get_attrs(values):
   else:
     tags_js['attribute']=tags_js['attribute'][0]
   return tags_js
+<<<<<<< HEAD
+def get_user_agent_manager(user_agent=None):
+=======
 def get_user_agent_mgr(user_agent=None):
+>>>>>>> ba4baf2 (Deploy version 0.1.6.147 at 2025-09-07 09:40:38 UTC)
   return UserAgentManager(user_agent=user_agent)
 def get_cipher_list():
   return CipherManager().get_default_ciphers()
@@ -128,17 +136,43 @@ def process_url(window,values):
     if warn_url=='' or warn_url == None:
       update_status(window,warn,warn_url,response_code,valid)
       return False
+    temp_url=UrlManager(url=warn_url).url
+    if temp_url:
+      valid='valid'
+      response_code = SafeRequest(url=temp_mgr).response.status_code
+<<<<<<< HEAD
+=======
     temp_url=urlManager(url=warn_url).url
     if temp_url:
       valid='valid'
       response_code = requestManager(url=temp_mgr).response.status_code
+>>>>>>> ba4baf2 (Deploy version 0.1.6.147 at 2025-09-07 09:40:38 UTC)
       warn = 'success'
       warn_url = temp_mgr
       update_status(window,warn,warn_url,response_code,valid)
       return temp_mgr
     update_status(window,warn,warn_url,response_code,valid)
     return False
-def update_url(url_mgr,request_mgr,soup_mgr,link_mgr,values,cipher_list=get_cipher_list(),user_agent=get_user_agents()[0]):
+def update_url(url_manager,request_manager,soup_manager,link_manager,values,cipher_list=get_cipher_list(),user_agent=get_user_agents()[0]):
+      ciphers = CipherManager(cipher_list=cipher_list).ciphers_string
+      request_manager = SafeRequest(url_manager=url_manager,ciphers=ciphers,user_agent=get_user_agents()[0])
+      if request_manager.source_code:
+        soup_manager= SoupManager(url_manager=url_manager,request_manager=request_manager)
+        link_manager= LinkManager(url_manager=url_manager,request_manager=request_manager,soup_manager=soup_manager)
+        window['-URL-'].update(value=url_manager.url)
+        window['-CIPHERS_OUTPUT-'].update(value=request_manager.ciphers)
+        return update_source_code(url_manager,request_manager,soup_manager,link_manager,values)
+      else:
+        return url_manager,request_manager,soup_manager,link_manager
+def update_source_code(url_manager,request_manager,soup_manager,link_manager,values):
+    parse_type = values['-parse_type-']
+    if parse_type != soup_manager.parse_type:
+      soup_manager.update_parse_type(parse_type=parse_type)
+    all_tags=soup_manager.get_all_tags_and_attribute_names()
+<<<<<<< HEAD
+    window['-SOURCECODE-'].update(value=soup_manager.soup)
+=======
+    window['-SOURCECODE-'].update(value=soup_manager.soupdef update_url(url_mgr,request_mgr,soup_mgr,link_mgr,values,cipher_list=get_cipher_list(),user_agent=get_user_agents()[0]):
       ciphers = CipherManager(cipher_list=cipher_list).ciphers_string
       request_mgr = requestManager(url_mgr=url_mgr,ciphers=ciphers,user_agent=get_user_agents()[0])
       if request_mgr.source_code:
@@ -155,12 +189,23 @@ def update_source_code(url_mgr,request_mgr,soup_mgr,link_mgr,values):
       soup_mgr.update_parse_type(parse_type=parse_type)
     all_tags=soup_mgr.get_all_tags_and_attribute_names()
     window['-SOURCECODE-'].update(value=soup_mgr.soup)
+>>>>>>> ba4baf2 (Deploy version 0.1.6.147 at 2025-09-07 09:40:38 UTC)
     if values['-SOUP_TAG-'] != all_tags['tags']:
       window['-SOUP_TAG-'].update(values=all_tags['tags'],value=all_tags['tags'][0])
     if values['-SOUP_ATTRIBUTE-'] != all_tags['attributes']:
       window['-SOUP_ATTRIBUTE-'].update(values=all_tags['attributes'],value=all_tags['attributes'][0])
       window['-SOUP_ATTRIBUTE_1-'].update(values=all_tags['attributes'],value=all_tags['attributes'][0])
       window['-SOUP_ATTRIBUTE_2-'].update(values=all_tags['attributes'],value=all_tags['attributes'][0])
+      return url_manager,request_manager,soup_manager,link_manager
+def url_grabber_while(window,initial_url="www.example.com"):
+    return_data=None
+    url_grab = False
+    url_manager=UrlManager(url=initial_url)
+    request_manager = SafeRequest(url_manager=url_manager)
+    soup_manager= SoupManager(url_manager=url_manager,request_manager=request_manager)
+    link_manager= LinkManager(url_manager=url_manager,request_manager=request_manager,soup_manager=soup_manager)
+<<<<<<< HEAD
+=======
       return url_mgr,request_mgr,soup_mgr,link_mgr
 def url_grabber_while(window,initial_url="www.example.com"):
     return_data=None
@@ -169,12 +214,25 @@ def url_grabber_while(window,initial_url="www.example.com"):
     request_mgr = requestManager(url_mgr=url_mgr)
     soup_mgr= SoupManager(url_mgr=url_mgr,request_mgr=request_mgr)
     link_mgr= LinkManager(url_mgr=url_mgr,request_mgr=request_mgr,soup_mgr=soup_mgr)
+>>>>>>> ba4baf2 (Deploy version 0.1.6.147 at 2025-09-07 09:40:38 UTC)
     while True:
         event, values = window.read()
         if event == sg.WINDOW_CLOSED:
             break
         if event=='-GRAB_URL-' or not url_grab:
           url=values['-URL-']
+          if UrlManager(url=url).url:
+            if url != url_manager.url or url == initial_url:
+              url_manager = UrlManager(url=url)
+              
+              url_manager,request_manager,soup_manager,link_manager=update_url(url_manager=url_manager,request_manager=request_manager,soup_manager=soup_manager,link_manager=link_manager,values=values)
+              window['-URL-'].update(value=url_manager.url)
+              url_grab=True
+        if event == 'get soup':
+            tags_js = get_attrs(values)
+            all_desired=soup_manager.find_tags_by_attributes(tag=tags_js['tag'], attr=tags_js['attribute'],attr_values=tags_js['input'])
+<<<<<<< HEAD
+=======
           if urlManager(url=url).url:
             if url != url_mgr.url or url == initial_url:
               url_mgr = urlManager(url=url)
@@ -185,21 +243,35 @@ def url_grabber_while(window,initial_url="www.example.com"):
         if event == 'get soup':
             tags_js = get_attrs(values)
             all_desired=soup_mgr.find_tags_by_attributes(tag=tags_js['tag'], attr=tags_js['attribute'],attr_values=tags_js['input'])
+>>>>>>> ba4baf2 (Deploy version 0.1.6.147 at 2025-09-07 09:40:38 UTC)
             window['-FIND_ALL_OUTPUT-'].update(value=all_desired)
         if event == '-CUSTOMUA-':
             window['-SOURCECODE-'].update(disabled=values['-CUSTOMUA-'])
             if not values['-CUSTOMUA-']:
+                window['-USERAGENT-'].update(value=user_agent_manager.user_agent_header)
+<<<<<<< HEAD
+=======
                 window['-USERAGENT-'].update(value=user_agent_mgr.user_agent_header)
+>>>>>>> ba4baf2 (Deploy version 0.1.6.147 at 2025-09-07 09:40:38 UTC)
                 window['-USERAGENT-'].update(disabled=True)
             else:
                 window['-USERAGENT-'].update(disabled=False)
         if event=='Get All Text':
+            window['-FIND_ALL_OUTPUT-'].update(value=soup_manager.extract_text_sections())
+        if event == 'Action':
+            parse_type = values['-parse_type-']
+            if parse_type != soup_manager.parse_type:
+              soup_manager.update_parse_type(parse_type=parse_type)
+            window['-SOURCECODE-'].update(value=soup_manager.soup)
+<<<<<<< HEAD
+=======
             window['-FIND_ALL_OUTPUT-'].update(value=soup_mgr.extract_text_sections())
         if event == 'Action':
             parse_type = values['-parse_type-']
             if parse_type != soup_mgr.parse_type:
               soup_mgr.update_parse_type(parse_type=parse_type)
             window['-SOURCECODE-'].update(value=soup_mgr.soup)
+>>>>>>> ba4baf2 (Deploy version 0.1.6.147 at 2025-09-07 09:40:38 UTC)
         elif event == 'Send Soup':
           return_data = values['-FIND_ALL_OUTPUT-']
           break