abstract-webtools 0.1.6.145__tar.gz → 0.1.6.146__tar.gz

{abstract_webtools-0.1.6.145 → abstract_webtools-0.1.6.146}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: abstract_webtools
-Version: 0.1.6.145
+Version: 0.1.6.146
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff

{abstract_webtools-0.1.6.145 → abstract_webtools-0.1.6.146}/setup.py

@@ -4,7 +4,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 setuptools.setup(
     name='abstract_webtools',
-    version='0.1.6.145',
+    version='0.1.6.146',
     author='putkoff',
     author_email='partners@abstractendeavors.com',
     description='Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.',

{abstract_webtools-0.1.6.145 → abstract_webtools-0.1.6.146}/src/abstract_webtools/managers/cipherManager.py

@@ -1,21 +1,20 @@
 class CipherManager:
     @staticmethod
-    def  get_default_ciphers()-> list:
+    def get_default_ciphers() -> list:
         return [
-            "ECDHE-RSA-AES256-GCM-SHA384", "ECDHE-ECDSA-AES256-GCM-SHA384",
-            "ECDHE-RSA-AES256-SHA384", "ECDHE-ECDSA-AES256-SHA384",
-            "ECDHE-RSA-AES256-SHA", "ECDHE-ECDSA-AES256-SHA",
-            "ECDHE-RSA-AES128-GCM-SHA256", "ECDHE-RSA-AES128-SHA256",
-            "ECDHE-ECDSA-AES128-GCM-SHA256", "ECDHE-ECDSA-AES128-SHA256",
-            "AES256-SHA", "AES128-SHA"
+            "ECDHE-RSA-AES256-GCM-SHA384","ECDHE-ECDSA-AES256-GCM-SHA384",
+            "ECDHE-RSA-AES256-SHA384","ECDHE-ECDSA-AES256-SHA384",
+            "ECDHE-RSA-AES256-SHA","ECDHE-ECDSA-AES256-SHA",
+            "ECDHE-RSA-AES128-GCM-SHA256","ECDHE-RSA-AES128-SHA256",
+            "ECDHE-ECDSA-AES128-GCM-SHA256","ECDHE-ECDSA-AES128-SHA256",
+            "AES256-SHA","AES128-SHA"
         ]
+    def __init__(self, cipher_list=None):
+        self.cipher_list = cipher_list or self.get_default_ciphers()
+        if isinstance(self.cipher_list, str):
+            self.cipher_list = [c.strip() for c in self.cipher_list.split(',') if c.strip()]
+        self.ciphers_string = ','.join(self.cipher_list) if self.cipher_list else ''
 
-    def __init__(self,cipher_list=None):
-        if cipher_list == None:
-            cipher_list=self.get_default_ciphers()
-        self.cipher_list = cipher_list
-        self.create_list()
-        self.ciphers_string = self.add_string_list()
     def add_string_list(self):
         if len(self.cipher_list)==0:
             return ''

{abstract_webtools-0.1.6.145 → abstract_webtools-0.1.6.146}/src/abstract_webtools/managers/crawlManager.py

@@ -119,14 +119,42 @@ if __name__ == "__main__":
     generator.run()
 
 class crawlManager:
-    def __init__(self, url=None, req_mgr=None, url_mgr=None, source_code=None, parse_type="html.parser"):
+    def __init__(self, url, req_mgr, url_mgr, source_code=None, parse_type="html.parser"):
+        self.url_mgr = url_mgr
+        self.req_mgr = req_mgr
         self.url = url
-        self.source_code = source_code
         self.parse_type = parse_type
-        self.url_mgr = url_mgr or urlManager(url=self.url)
-        self.req_mgr = req_mgr or requestManager(url_mgr=self.url_mgr)
-        self.get_new_source_and_url(url)
-        self.sime_map_mgr = SitemapGenerator(self.url_mgr.domain)
+        self.source_code = source_code or req_mgr.source_code
+        self.soup = BeautifulSoup(self.source_code or "", parse_type)
+        self.base_netloc = urlparse(self.url).netloc
+
+    def is_internal(self, link):
+        u = urlparse(link)
+        return (not u.netloc) or (u.netloc == self.base_netloc)
+
+    def links_on_page(self):
+        out = set()
+        for a in self.soup.find_all("a", href=True):
+            out.add(urljoin(self.url, a["href"]))
+        return out
+
+    def crawl(self, start=None, max_depth=2, _depth=0, visited=None, session=None):
+        start = start or self.url
+        visited = visited or set()
+        if _depth > max_depth or start in visited:
+            return visited
+        visited.add(start)
+
+        # fetch
+        r = self.req_mgr.session.get(start, timeout=30)
+        r.raise_for_status()
+        soup = BeautifulSoup(r.text, self.parse_type)
+
+        for a in soup.find_all("a", href=True):
+            link = urljoin(start, a["href"])
+            if self.is_internal(link) and link not in visited:
+                self.crawl(link, max_depth=max_depth, _depth=_depth+1, visited=visited)
+        return visited
     def get_new_source_and_url(self, url=None):
         """Fetches new source code and response for a given URL."""
         url = url
@@ -194,26 +222,7 @@ class crawlManager:
             return ('yearly', '0.3')
         return ('weekly', '1.0')
 
-    def crawl(self, url, max_depth=3, depth=1, visited=None):
-        """Recursively crawls the site up to max_depth and returns valid internal links."""
-        visited = visited or set()
-        if depth > max_depth or url in visited:
-            return []
-
-        visited.add(url)
-        try:
-            soup = get_soup(url)
-            links = []
-            for tag in soup.find_all('a', href=True):
-                link = urljoin(url, tag['href'])
-                if urlparse(link).netloc == urlparse(url).netloc and link not in visited:
-                    links.append(link)
-                    self.crawl(link, max_depth, depth + 1, visited)
-            return links
-        except Exception as e:
-            print(f"Error crawling {url}: {e}")
-            return []
-
+  
     def get_meta_info(self, url=None):
         """Fetches metadata, including title and meta tags, from the page."""
         url = url or self.url

abstract_webtools-0.1.6.146/src/abstract_webtools/managers/curlMgr.py

@@ -0,0 +1,26 @@
+import os, subprocess, requests
+
+def curl_download(website, destination_path, user_agent=None):
+    os.makedirs(os.path.dirname(destination_path), exist_ok=True)
+    ua = user_agent or ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                        "AppleWebKit/537.36 (KHTML, like Gecko) "
+                        "Chrome/91.0.4472.124 Safari/537.36")
+    subprocess.run([
+        "curl","-L","--output", destination_path,
+        "-H", f"User-Agent: {ua}",
+        "-H", "Accept: */*",
+        website
+    ], check=True)
+
+def requests_download(website, destination_path, headers=None):
+    os.makedirs(os.path.dirname(destination_path), exist_ok=True)
+    hdr = {"User-Agent": ("Mozilla/5.0 ... Chrome/91.0 Safari/537.36"),
+           "Accept": "*/*"}
+    if headers: hdr.update(headers)
+    r = requests.get(website, headers=hdr, allow_redirects=True, timeout=30)
+    r.raise_for_status()
+    with open(destination_path, "wb") as f:
+        f.write(r.content)
+
+if __name__ == "__main__":
+    pass  # no side effects

abstract_webtools-0.1.6.146/src/abstract_webtools/managers/meta_dump.py

@@ -0,0 +1,27 @@
+# meta_dump.py
+from abstract_webtools.managers.networkManager import NetworkManager
+from abstract_webtools.managers.userAgentManager import UserAgentManager
+from abstract_webtools.managers.soupManager.soupManager import soupManager
+import json, sys
+
+def dump_all_meta(url: str):
+    ua = UserAgentManager(browser="Chrome", operating_system="Windows")
+    net = NetworkManager(user_agent_manager=ua)
+
+    r = net.session.get(url, timeout=30)
+    r.raise_for_status()
+
+    sm = soupManager(url=url, source_code=r.text, req_mgr=net)
+    out = {
+        "url": url,
+        "title": sm.soup.title.string.strip() if sm.soup.title and sm.soup.title.string else None,
+        "meta": sm.all_meta(),
+        "citation": sm.citation_dict(),
+        "links": sm.all_links(),
+        "json_ld": sm.all_jsonld(),
+    }
+    print(json.dumps(out, indent=2, ensure_ascii=False))
+
+if __name__ == "__main__":
+    url = sys.argv[1]
+    dump_all_meta(url)

abstract_webtools-0.1.6.146/src/abstract_webtools/managers/networkManager.py

@@ -0,0 +1,50 @@
+from typing import Optional, List
+import requests
+from ..abstract_webtools import *
+from .sslManager import SSLManager
+from .cipherManager import CipherManager
+
+class TLSAdapter(HTTPAdapter):
+    def __init__(self, ssl_manager: SSLManager=None):
+        ssl_manager = ssl_manager or SSLManager()
+        self.ssl_context = ssl_manager.ssl_context
+        super().__init__()
+    def init_poolmanager(self, *args, **kwargs):
+        kwargs['ssl_context'] = self.ssl_context
+        return super().init_poolmanager(*args, **kwargs)
+
+class NetworkManager:
+    def __init__(self, user_agent_manager=None, ssl_manager=None, proxies=None, cookies=None,
+                 ciphers=None, certification: Optional[str]=None, ssl_options: Optional[List[str]]=None):
+        self.ua_mgr = user_agent_manager or UserAgentManager()
+        self.ssl_mgr = ssl_manager or SSLManager(
+            ciphers=ciphers or CipherManager().ciphers_string,
+            ssl_options=ssl_options,
+            certification=certification
+        )
+
+        self.session = requests.Session()
+        self.session.headers.update({
+            "User-Agent": self.ua_mgr.user_agent,
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.9",
+            "Connection": "keep-alive"
+        })
+        adapter = TLSAdapter(self.ssl_mgr)
+        self.session.mount("https://", adapter)
+        self.session.mount("http://", HTTPAdapter())
+
+        if proxies:
+            self.session.proxies = proxies
+        if cookies:
+            if isinstance(cookies, requests.cookies.RequestsCookieJar):
+                self.session.cookies = cookies
+            elif isinstance(cookies, dict):
+                jar = requests.cookies.RequestsCookieJar()
+                for k,v in cookies.items(): jar.set(k,v)
+                self.session.cookies = jar
+            # if string: up to you—parse or ignore
+
+        # retries (optional)
+        from requests.adapters import Retry
+        self.session.adapters['https://'].max_retries = Retry(total=5, backoff_factor=0.5, status_forcelist=[429,500,502,503,504])

{abstract_webtools-0.1.6.145 → abstract_webtools-0.1.6.146}/src/abstract_webtools/managers/soupManager/soupManager.py

@@ -1,6 +1,9 @@
 from ...abstract_webtools import *
 from ..urlManager import *
 from ..requestManager import *
+from bs4 import BeautifulSoup
+import re, json
+
 class soupManager:
     """
     SoupManager is a class for managing and parsing HTML source code using BeautifulSoup.
@@ -39,25 +42,49 @@ class soupManager:
         - The SoupManager class is designed for parsing HTML source code using BeautifulSoup.
         - It provides various methods to extract data and discover elements within the source code.
     """
-    def __init__(self,url=None,source_code=None,url_mgr=None,req_mgr=None,soup=None, parse_type="html.parser"):
-        self.soup=[]
-        url = get_url(url=url,url_mgr=url_mgr)
-        self.url_mgr = get_url_mgr(url=url,url_mgr=url_mgr)
-        self.url=self.url_mgr.url
-        self.req_mgr = get_req_mgr(req_mgr=req_mgr,url=self.url,url_mgr=self.url_mgr,source_code=source_code)
-        self.parse_type = parse_type
-        source_code = source_code or self.req_mgr.source_code or self.req_mgr.source_code_bytes
-        if source_code:
-            source_code = str(source_code)
-        self.source_code = source_code or ''
-        self.soup= soup or BeautifulSoup(self.source_code, self.parse_type)
-        self.all_tags_and_attribute_names = self.get_all_tags_and_attribute_names()
-        self.all_tags = self.all_tags_and_attribute_names.get('tags')
-        self.all_attribute_names = self.all_tags_and_attribute_names.get('attributes')
-        self.all_tags_and_attributes = self.all_tags + self.all_attribute_names
-        
-        self._all_links_data = None
-        self._meta_tags_data = None
+
+    def __init__(self, url=None, source_code=None, req_mgr=None, parse_type="html.parser"):
+        self.url = url
+        self.req_mgr = req_mgr
+        self.source_code = (source_code or (req_mgr.source_code if req_mgr else "")) or ""
+        self.soup = BeautifulSoup(self.source_code, parse_type)
+
+    def all_meta(self):
+        out = []
+        for m in self.soup.find_all("meta"):
+            row = {}
+            for k in ("name","property","http-equiv","itemprop","charset","content"):
+                v = m.get(k)
+                if v: row[k] = v
+            if row: out.append(row)
+        return out
+
+    def citation_dict(self):
+        out = {}
+        for m in self.soup.find_all("meta"):
+            k = (m.get("name") or m.get("property") or "").lower()
+            if k.startswith("citation_") and m.get("content"):
+                out.setdefault(k, []).append(m["content"])
+        return out
+
+    def all_links(self):
+        res = []
+        for l in self.soup.find_all("link"):
+            rel = l.get("rel")
+            if isinstance(rel, list): rel = " ".join(rel)
+            res.append({
+                "rel": rel, "href": l.get("href"),
+                "type": l.get("type"), "title": l.get("title"), "hreflang": l.get("hreflang")
+            })
+        return res
+
+    def all_jsonld(self):
+        blocks = []
+        for s in self.soup.find_all("script", type=re.compile("application/ld\\+json", re.I)):
+            txt = s.get_text(strip=True)
+            try: blocks.append(json.loads(txt))
+            except Exception: blocks.append({"raw": txt})
+        return blocks
     def re_initialize(self):
         self.soup= BeautifulSoup(self.source_code, self.parse_type)
         self._all_links_data = None

{abstract_webtools-0.1.6.145 → abstract_webtools-0.1.6.146}/src/abstract_webtools/managers/sslManager.py

@@ -1,12 +1,21 @@
 from ..abstract_webtools import *
+# sslManager.py
+from ..abstract_webtools import *  # must expose ssl, ssl_
+from .cipherManager import CipherManager  # be explicit, safer
+
 class SSLManager:
     def __init__(self, ciphers=None, ssl_options=None, certification=None):
         self.ciphers = ciphers or CipherManager().ciphers_string
         self.ssl_options = ssl_options or self.get_default_ssl_settings()
         self.certification = certification or ssl.CERT_REQUIRED
         self.ssl_context = self.get_context()
+
     def get_default_ssl_settings(self):
         return ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1 | ssl.OP_NO_COMPRESSION
-    def get_context(self):
-        return ssl_.create_urllib3_context(ciphers=self.ciphers, cert_reqs=self.certification, options=self.ssl_options)
 
+    def get_context(self):
+        return ssl_.create_urllib3_context(
+            ciphers=self.ciphers,
+            cert_reqs=self.certification,
+            options=self.ssl_options
+        )

abstract_webtools-0.1.6.146/src/abstract_webtools/managers/userAgentManager.py

@@ -0,0 +1,51 @@
+# userAgentManager.py
+from ..abstract_webtools import *
+import random
+
+operating_systems = ['Macintosh','Windows','Linux']
+browsers = ['Firefox','Chrome','IceDragon','Waterfox','Gecko','Safari','MetaSr']
+
+def _pick(val, options):
+    if not val: return options[0]
+    if val in options: return val
+    l = val.lower()
+    for o in options:
+        if l in o.lower():
+            return o
+    return options[0]
+
+class UserAgentManager:
+    def __init__(self, operating_system=None, browser=None, version=None, user_agent=None):
+        self.operating_system = _pick(operating_system, operating_systems)
+        self.browser = _pick(browser, browsers)
+        self.version = version or '42.0'
+        self.user_agent = user_agent or self.get_user_agent()
+        self.header = {"user-agent": self.user_agent}
+
+    @staticmethod
+    def user_agent_db():
+        from ..big_user_agent_list import big_user_agent_dict
+        return big_user_agent_dict
+
+    def get_user_agent(self):
+        ua_db = self.user_agent_db()
+        os_db = ua_db.get(self.operating_system) or random.choice(list(ua_db.values()))
+        br_db = os_db.get(self.browser) or random.choice(list(os_db.values()))
+        if self.version in br_db:
+            return br_db[self.version]
+        return random.choice(list(br_db.values()))
+
+class UserAgentManagerSingleton:
+    _instance = None
+
+    @staticmethod
+    def get_instance(**kwargs):
+        ua = kwargs.get("user_agent")
+        if UserAgentManagerSingleton._instance is None:
+            UserAgentManagerSingleton._instance = UserAgentManager(**kwargs)
+        else:
+            # rebuild if user_agent explicitly changed
+            inst = UserAgentManagerSingleton._instance
+            if ua and ua != inst.user_agent:
+                UserAgentManagerSingleton._instance = UserAgentManager(**kwargs)
+        return UserAgentManagerSingleton._instance

{abstract_webtools-0.1.6.145 → abstract_webtools-0.1.6.146}/src/abstract_webtools.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: abstract_webtools
-Version: 0.1.6.145
+Version: 0.1.6.146
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff

{abstract_webtools-0.1.6.145 → abstract_webtools-0.1.6.146}/src/abstract_webtools.egg-info/SOURCES.txt

@@ -28,6 +28,7 @@ src/abstract_webtools/managers/curlMgr.py
 src/abstract_webtools/managers/domainManager.py
 src/abstract_webtools/managers/dynamicRateLimiter.py
 src/abstract_webtools/managers/get_test.py
+src/abstract_webtools/managers/meta_dump.py
 src/abstract_webtools/managers/mySocketClient.py
 src/abstract_webtools/managers/networkManager.py
 src/abstract_webtools/managers/seleneumManager.py

abstract_webtools-0.1.6.145/src/abstract_webtools/managers/curlMgr.py

@@ -1,48 +0,0 @@
-import os
-import requests
-import os
-import subprocess
-import stat
-
-def get_site(website, destination_dir, filename):
-    # Ensure the directory exists
-    os.makedirs(destination_dir, exist_ok=True)
-
-    # Adjust directory permissions if needed (e.g. rwxr-xr-x -> 0o755)
-    os.chmod(destination_dir, 0o755)
-
-    # Construct the complete file path
-    destination_path = os.path.join(destination_dir, filename)
-
-    # Use curl to download the site
-    # The example user-agent is arbitrary; you can change it to your needs
-    os.system(
-        f'curl -L --output "{destination_path}" '
-        f'-H "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
-        f'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 '
-        f'Safari/537.36" -H "Accept: */*" "{website}"'
-    )
-
-def download_site(website, destination_dir, filename):
-    os.makedirs(destination_dir, exist_ok=True)
-    os.chmod(destination_dir, 0o755)  # set directory permissions if needed
-
-    destination_path = os.path.join(destination_dir, filename)
-
-    # GET the resource
-    response = requests.get(website, headers={
-        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-                      "AppleWebKit/537.36 (KHTML, like Gecko) "
-                      "Chrome/91.0.4472.124 Safari/537.36",
-        "Accept": "*/*"
-    }, allow_redirects=True)
-
-    # Raise an exception if the download fails
-    response.raise_for_status()
-
-    # Write content to file
-    with open(destination_path, "wb") as f:
-        f.write(response.content)
-website = 'https://www.pornhub.com'
-destination = '/home/computron/Documents/doge'
-get_site(website,destination,'doge')

abstract_webtools-0.1.6.145/src/abstract_webtools/managers/networkManager.py

@@ -1,15 +0,0 @@
-from ..abstract_webtools import *
-from ..big_user_agent_list import *
-class NetworkManager:
-    def __init__(self, user_agent_manager=None,ssl_manager=None, tls_adapter=None,user_agent=None,proxies=None,cookies=None,ciphers=None, certification: Optional[str] = None, ssl_options: Optional[List[str]] = None):
-        if ssl_manager == None:
-            ssl_manager = SSLManager(ciphers=ciphers, ssl_options=ssl_options, certification=certification)
-        self.ssl_manager=ssl_manager
-        if tls_adapter == None:
-            tls_adapter=TLSAdapter(ssl_manager=ssl_manager,ciphers=ciphers, certification=certification, ssl_options=ssl_options)
-        self.tls_adapter=tls_adapter
-        self.ciphers=tls_adapter.ciphers
-        self.certification=tls_adapter.certification
-        self.ssl_options=tls_adapter.ssl_options
-        self.proxies=None or {}
-        self.cookies=cookies or "cb4c883efc59d0e990caf7508902591f4569e7bf-1617321078-0-150"

abstract_webtools-0.1.6.145/src/abstract_webtools/managers/userAgentManager.py

@@ -1,60 +0,0 @@
-from ..abstract_webtools import *
-import random
-operating_systems = ['Macintosh','Windows','Linux']
-browsers = ['Firefox','Chrome','IceDragon','Waterfox','Gecko','Safari','MetaSr']
-def get_itter(iter_input,itter_list):
-    if not iter_input:
-        return itter_list[0]
-    if iter_input in itter_list:
-        return iter_input
-    iter_input_lower = iter_input.lower()
-    for itter in itter_list:
-        itter_lower = itter.lower()
-        if iter_input_lower in itter_lower:
-            return itter
-    return itter_list[0]
-def get_browser(browser=None):
-    return get_itter(browser,browsers)
-def get_operating_system(operating_system=None):
-    return get_itter(operating_system,operating_systems)
-class UserAgentManager:
-    def __init__(self, operating_system=None, browser=None, version=None,user_agent=None):
-        self.operating_system = get_operating_system(operating_system=operating_system)
-        self.browser = get_browser(browser=browser)
-        self.version = version or '42.0'
-        self.user_agent = user_agent or self.get_user_agent()
-        self.header = self.user_agent_header()
-    @staticmethod
-    def user_agent_db():
-        from ..big_user_agent_list import big_user_agent_dict
-        return big_user_agent_dict
-
-    def get_user_agent(self):
-        ua_db = self.user_agent_db()
-
-        if self.operating_system and self.operating_system in ua_db:
-            operating_system_db = ua_db[self.operating_system]
-        else:
-            operating_system_db = random.choice(list(ua_db.values()))
-
-        if self.browser and self.browser in operating_system_db:
-            browser_db = operating_system_db[self.browser]
-        else:
-            browser_db = random.choice(list(operating_system_db.values()))
-
-        if self.version and self.version in browser_db:
-            return browser_db[self.version]
-        else:
-            return random.choice(list(browser_db.values()))
-
-    def user_agent_header(self):
-        return {"user-agent": self.user_agent}
-class UserAgentManagerSingleton:
-    _instance = None
-    @staticmethod
-    def get_instance(user_agent=UserAgentManager().get_user_agent()[0]):
-        if UserAgentManagerSingleton._instance is None:
-            UserAgentManagerSingleton._instance = UserAgentManager(user_agent=user_agent)
-        elif UserAgentManagerSingleton._instance.user_agent != user_agent:
-            UserAgentManagerSingleton._instance = UserAgentManager(user_agent=user_agent)
-        return UserAgentManagerSingleton._instance