abstract-webtools 0.1.5.81__tar.gz → 0.1.5.82__tar.gz

{abstract_webtools-0.1.5.81 → abstract_webtools-0.1.5.82}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: abstract_webtools
-Version: 0.1.5.81
+Version: 0.1.5.82
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff
@@ -13,6 +13,10 @@ Classifier: Programming Language :: Python :: 3.11
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: abstract_utilities>=0.2.2.30
+Requires-Dist: PySimpleGUI>=4.60.5
+Requires-Dist: urllib3>=2.0.4
+Requires-Dist: requests>=2.31.0
 
 # Abstract WebTools
 Provides utilities for inspecting and parsing web content, including React components and URL utilities, with enhanced capabilities for managing HTTP requests and TLS configurations.

{abstract_webtools-0.1.5.81 → abstract_webtools-0.1.5.82}/setup.py

@@ -4,7 +4,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 setuptools.setup(
     name='abstract_webtools',
-    version='0.1.5.81',
+    version='0.1.5.82',
     author='putkoff',
     author_email='partners@abstractendeavors.com',
     description='Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.',

abstract_webtools-0.1.5.82/src/abstract_webtools/__init__.py

@@ -0,0 +1,3 @@
+from .abstract_webtools import get_url_mgr
+from .managers import *
+from .url_grabber import url_grabber_component

abstract_webtools-0.1.5.82/src/abstract_webtools/abstract_webtools.py

@@ -0,0 +1,100 @@
+"""
+# `abstract_webtools.py` Documentation
+
+This script, `abstract_webtools.py`, is a component of the `abstract_webtools` module and is a part of the `abstract_essentials` package. It provides a set of tools and functions to interact with and parse web content.
+
+## Contents
+
+1. **Imports**
+   - Essential modules and classes for web requests, SSL configurations, and URL parsing are imported at the beginning.
+
+2. **Core Functions**
+
+   - `get_status(url: str) -> int or None`:
+     Fetches the HTTP status code for a given URL.
+
+   - `clean_url(url: str) -> list`:
+     Returns variations of the given URL with different protocols.
+
+   - `get_correct_url(url: str, session: requests.Session) -> str or None`:
+     Identifies the correct URL from possible variations using HTTP requests.
+
+   - `try_request(url: str, session: requests.Session) -> requests.Response or None`:
+     Attempts to make an HTTP request to a given URL.
+
+   - `is_valid(url: str) -> bool`:
+     Validates if a given URL is structurally correct.
+
+   - `desktop_user_agents() -> list`:
+     Returns a list of popular desktop user-agent strings.
+
+   - `get_user_agent(user_agent: str) -> dict`:
+     Returns a dictionary containing the user-agent header.
+
+3. **TLSAdapter Class**
+
+   A custom HTTPAdapter class that manages SSL options and ciphers for web requests.
+
+   - `TLSAdapter.__init__(self, ssl_options: int)`: 
+     Initializes the adapter with specific SSL options.
+
+   - Several methods to handle cipher strings, creation of cipher strings, and initialization of the pool manager with custom SSL configurations.
+
+4. **Advanced Web Functions**
+
+   - `get_Source_code(url: str, user_agent: str) -> str or None`:
+     Retrieves the source code of a website with a custom user-agent.
+
+   - `parse_react_source(url: str) -> list`:
+     Extracts JavaScript and JSX source code from the specified URL.
+
+   - `get_all_website_links(url: str) -> list`:
+     Lists all the internal URLs found on a specific webpage.
+
+   - `parse_all(url: str) -> dict`:
+     Parses source code to extract details about elements, attributes, and class names.
+
+   - `extract_elements(url: str, element_type: str, attribute_name: str, class_name: str)`:
+     Extracts specific portions of source code based on provided filters. The function signature seems to be cut off, so the full details aren't available.
+
+## Usage
+
+The functions and classes provided in this module allow users to interact with websites, from simple actions like getting the status code of a URL to more advanced functionalities such as parsing ReactJS source codes or extracting specific HTML elements from a website.
+
+To utilize this module, simply import the required function or class and use it in your application. The functions have been designed to be intuitive and the provided docstrings give clear guidance on their usage.
+
+Author: putkoff
+Version: 1.0
+"""
+# -*- coding: UTF-8 -*-
+# Google Chrome Driver
+import os
+import ssl
+import re
+import yt_dlp
+import socket
+import shutil
+import logging
+import requests
+from bs4 import BeautifulSoup
+from selenium import webdriver
+import xml.etree.ElementTree as ET
+from typing import Optional,List,Union
+from requests.adapters import HTTPAdapter
+from urllib.parse import urlparse, urljoin
+from requests.packages.urllib3.util import ssl_
+from requests.packages.urllib3.poolmanager import PoolManager
+from urllib.parse import urlparse, parse_qs
+import time
+import requests
+from .managers import *
+from abstract_utilities import get_time_stamp,get_sleep,sleep_count_down,eatInner,eatAll,eatOuter,ThreadManager
+logging.basicConfig(level=logging.INFO)
+def try_request(request):
+    try:
+        respnse = requests.get(url)
+    except Exception as e:
+        print(f'request for url failed: {e}')
+        response = None
+    return response
+

abstract_webtools-0.1.5.82/src/abstract_webtools/managers/__init__.py

@@ -0,0 +1,14 @@
+from .cipherManager import *
+from .crawlManager import *
+from .dynamicRateLimiter import *
+from .linkManager import *
+from .mySocketClient import *
+from .networkManager import *
+from .requestManager import *
+from .soupManager import *
+from .sslManager import *
+from .tlsAdapter import *
+from .urlManager import *
+from .userAgentManager import *
+from .videoDownloader import *
+from .seleniumManager import *

abstract_webtools-0.1.5.82/src/abstract_webtools/managers/cipherManager.py

@@ -0,0 +1,38 @@
+class CipherManager:
+    @staticmethod
+    def  get_default_ciphers()-> list:
+        return [
+            "ECDHE-RSA-AES256-GCM-SHA384", "ECDHE-ECDSA-AES256-GCM-SHA384",
+            "ECDHE-RSA-AES256-SHA384", "ECDHE-ECDSA-AES256-SHA384",
+            "ECDHE-RSA-AES256-SHA", "ECDHE-ECDSA-AES256-SHA",
+            "ECDHE-RSA-AES128-GCM-SHA256", "ECDHE-RSA-AES128-SHA256",
+            "ECDHE-ECDSA-AES128-GCM-SHA256", "ECDHE-ECDSA-AES128-SHA256",
+            "AES256-SHA", "AES128-SHA"
+        ]
+
+    def __init__(self,cipher_list=None):
+        if cipher_list == None:
+            cipher_list=self.get_default_ciphers()
+        self.cipher_list = cipher_list
+        self.create_list()
+        self.ciphers_string = self.add_string_list()
+    def add_string_list(self):
+        if len(self.cipher_list)==0:
+            return ''
+        return','.join(self.cipher_list)
+    def create_list(self):
+        if self.cipher_list == None:
+            self.cipher_list= []
+        elif isinstance(self.cipher_list, str):
+            self.cipher_list=self.cipher_list.split(',')
+        if isinstance(self.cipher_list, str):
+            self.cipher_list=[self.cipher_list]
+class CipherManagerSingleton:
+    _instance = None
+    @staticmethod
+    def get_instance(cipher_list=None):
+        if CipherManagerSingleton._instance is None:
+            CipherManagerSingleton._instance = CipherManager(cipher_list=cipher_list)
+        elif CipherManagerSingleton._instance.cipher_list != cipher_list:
+            CipherManagerSingleton._instance = CipherManager(cipher_list=cipher_list)
+        return CipherManagerSingleton._instance

abstract_webtools-0.1.5.82/src/abstract_webtools/managers/crawlManager.py

@@ -0,0 +1,187 @@
+class CrawlManager:
+    def __init__(self,url=None,source_code=None,parse_type="html.parser"):
+        self.url=url
+        self.source_code=source_code
+        self.parse_type=parse_type
+        get_new_source_and_url(self,url)
+    def get_new_source_and_url(self,url=None):
+        if url == None:
+            url = self.url
+        self.response = self.request_mgr.response
+        self.source_code=self.request_mgr.source_code
+    def get_classes_and_meta_info():
+        class_name_1,class_name_2, class_value = 'meta','class','property','og:image'
+        attrs = 'href','src'
+        unique_classes, images=discover_classes_and_images(self,tag_name,class_name_1,class_name_2,class_value,attrs)
+        return unique_classes, images
+    def extract_links_from_url(self):
+        """
+        Extracts all href and src links from a given URL's source code.
+
+        Args:
+            base_url (str): The URL from which to extract links.
+
+        Returns:
+            dict: Dictionary containing image links and external links under the parent page.
+        """
+        agg_js = {'images':[],'external_links':[]}
+        
+        if self.response != None:
+            attrs = 'href','src'
+            href_links,src_links='',''
+            links = [href_links,src_links]
+            for i,each in enumerate(attrs):
+                 links[i]= [a[attr[i]] for a in get_find_all_with_attributes(self, attrs[i])]
+            # Convert all links to absolute links
+            absolute_links = [(url, link) for link in links[0] + links[1]]
+            # Separate images and external links
+            images = [link for link in absolute_links if link.lower().endswith(('.png', '.jpg', '.jpeg', '.gif', '.svg', '.webp'))]
+            external_links = [link for link in absolute_links if urlparse(link).netloc != urlparse(url).netloc]
+            agg_js['images']=images
+            agg_js['external_links']=external_links
+           
+        return agg_js
+
+
+    def correct_xml(xml_string):
+        # Parse the XML string
+        root = ET.fromstring(xml_string)
+
+        # Loop through each <image:loc> element and correct its text if needed
+        for image_loc in root.findall(".//image:loc", namespaces={'image': 'http://www.google.com/schemas/sitemap-image/1.1'}):
+            # Replace '&' with '&amp;' in the element's text
+            if '&' in image_loc.text:
+                image_loc.text = image_loc.text.replace('&', '&amp;')
+
+        # Convert the corrected XML back to string
+        corrected_xml = ET.tostring(root, encoding='utf-8').decode('utf-8')
+        return corrected_xml
+
+
+    def determine_values(self):
+        # This is just a mockup. In a real application, you'd analyze the URL or its content.
+
+        # Assuming a blog site
+        if 'blog' in self.url:
+            if '2023' in self.url:  # Assuming it's a current year article
+                return ('weekly', '0.8')
+            else:
+                return ('monthly', '0.6')
+        elif 'contact' in self.url:
+            return ('yearly', '0.3')
+        else:  # Homepage or main categories
+            return ('weekly', '1.0')
+    def crawl(url, max_depth=3, depth=1):
+        
+        if depth > max_depth:
+            return []
+
+        if url in visited:
+            return []
+
+        visited.add(url)
+
+        try:
+            
+            links = [a['href'] for a in self.soup.find_all('a', href=True)]
+            valid_links = []
+
+            for link in links:
+                parsed_link = urlparse(link)
+                base_url = "{}://{}".format(parsed_link.scheme, parsed_link.netloc)
+            
+                if base_url == url:  # Avoiding external URLs
+                    final_link = urljoin(url, parsed_link.path)
+                    if final_link not in valid_links:
+                        valid_links.append(final_link)
+
+            for link in valid_links:
+                crawl(link, max_depth, depth+1)
+
+            return valid_links
+
+        except Exception as e:
+            print(f"Error crawling {url}: {e}")
+            return []
+
+
+    # Define or import required functions here, like get_all_website_links, determine_values, 
+    # discover_classes_and_meta_images, and extract_links_from_url.
+    def get_meta_info(self):
+        
+        meta_info = {}
+        # Fetch the title if available
+        title_tag = parse_title()
+        if title_tag:
+            meta_info["title"] = title_tag
+        # Fetch meta tags
+        for meta_tag in soup.find_all('meta'):
+            name = meta_tag.get('name') or meta_tag.get('property')
+            if name:
+                content = meta_tag.get('content')
+                if content:
+                    meta_info[name] = content
+
+        return meta_info
+    def generate_sitemap(self,domain):
+        
+        with open('sitemap.xml', 'w', encoding='utf-8') as f:
+            string = '<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:image="http://www.google.com/schemas/sitemap-image/1.1">\n'
+            
+            for url in self.all_site_links:
+                string += f'  <url>\n    <loc>{url}</loc>\n'
+                preprocess=[]
+                self.get_new_source_and_url(url=url)
+                links = extract_links_from_url(url)
+                
+                for img in links['images']:
+                    if str(img).lower() not in preprocess:
+                        try:
+                            escaped_img = img.replace('&', '&amp;')
+
+                            str_write = f'    <image:image>\n      <image:loc>{escaped_img}</image:loc>\n    </image:image>\n'
+                            string += str_write
+                        except:
+                            pass
+                        preprocess.append(str(img).lower())
+                frequency, priority = determine_values(url)
+                string += f'    <changefreq>{frequency}</changefreq>\n'
+                string += f'    <priority>{priority}</priority>\n'
+                string += f'  </url>\n'
+                
+            string += '</urlset>\n'
+            f.write(string)            
+        # Output summary
+        print(f'Sitemap saved to sitemap.xml with {len(urls)} URLs.')
+        
+        # Output class and link details
+        for url in urls:
+            print(f"\nDetails for {url}:")
+            classes, meta_img_refs = discover_classes_and_meta_images(url)
+
+            print("\nClasses with href or src attributes:")
+            for class_name in classes:
+                print(f"\t{class_name}")
+            
+            print("\nMeta Image References:")
+            for img_ref in meta_img_refs:
+                print(f"\t{img_ref}")
+            
+            links = extract_links_from_url(url)
+
+            print("\nImages:")
+            for img in links['images']:
+                print(f"\t{img}")
+            
+            print("\nExternal Links:")
+            for ext_link in links['external_links']:
+                print(f"\t{ext_link}")
+class CrawlManagerSingleton():
+    _instance = None
+    @staticmethod
+    def get_instance(url=None,source_code=None,parse_type="html.parser"):
+        if CrawlManagerSingleton._instance is None:
+            CrawlManagerSingleton._instance = CrawlManager(url=url,parse_type=parse_type,source_code=source_code)
+        elif parse_type != CrawlManagerSingleton._instance.parse_type or url != CrawlManagerSingleton._instance.url  or source_code != CrawlManagerSingleton._instance.source_code:
+            CrawlManagerSingleton._instance = CrawlManager(url=url,parse_type=parse_type,source_code=source_code)
+        return CrawlManagerSingleton._instance

abstract_webtools-0.1.5.82/src/abstract_webtools/managers/domainManager.py

@@ -0,0 +1,48 @@
+from ..abstract_webtools import *
+class domainManager(metaclass=SingletonMeta):
+    def __init__(self, url):
+        if not hasattr(self, 'initialized'):  # Prevent reinitialization
+            self.initialized = True
+            parsed_url = urlparse(url)
+            self.domain = parsed_url.netloc
+            self.scheme = parsed_url.scheme
+    def get_url_to_path(self, url):
+        url = eatAll(str(url),['',' ','\n','\t','\\','/'])
+        parsed_url = urlparse(url)
+        if 'data:image' in url:
+            input(url)
+        if parsed_url.netloc == self.domain:
+            paths = parsed_url.path.split('/')
+            dir_path =self.site_dir
+            for path in paths[:-1]:
+                dir_path = os.path.join(dir_path, path)
+                os.makedirs(dir_path, exist_ok=True)
+        #if 'svg' in url:
+        #$    input(url)
+         #   dir_path = get_image_name('contents',directory=dir_path,ext='png',url=item_url)
+
+
+        self.page_type.append(os.path.splitext(paths[-1])[-1] or 'html' if len(self.page_type) == 0 else self.page_type[-1])
+            
+        dir_path = os.path.join(dir_path, paths[-1])
+        return dir_path
+
+    def saved_url_check(self, url):
+  
+        path = self.get_url_to_path(url)
+        return path
+
+    def get_with_netloc(self, url):
+        parsed_url = urlparse(url)
+        if parsed_url.netloc == '':
+            url = f"{self.scheme}://{self.domain}/{url.strip()}"
+        return url
+
+    def get_driver(self, url):
+        if url and url not in self.drivers:
+            chrome_options = Options()
+            chrome_options.add_argument("--headless")
+            driver = webdriver.Chrome(options=chrome_options)
+            self.drivers[url] = driver
+            driver.get(url)
+        return self.drivers[url]

abstract_webtools-0.1.5.82/src/abstract_webtools/managers/dynamicRateLimiter.py

@@ -0,0 +1,138 @@
+class DynamicRateLimiterManager:
+    def __init__(self, service_name='ethereum'):
+        self.services = {}
+        self.service_name = service_name
+        self.add_service(service_name)
+
+    def add_service(self, service_name="default", low_limit=10, high_limit=30, limit_epoch=60, starting_tokens=10, epoch_cycle_adjustment=True):
+        if service_name in self.services:
+            print(f"Service {service_name} already exists!")
+            return
+        self.services[service_name] = DynamicRateLimiter(low_limit=low_limit, high_limit=high_limit, limit_epoch=limit_epoch, starting_tokens=starting_tokens, epoch_cycle_adjustment=epoch_cycle_adjustment)
+
+    def request(self, request_url, service_name=None):
+        service_name = service_name or self.service_name
+        if service_name not in self.services:
+            self.add_service(service_name)
+
+        limiter = self.services[service_name]
+
+        while True:
+            if limiter.request():
+                response = requests.get(request_url)  # Actual request
+                if response.status_code == 200:
+                    limiter.request_tracker(True)
+                    return response.json()
+                elif response.status_code == 429:
+                    limiter.request_tracker(False)
+                    print(f"Rate limited by {service_name}. Adjusting limit and retrying...")
+                    time.sleep(limiter.get_sleep()["current_sleep"])
+                else:
+                    print(f"Unexpected response: {response.status_code}. Message: {response.text}")
+                    return None
+            else:
+                print(f"Rate limit reached for {service_name}. Waiting for the next epoch...")
+                time.sleep(limiter.get_sleep()["current_sleep"])
+
+    def log_request(self, service_name, success):
+        print(f"[{service_name}] Request {'succeeded' if success else 'denied'}. Current tokens: {self.services[service_name].get_current_tokens()}")
+
+class DynamicRateLimiter:
+    def __init__(self, low_limit, high_limit, limit_epoch, starting_tokens=None,epoch_cycle_adjustment:int=None):
+        self.low_limit = low_limit
+        self.high_limit = high_limit
+        self.limit_epoch = limit_epoch  # in seconds
+        self.request_status_json = {"succesful":[],"unsuccesful":[],"last_requested":get_time_stamp(),"first_requested":get_time_stamp(),"epoch_left":self.limit_epoch,"last_fail":get_time_stamp(),"count_since_fail":0}
+        self.current_limit = starting_tokens or low_limit  # Default to high_limit if starting_tokens isn't provided
+        self.epoch_cycle_adjustment = epoch_cycle_adjustment
+        # Additional attributes for tracking adjustment logic
+        self.last_adjusted_time = get_time_stamp()
+        self.successful_epochs_since_last_adjustment = 0
+        self.request_count_in_current_epoch = 0
+
+    def _refill_tokens(self):
+        time_since_last_request = get_time_stamp() - self.request_status_json["last_requested"]
+        new_tokens = (time_since_last_request / self.limit_epoch) * self.current_limit
+        self.tokens = min(self.current_limit, self.get_current_tokens())
+    def request_tracker(self,success):
+        if success:
+            self.request_status_json["succesful"].append(get_time_stamp())
+        else:
+            self.request_status_json["unsuccesful"].append(get_time_stamp())
+            self.request_status_json["last_fail"]=get_time_stamp()
+            self.request_status_json["count_since_fail"]=0
+            self.adjust_limit()
+        self.request_status_json["last_requested"]=get_time_stamp()
+    def calculate_tokens(self):
+        successful = []
+        for each in self.request_status_json["succesful"]:
+            if (get_time_stamp() - each)<self.limit_epoch:
+                successful.append(each)
+        self.request_status_json["succesful"]=successful
+        unsuccessful = []
+        for each in self.request_status_json["unsuccesful"]:
+            if (get_time_stamp() - each)<self.limit_epoch:
+                unsuccessful.append(each)
+        self.request_status_json["unsuccesful"]=unsuccessful
+        if len(successful)==0 and len(unsuccessful)==0:
+            pass
+        elif len(successful)!=0 and len(unsuccessful)==0:
+            self.request_status_json["first_requested"] = successful[0]
+        elif len(successful)==0 and len(unsuccessful)!=0:
+            self.request_status_json["first_requested"] = unsuccessful[0]
+        else:
+            self.request_status_json["first_requested"] = min(unsuccessful[0],successful[0])
+        self.request_status_json["epoch_left"]=self.limit_epoch-(self.request_status_json["last_requested"]-self.request_status_json["first_requested"])
+        
+        return self.request_status_json
+    def get_current_tokens(self):
+        self.request_status_json = self.calculate_tokens()
+        total_requests = len(self.request_status_json["succesful"])+len(self.request_status_json["unsuccesful"])
+        return max(0,self.current_limit-total_requests)
+    def get_sleep(self):
+        self.request_status_json = self.calculate_tokens()
+        self.request_status_json["current_sleep"]=self.request_status_json["epoch_left"]/max(1,self.get_current_tokens())
+        return self.request_status_json
+    def request(self):
+        self._refill_tokens()
+        if self.tokens > 0:
+            return True  # The request can be made
+        else:
+            if self.tokens == 0:
+                self.request_status_json["count_since_fail"]+=1
+                if self.epoch_cycle_adjustment != None:
+                    if self.request_status_json["count_since_fail"] >=self.epoch_cycle_adjustment:
+                        self.current_limit=min(self.current_limit+1,self.high_limit)
+            return False  # The request cannot be made
+    def _adjust_limit(self):
+        current_time = get_time_stamp()
+        if current_time - self.last_adjusted_time >= self.limit_epoch:
+            if len(self.clear_epoch()["succesful"]) >= self.tokens:
+                # We hit the rate limit this epoch, decrease our limit
+                self.tokens = max(1, self.tokens - 1)
+            else:
+                self.successful_epochs_since_last_adjustment += 1
+                if self.successful_epochs_since_last_adjustment >= 5:
+                    # We've had 5 successful epochs, increase our limit
+                    self.current_limit = min(self.high_limit, self.tokens + 1)
+                    self.successful_epochs_since_last_adjustment = 0
+            
+            # Reset our counters for the new epoch
+            self.last_adjusted_time = current_time
+            self.request_count_in_current_epoch = 0
+    def adjust_limit(self):
+        # Set the tokens to succesful requests_made - 1
+        self.tokens = len(self.calculate_tokens()["succesful"])
+
+        # Adjust the high_limit
+        self.current_limit = self.tokens
+
+        # Log the adjustment
+        print(f"Adjusted tokens to: {self.tokens} and high_limit to: {self.current_limit}")
+class DynamicRateLimiterManagerSingleton:
+    _instance = None
+    @staticmethod
+    def get_instance(service_name="default", low_limit=10, high_limit=30, limit_epoch=60,starting_tokens=10,epoch_cycle_adjustment=True):
+        if DynamicRateLimiterManagerSingleton._instance is None:
+            DynamicRateLimiterManagerSingleton._instance = DynamicRateLimiterManager(service_name=service_name, low_limit=low_limit, high_limit=limit_epoch, limit_epoch=60,starting_tokens=starting_tokens,epoch_cycle_adjustment=epoch_cycle_adjustment)
+        return DynamicRateLimiterManagerSingleton._instance