abstract-webtools 0.1.5.8__py3-none-any.whl → 0.1.5.82__py3-none-any.whl

abstract_webtools/managers/__init__.py

@@ -0,0 +1,14 @@
+from .cipherManager import *
+from .crawlManager import *
+from .dynamicRateLimiter import *
+from .linkManager import *
+from .mySocketClient import *
+from .networkManager import *
+from .requestManager import *
+from .soupManager import *
+from .sslManager import *
+from .tlsAdapter import *
+from .urlManager import *
+from .userAgentManager import *
+from .videoDownloader import *
+from .seleniumManager import *

abstract_webtools/managers/cipherManager.py

@@ -0,0 +1,38 @@
+class CipherManager:
+    @staticmethod
+    def  get_default_ciphers()-> list:
+        return [
+            "ECDHE-RSA-AES256-GCM-SHA384", "ECDHE-ECDSA-AES256-GCM-SHA384",
+            "ECDHE-RSA-AES256-SHA384", "ECDHE-ECDSA-AES256-SHA384",
+            "ECDHE-RSA-AES256-SHA", "ECDHE-ECDSA-AES256-SHA",
+            "ECDHE-RSA-AES128-GCM-SHA256", "ECDHE-RSA-AES128-SHA256",
+            "ECDHE-ECDSA-AES128-GCM-SHA256", "ECDHE-ECDSA-AES128-SHA256",
+            "AES256-SHA", "AES128-SHA"
+        ]
+
+    def __init__(self,cipher_list=None):
+        if cipher_list == None:
+            cipher_list=self.get_default_ciphers()
+        self.cipher_list = cipher_list
+        self.create_list()
+        self.ciphers_string = self.add_string_list()
+    def add_string_list(self):
+        if len(self.cipher_list)==0:
+            return ''
+        return','.join(self.cipher_list)
+    def create_list(self):
+        if self.cipher_list == None:
+            self.cipher_list= []
+        elif isinstance(self.cipher_list, str):
+            self.cipher_list=self.cipher_list.split(',')
+        if isinstance(self.cipher_list, str):
+            self.cipher_list=[self.cipher_list]
+class CipherManagerSingleton:
+    _instance = None
+    @staticmethod
+    def get_instance(cipher_list=None):
+        if CipherManagerSingleton._instance is None:
+            CipherManagerSingleton._instance = CipherManager(cipher_list=cipher_list)
+        elif CipherManagerSingleton._instance.cipher_list != cipher_list:
+            CipherManagerSingleton._instance = CipherManager(cipher_list=cipher_list)
+        return CipherManagerSingleton._instance

abstract_webtools/managers/crawlManager.py

@@ -0,0 +1,187 @@
+class CrawlManager:
+    def __init__(self,url=None,source_code=None,parse_type="html.parser"):
+        self.url=url
+        self.source_code=source_code
+        self.parse_type=parse_type
+        get_new_source_and_url(self,url)
+    def get_new_source_and_url(self,url=None):
+        if url == None:
+            url = self.url
+        self.response = self.request_mgr.response
+        self.source_code=self.request_mgr.source_code
+    def get_classes_and_meta_info():
+        class_name_1,class_name_2, class_value = 'meta','class','property','og:image'
+        attrs = 'href','src'
+        unique_classes, images=discover_classes_and_images(self,tag_name,class_name_1,class_name_2,class_value,attrs)
+        return unique_classes, images
+    def extract_links_from_url(self):
+        """
+        Extracts all href and src links from a given URL's source code.
+
+        Args:
+            base_url (str): The URL from which to extract links.
+
+        Returns:
+            dict: Dictionary containing image links and external links under the parent page.
+        """
+        agg_js = {'images':[],'external_links':[]}
+        
+        if self.response != None:
+            attrs = 'href','src'
+            href_links,src_links='',''
+            links = [href_links,src_links]
+            for i,each in enumerate(attrs):
+                 links[i]= [a[attr[i]] for a in get_find_all_with_attributes(self, attrs[i])]
+            # Convert all links to absolute links
+            absolute_links = [(url, link) for link in links[0] + links[1]]
+            # Separate images and external links
+            images = [link for link in absolute_links if link.lower().endswith(('.png', '.jpg', '.jpeg', '.gif', '.svg', '.webp'))]
+            external_links = [link for link in absolute_links if urlparse(link).netloc != urlparse(url).netloc]
+            agg_js['images']=images
+            agg_js['external_links']=external_links
+           
+        return agg_js
+
+
+    def correct_xml(xml_string):
+        # Parse the XML string
+        root = ET.fromstring(xml_string)
+
+        # Loop through each <image:loc> element and correct its text if needed
+        for image_loc in root.findall(".//image:loc", namespaces={'image': 'http://www.google.com/schemas/sitemap-image/1.1'}):
+            # Replace '&' with '&amp;' in the element's text
+            if '&' in image_loc.text:
+                image_loc.text = image_loc.text.replace('&', '&amp;')
+
+        # Convert the corrected XML back to string
+        corrected_xml = ET.tostring(root, encoding='utf-8').decode('utf-8')
+        return corrected_xml
+
+
+    def determine_values(self):
+        # This is just a mockup. In a real application, you'd analyze the URL or its content.
+
+        # Assuming a blog site
+        if 'blog' in self.url:
+            if '2023' in self.url:  # Assuming it's a current year article
+                return ('weekly', '0.8')
+            else:
+                return ('monthly', '0.6')
+        elif 'contact' in self.url:
+            return ('yearly', '0.3')
+        else:  # Homepage or main categories
+            return ('weekly', '1.0')
+    def crawl(url, max_depth=3, depth=1):
+        
+        if depth > max_depth:
+            return []
+
+        if url in visited:
+            return []
+
+        visited.add(url)
+
+        try:
+            
+            links = [a['href'] for a in self.soup.find_all('a', href=True)]
+            valid_links = []
+
+            for link in links:
+                parsed_link = urlparse(link)
+                base_url = "{}://{}".format(parsed_link.scheme, parsed_link.netloc)
+            
+                if base_url == url:  # Avoiding external URLs
+                    final_link = urljoin(url, parsed_link.path)
+                    if final_link not in valid_links:
+                        valid_links.append(final_link)
+
+            for link in valid_links:
+                crawl(link, max_depth, depth+1)
+
+            return valid_links
+
+        except Exception as e:
+            print(f"Error crawling {url}: {e}")
+            return []
+
+
+    # Define or import required functions here, like get_all_website_links, determine_values, 
+    # discover_classes_and_meta_images, and extract_links_from_url.
+    def get_meta_info(self):
+        
+        meta_info = {}
+        # Fetch the title if available
+        title_tag = parse_title()
+        if title_tag:
+            meta_info["title"] = title_tag
+        # Fetch meta tags
+        for meta_tag in soup.find_all('meta'):
+            name = meta_tag.get('name') or meta_tag.get('property')
+            if name:
+                content = meta_tag.get('content')
+                if content:
+                    meta_info[name] = content
+
+        return meta_info
+    def generate_sitemap(self,domain):
+        
+        with open('sitemap.xml', 'w', encoding='utf-8') as f:
+            string = '<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:image="http://www.google.com/schemas/sitemap-image/1.1">\n'
+            
+            for url in self.all_site_links:
+                string += f'  <url>\n    <loc>{url}</loc>\n'
+                preprocess=[]
+                self.get_new_source_and_url(url=url)
+                links = extract_links_from_url(url)
+                
+                for img in links['images']:
+                    if str(img).lower() not in preprocess:
+                        try:
+                            escaped_img = img.replace('&', '&amp;')
+
+                            str_write = f'    <image:image>\n      <image:loc>{escaped_img}</image:loc>\n    </image:image>\n'
+                            string += str_write
+                        except:
+                            pass
+                        preprocess.append(str(img).lower())
+                frequency, priority = determine_values(url)
+                string += f'    <changefreq>{frequency}</changefreq>\n'
+                string += f'    <priority>{priority}</priority>\n'
+                string += f'  </url>\n'
+                
+            string += '</urlset>\n'
+            f.write(string)            
+        # Output summary
+        print(f'Sitemap saved to sitemap.xml with {len(urls)} URLs.')
+        
+        # Output class and link details
+        for url in urls:
+            print(f"\nDetails for {url}:")
+            classes, meta_img_refs = discover_classes_and_meta_images(url)
+
+            print("\nClasses with href or src attributes:")
+            for class_name in classes:
+                print(f"\t{class_name}")
+            
+            print("\nMeta Image References:")
+            for img_ref in meta_img_refs:
+                print(f"\t{img_ref}")
+            
+            links = extract_links_from_url(url)
+
+            print("\nImages:")
+            for img in links['images']:
+                print(f"\t{img}")
+            
+            print("\nExternal Links:")
+            for ext_link in links['external_links']:
+                print(f"\t{ext_link}")
+class CrawlManagerSingleton():
+    _instance = None
+    @staticmethod
+    def get_instance(url=None,source_code=None,parse_type="html.parser"):
+        if CrawlManagerSingleton._instance is None:
+            CrawlManagerSingleton._instance = CrawlManager(url=url,parse_type=parse_type,source_code=source_code)
+        elif parse_type != CrawlManagerSingleton._instance.parse_type or url != CrawlManagerSingleton._instance.url  or source_code != CrawlManagerSingleton._instance.source_code:
+            CrawlManagerSingleton._instance = CrawlManager(url=url,parse_type=parse_type,source_code=source_code)
+        return CrawlManagerSingleton._instance

abstract_webtools/managers/domainManager.py

@@ -0,0 +1,48 @@
+from ..abstract_webtools import *
+class domainManager(metaclass=SingletonMeta):
+    def __init__(self, url):
+        if not hasattr(self, 'initialized'):  # Prevent reinitialization
+            self.initialized = True
+            parsed_url = urlparse(url)
+            self.domain = parsed_url.netloc
+            self.scheme = parsed_url.scheme
+    def get_url_to_path(self, url):
+        url = eatAll(str(url),['',' ','\n','\t','\\','/'])
+        parsed_url = urlparse(url)
+        if 'data:image' in url:
+            input(url)
+        if parsed_url.netloc == self.domain:
+            paths = parsed_url.path.split('/')
+            dir_path =self.site_dir
+            for path in paths[:-1]:
+                dir_path = os.path.join(dir_path, path)
+                os.makedirs(dir_path, exist_ok=True)
+        #if 'svg' in url:
+        #$    input(url)
+         #   dir_path = get_image_name('contents',directory=dir_path,ext='png',url=item_url)
+
+
+        self.page_type.append(os.path.splitext(paths[-1])[-1] or 'html' if len(self.page_type) == 0 else self.page_type[-1])
+            
+        dir_path = os.path.join(dir_path, paths[-1])
+        return dir_path
+
+    def saved_url_check(self, url):
+  
+        path = self.get_url_to_path(url)
+        return path
+
+    def get_with_netloc(self, url):
+        parsed_url = urlparse(url)
+        if parsed_url.netloc == '':
+            url = f"{self.scheme}://{self.domain}/{url.strip()}"
+        return url
+
+    def get_driver(self, url):
+        if url and url not in self.drivers:
+            chrome_options = Options()
+            chrome_options.add_argument("--headless")
+            driver = webdriver.Chrome(options=chrome_options)
+            self.drivers[url] = driver
+            driver.get(url)
+        return self.drivers[url]

abstract_webtools/managers/dynamicRateLimiter.py

@@ -0,0 +1,138 @@
+class DynamicRateLimiterManager:
+    def __init__(self, service_name='ethereum'):
+        self.services = {}
+        self.service_name = service_name
+        self.add_service(service_name)
+
+    def add_service(self, service_name="default", low_limit=10, high_limit=30, limit_epoch=60, starting_tokens=10, epoch_cycle_adjustment=True):
+        if service_name in self.services:
+            print(f"Service {service_name} already exists!")
+            return
+        self.services[service_name] = DynamicRateLimiter(low_limit=low_limit, high_limit=high_limit, limit_epoch=limit_epoch, starting_tokens=starting_tokens, epoch_cycle_adjustment=epoch_cycle_adjustment)
+
+    def request(self, request_url, service_name=None):
+        service_name = service_name or self.service_name
+        if service_name not in self.services:
+            self.add_service(service_name)
+
+        limiter = self.services[service_name]
+
+        while True:
+            if limiter.request():
+                response = requests.get(request_url)  # Actual request
+                if response.status_code == 200:
+                    limiter.request_tracker(True)
+                    return response.json()
+                elif response.status_code == 429:
+                    limiter.request_tracker(False)
+                    print(f"Rate limited by {service_name}. Adjusting limit and retrying...")
+                    time.sleep(limiter.get_sleep()["current_sleep"])
+                else:
+                    print(f"Unexpected response: {response.status_code}. Message: {response.text}")
+                    return None
+            else:
+                print(f"Rate limit reached for {service_name}. Waiting for the next epoch...")
+                time.sleep(limiter.get_sleep()["current_sleep"])
+
+    def log_request(self, service_name, success):
+        print(f"[{service_name}] Request {'succeeded' if success else 'denied'}. Current tokens: {self.services[service_name].get_current_tokens()}")
+
+class DynamicRateLimiter:
+    def __init__(self, low_limit, high_limit, limit_epoch, starting_tokens=None,epoch_cycle_adjustment:int=None):
+        self.low_limit = low_limit
+        self.high_limit = high_limit
+        self.limit_epoch = limit_epoch  # in seconds
+        self.request_status_json = {"succesful":[],"unsuccesful":[],"last_requested":get_time_stamp(),"first_requested":get_time_stamp(),"epoch_left":self.limit_epoch,"last_fail":get_time_stamp(),"count_since_fail":0}
+        self.current_limit = starting_tokens or low_limit  # Default to high_limit if starting_tokens isn't provided
+        self.epoch_cycle_adjustment = epoch_cycle_adjustment
+        # Additional attributes for tracking adjustment logic
+        self.last_adjusted_time = get_time_stamp()
+        self.successful_epochs_since_last_adjustment = 0
+        self.request_count_in_current_epoch = 0
+
+    def _refill_tokens(self):
+        time_since_last_request = get_time_stamp() - self.request_status_json["last_requested"]
+        new_tokens = (time_since_last_request / self.limit_epoch) * self.current_limit
+        self.tokens = min(self.current_limit, self.get_current_tokens())
+    def request_tracker(self,success):
+        if success:
+            self.request_status_json["succesful"].append(get_time_stamp())
+        else:
+            self.request_status_json["unsuccesful"].append(get_time_stamp())
+            self.request_status_json["last_fail"]=get_time_stamp()
+            self.request_status_json["count_since_fail"]=0
+            self.adjust_limit()
+        self.request_status_json["last_requested"]=get_time_stamp()
+    def calculate_tokens(self):
+        successful = []
+        for each in self.request_status_json["succesful"]:
+            if (get_time_stamp() - each)<self.limit_epoch:
+                successful.append(each)
+        self.request_status_json["succesful"]=successful
+        unsuccessful = []
+        for each in self.request_status_json["unsuccesful"]:
+            if (get_time_stamp() - each)<self.limit_epoch:
+                unsuccessful.append(each)
+        self.request_status_json["unsuccesful"]=unsuccessful
+        if len(successful)==0 and len(unsuccessful)==0:
+            pass
+        elif len(successful)!=0 and len(unsuccessful)==0:
+            self.request_status_json["first_requested"] = successful[0]
+        elif len(successful)==0 and len(unsuccessful)!=0:
+            self.request_status_json["first_requested"] = unsuccessful[0]
+        else:
+            self.request_status_json["first_requested"] = min(unsuccessful[0],successful[0])
+        self.request_status_json["epoch_left"]=self.limit_epoch-(self.request_status_json["last_requested"]-self.request_status_json["first_requested"])
+        
+        return self.request_status_json
+    def get_current_tokens(self):
+        self.request_status_json = self.calculate_tokens()
+        total_requests = len(self.request_status_json["succesful"])+len(self.request_status_json["unsuccesful"])
+        return max(0,self.current_limit-total_requests)
+    def get_sleep(self):
+        self.request_status_json = self.calculate_tokens()
+        self.request_status_json["current_sleep"]=self.request_status_json["epoch_left"]/max(1,self.get_current_tokens())
+        return self.request_status_json
+    def request(self):
+        self._refill_tokens()
+        if self.tokens > 0:
+            return True  # The request can be made
+        else:
+            if self.tokens == 0:
+                self.request_status_json["count_since_fail"]+=1
+                if self.epoch_cycle_adjustment != None:
+                    if self.request_status_json["count_since_fail"] >=self.epoch_cycle_adjustment:
+                        self.current_limit=min(self.current_limit+1,self.high_limit)
+            return False  # The request cannot be made
+    def _adjust_limit(self):
+        current_time = get_time_stamp()
+        if current_time - self.last_adjusted_time >= self.limit_epoch:
+            if len(self.clear_epoch()["succesful"]) >= self.tokens:
+                # We hit the rate limit this epoch, decrease our limit
+                self.tokens = max(1, self.tokens - 1)
+            else:
+                self.successful_epochs_since_last_adjustment += 1
+                if self.successful_epochs_since_last_adjustment >= 5:
+                    # We've had 5 successful epochs, increase our limit
+                    self.current_limit = min(self.high_limit, self.tokens + 1)
+                    self.successful_epochs_since_last_adjustment = 0
+            
+            # Reset our counters for the new epoch
+            self.last_adjusted_time = current_time
+            self.request_count_in_current_epoch = 0
+    def adjust_limit(self):
+        # Set the tokens to succesful requests_made - 1
+        self.tokens = len(self.calculate_tokens()["succesful"])
+
+        # Adjust the high_limit
+        self.current_limit = self.tokens
+
+        # Log the adjustment
+        print(f"Adjusted tokens to: {self.tokens} and high_limit to: {self.current_limit}")
+class DynamicRateLimiterManagerSingleton:
+    _instance = None
+    @staticmethod
+    def get_instance(service_name="default", low_limit=10, high_limit=30, limit_epoch=60,starting_tokens=10,epoch_cycle_adjustment=True):
+        if DynamicRateLimiterManagerSingleton._instance is None:
+            DynamicRateLimiterManagerSingleton._instance = DynamicRateLimiterManager(service_name=service_name, low_limit=low_limit, high_limit=limit_epoch, limit_epoch=60,starting_tokens=starting_tokens,epoch_cycle_adjustment=epoch_cycle_adjustment)
+        return DynamicRateLimiterManagerSingleton._instance

abstract_webtools/managers/linkManager.py

@@ -0,0 +1,189 @@
+from ..abstract_webtools import *
+from .urlManager import *
+from .requestManager import *
+from .soupManager import *
+def get_url(url=None,url_mgr=None):
+    if not url and not url_mgr:
+        return None
+    if url:
+        url_mgr = urlManager(url)
+    return url_mgr.url
+def get_url_mgr(url=None,url_mgr=None):
+    if not url and not url_mgr:
+        return None
+    if url:
+        url_mgr = urlManager(url)
+    return url_mgr 
+class linkManager:
+    """
+    LinkManager is a class for managing and extracting links and image links from a web page.
+
+    Args:
+        url (str): The URL of the web page (default is "https://example.com").
+        source_code (str or None): The source code of the web page (default is None).
+        url_mgr (UrlManager or None): An instance of UrlManager (default is None).
+        request_manager (requestManager or None): An instance of requestManager (default is None).
+        soup_manager (SoupManager or None): An instance of SoupManager (default is None).
+        image_link_tags (str): HTML tags to identify image links (default is 'img').
+        img_link_attrs (str): HTML attributes to identify image link URLs (default is 'src').
+        link_tags (str): HTML tags to identify links (default is 'a').
+        link_attrs (str): HTML attributes to identify link URLs (default is 'href').
+        strict_order_tags (bool): Flag to indicate if tags and attributes should be matched strictly (default is False).
+        img_attr_value_desired (list or None): Desired attribute values for image links (default is None).
+        img_attr_value_undesired (list or None): Undesired attribute values for image links (default is None).
+        link_attr_value_desired (list or None): Desired attribute values for links (default is None).
+        link_attr_value_undesired (list or None): Undesired attribute values for links (default is None).
+        associated_data_attr (list): HTML attributes to associate with the extracted links (default is ["data-title", 'alt', 'title']).
+        get_img (list): HTML attributes used to identify associated images (default is ["data-title", 'alt', 'title']).
+
+    Methods:
+        re_initialize(): Reinitialize the LinkManager with the current settings.
+        update_url_mgr(url_mgr): Update the URL manager with a new instance.
+        update_url(url): Update the URL and reinitialize the LinkManager.
+        update_source_code(source_code): Update the source code and reinitialize the LinkManager.
+        update_soup_manager(soup_manager): Update the SoupManager and reinitialize the LinkManager.
+        update_desired(...): Update the desired settings and reinitialize the LinkManager.
+        find_all_desired(...): Find all desired links or image links based on the specified criteria.
+        find_all_domain(): Find all unique domain names in the extracted links.
+
+    Note:
+        - The LinkManager class helps manage and extract links and image links from web pages.
+        - The class provides flexibility in specifying criteria for link extraction.
+    """
+    def __init__(self,url=None,source_code=None,url_mgr=None,req_mgr=None,soup_manager=None,image_link_tags='img',img_link_attrs='src',link_tags='a',link_attrs='href',strict_order_tags=False,img_attr_value_desired=None,img_attr_value_undesired=None,link_attr_value_desired=None,link_attr_value_undesired=None,associated_data_attr=["data-title",'alt','title'],get_img=["data-title",'alt','title']):
+        self.url_mgr = get_url_mgr(url=url,url_mgr=url_mgr)
+        self.url=self.url_mgr.url
+        self.req_mgr = get_req_mgr(req_mgr=req_mgr,url=self.url,url_mgr=self.url_mgr,source_code=source_code)
+        self.source_code = source_code or self.req_mgr.source_code or self.req_mgr.source_code_bytes
+        self.soup_mgr = get_soup_mgr(req_mgr=self.req_mgr,url_mgr=self.url_mgr,source_code = source_code)
+
+        self.strict_order_tags=strict_order_tags
+        self.image_link_tags=image_link_tags
+        self.img_link_attrs=img_link_attrs
+        self.link_tags=link_tags
+        self.link_attrs=link_attrs
+        self.img_attr_value_desired=img_attr_value_desired
+        self.img_attr_value_undesired=img_attr_value_undesired
+        self.link_attr_value_desired=link_attr_value_desired
+        self.link_attr_value_undesired=link_attr_value_undesired
+        self.associated_data_attr=associated_data_attr
+        self.get_img=get_img
+        self.all_desired_image_links=self.find_all_desired_links(tag=self.image_link_tags,
+                                                                 attr=self.img_link_attrs,
+                                                                 attr_value_desired=self.img_attr_value_desired,
+                                                                 attr_value_undesired=self.img_attr_value_undesired)
+        self.all_desired_links=self.find_all_desired_links(tag=self.link_tags,
+                                                           attr=self.link_attrs,
+                                                           attr_value_desired=self.link_attr_value_desired,
+                                                           attr_value_undesired=self.link_attr_value_undesired,
+                                                           associated_data_attr=self.associated_data_attr,
+                                                           get_img=get_img)
+    def re_initialize(self):
+        self.all_desired_image_links=self.find_all_desired_links(tag=self.image_link_tags,attr=self.img_link_attrs,strict_order_tags=self.strict_order_tags,attr_value_desired=self.img_attr_value_desired,attr_value_undesired=self.img_attr_value_undesired)
+        self.all_desired_links=self.find_all_desired_links(tag=self.link_tags,attr=self.link_attrs,strict_order_tags=self.strict_order_tags,attr_value_desired=self.link_attr_value_desired,attr_value_undesired=self.link_attr_value_undesired,associated_data_attr=self.associated_data_attr,get_img=self.get_img)
+    def update_url_mgr(self,url_mgr):
+        self.url_mgr=url_mgr
+        self.url=self.url_mgr.url
+        self.req_mgr.update_url_mgr(url_mgr=self.url_mgr)
+        self.soup_mgr.update_url_mgr(url_mgr=self.url_mgr)
+        self.source_code=self.soup_mgr.source_code
+        self.re_initialize()
+    def update_url(self,url):
+        self.url=url
+        self.url_mgr.update_url(url=self.url)
+        self.url=self.url_mgr.url
+        self.req_mgr.update_url(url=self.url)
+        self.soup_mgr.update_url(url=self.url)
+        self.source_code=self.soup_mgr.source_code
+        self.re_initialize()
+    def update_source_code(self,source_code):
+        self.source_code=source_code
+        if self.source_code != self.soup_mgr.source_code:
+            self.soup_mgr.update_source_code(source_code=self.source_code)
+        self.re_initialize()
+    def update_soup_manager(self,soup_manager):
+        self.soup_mgr=soup_manager
+        self.source_code=self.soup_mgr.source_code
+        self.re_initialize()
+    def update_desired(self,img_attr_value_desired=None,img_attr_value_undesired=None,link_attr_value_desired=None,link_attr_value_undesired=None,image_link_tags=None,img_link_attrs=None,link_tags=None,link_attrs=None,strict_order_tags=None,associated_data_attr=None,get_img=None):
+       self.strict_order_tags = strict_order_tags or self.strict_order_tags
+       self.img_attr_value_desired=img_attr_value_desired or self.img_attr_value_desired
+       self.img_attr_value_undesired=img_attr_value_undesired or self.img_attr_value_undesired
+       self.link_attr_value_desired=link_attr_value_desired or self.link_attr_value_desired
+       self.link_attr_value_undesired=link_attr_value_undesired or self.link_attr_value_undesired
+       self.image_link_tags=image_link_tags or self.image_link_tags
+       self.img_link_attrs=img_link_attrs or self.img_link_attrs
+       self.link_tags=link_tags or self.link_tags
+       self.link_attrs=link_attrs or self.link_attrs
+       self.associated_data_attr=associated_data_attr or self.associated_data_attr
+       self.get_img=get_img or self.get_img
+       self.re_initialize()
+    def find_all_desired(self,tag='img',attr='src',strict_order_tags=False,attr_value_desired=None,attr_value_undesired=None,associated_data_attr=None,get_img=None):
+        def make_list(obj):
+            if isinstance(obj,list) or obj==None:
+                return obj
+            return [obj]
+        def get_desired_value(attr,attr_value_desired=None,attr_value_undesired=None):
+            if attr_value_desired:
+                for value in attr_value_desired:
+                    if value not in attr:
+                        return False
+            if attr_value_undesired:
+                for value in attr_value_undesired:
+                    if value in attr:
+                        return False
+            return True
+        attr_value_desired,attr_value_undesired,associated_data_attr,tags,attribs=make_list(attr_value_desired),make_list(attr_value_undesired),make_list(associated_data_attr),make_list(tag),make_list(attr)
+        desired_ls = []
+        assiciated_data=[]
+        for i,tag in enumerate(tags):
+            attribs_list=attribs
+            if strict_order_tags:
+                if len(attribs)<=i:
+                    attribs_list=[None]
+                else:
+                    attribs_list=make_list(attribs[i])
+            for attr in attribs_list:
+                for component in self.soup_mgr.soup.find_all(tag):
+                    if attr in component.attrs and get_desired_value(attr=component[attr],attr_value_desired=attr_value_desired,attr_value_undesired=attr_value_undesired):
+                        if component[attr] not in desired_ls:
+                            desired_ls.append(component[attr])
+                            assiciated_data.append({"value":component[attr]})
+                            if associated_data_attr:
+                                for data in associated_data_attr:
+                                    if data in component.attrs:
+                                        assiciated_data[-1][data]=component.attrs[data]
+                                        if get_img and component.attrs[data]:
+                                            if data in get_img and len(component.attrs[data])!=0:
+                                                for each in self.soup_mgr.soup.find_all('img'):
+                                                    if 'alt' in each.attrs:
+                                                        if each.attrs['alt'] == component.attrs[data] and 'src' in each.attrs:
+                                                            assiciated_data[-1]['image']=each.attrs['src']
+        desired_ls.append(assiciated_data)
+        return desired_ls
+    def find_all_domain(self):
+        domain = urlparse(self.url_mgr.url).netloc
+        domains_ls=[self.url_mgr.url]
+        for url in self.all_desired_links[:-1]:
+            if self.url_mgr.is_valid_url(url):
+                parse = urlparse(url)
+                comp_domain = parse.netloc
+                if url not in domains_ls and comp_domain == domain:
+                    domains_ls.append(url)
+        return domains_ls
+    def find_all_desired_links(self,tag='img', attr='src',attr_value_desired=None,strict_order_tags=False,attr_value_undesired=None,associated_data_attr=None,all_desired=None,get_img=None):
+        all_desired = all_desired or self.find_all_desired(tag=tag,attr=attr,strict_order_tags=strict_order_tags,attr_value_desired=attr_value_desired,attr_value_undesired=attr_value_undesired,associated_data_attr=associated_data_attr,get_img=get_img)
+        assiciated_attrs = all_desired[-1]
+        valid_assiciated_attrs = []
+        desired_links=[]
+        for i,attr in enumerate(all_desired[:-1]):
+            valid_attr=self.url_mgr.make_valid(attr,self.url_mgr.protocol+'://'+self.url_mgr.domain) 
+            if valid_attr:
+                desired_links.append(valid_attr)
+                valid_assiciated_attrs.append(assiciated_attrs[i])
+                valid_assiciated_attrs[-1]["link"]=valid_attr
+        desired_links.append(valid_assiciated_attrs)
+        return desired_links
+
+
+    

abstract_webtools/managers/mySocketClient.py

@@ -0,0 +1,46 @@
+class MySocketClient:
+    def __init__(self, ip_address=None, port=None,domain=None):
+        self.sock
+        self.ip_address= ip_address or None
+        self.port = port  or None
+        
+        self.domain = domain  or None
+    def receive_data(self):
+        chunks = []
+        while True:
+            chunk = self.sock.recv(4096)
+            if chunk:
+                chunks.append(chunk)
+            else:
+                break
+        return b''.join(chunks).decode('utf-8')
+    def _parse_socket_response_as_json(self, data, *args, **kwargs):
+        return self._parse_json(data[data.find('{'):data.rfind('}') + 1], *args, **kwargs)
+    def process_data(self):
+        data = self.receive_data()
+        return self._parse_socket_response_as_json(data)
+    def _parse_json(self,json_string):
+        return json.loads(json_string)
+    def get_ip(self,domain=None):
+        try:
+            return self.sock.gethostbyname(domain if domain != None else self.domain)
+        except self.sock.gaierror:
+            return None
+    def grt_host_name(self,ip_address=None):
+        return self.sock.gethostbyaddr(ip_address if ip_address != None else self.ip_address)
+    def toggle_sock(self):
+        if self.sock != None:
+            self.sock.close()
+        else:
+            self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            if host and socket:
+                self.sock.connect((host, port))
+class MySocketClient():
+    _instance = None
+    @staticmethod
+    def get_instance(ip_address='local_host',port=22,domain="example.com"):
+        if MySocketClientSingleton._instance is None:
+            MySocketClientSingleton._instance = MySocketClient(ip_address=ip_address,port=port,domain=domain)
+        elif MySocketClientSingleton._instance.ip_address != ip_address or MySocketClientSingleton._instance.port != port or UrlManagerSingleton._instance.domain != domain:
+            MySocketClientSingleton._instance = MySocketClient(ip_address=ip_address,port=port,domain=domain)
+        return MySocketClient