abstract-webtools 0.1.6.138__tar.gz → 0.1.6.140__tar.gz

{abstract_webtools-0.1.6.138 → abstract_webtools-0.1.6.140}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: abstract_webtools
-Version: 0.1.6.138
+Version: 0.1.6.140
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff

{abstract_webtools-0.1.6.138 → abstract_webtools-0.1.6.140}/setup.py

@@ -4,7 +4,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 setuptools.setup(
     name='abstract_webtools',
-    version='0.1.6.138',
+    version='0.1.6.140',
     author='putkoff',
     author_email='partners@abstractendeavors.com',
     description='Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.',

{abstract_webtools-0.1.6.138 → abstract_webtools-0.1.6.140}/src/abstract_webtools/managers/__init__.py

@@ -12,3 +12,4 @@ from .urlManager import *
 from .userAgentManager import *
 from .seleniumManager import *
 from .videoDownloader import *
+from .middleManager import *

{abstract_webtools-0.1.6.138 → abstract_webtools-0.1.6.140}/src/abstract_webtools/managers/linkManager/linkManager.py

@@ -2,18 +2,7 @@ from ...abstract_webtools import *
 from ..urlManager import *
 from ..requestManager import *
 from ..soupManager import *
-def get_url(url=None,url_mgr=None):
-    if not url and not url_mgr:
-        return None
-    if url:
-        url_mgr = urlManager(url)
-    return url_mgr.url
-def get_url_mgr(url=None,url_mgr=None):
-    if not url and not url_mgr:
-        return None
-    if url:
-        url_mgr = urlManager(url)
-    return url_mgr 
+from ..middleManager import *
 class linkManager:
     """
     LinkManager is a class for managing and extracting links and image links from a web page.
@@ -50,13 +39,40 @@ class linkManager:
         - The LinkManager class helps manage and extract links and image links from web pages.
         - The class provides flexibility in specifying criteria for link extraction.
     """
-    def __init__(self,url=None,source_code=None,url_mgr=None,req_mgr=None,soup_manager=None,image_link_tags='img',img_link_attrs='src',link_tags='a',link_attrs='href',strict_order_tags=False,img_attr_value_desired=None,img_attr_value_undesired=None,link_attr_value_desired=None,link_attr_value_undesired=None,associated_data_attr=["data-title",'alt','title'],get_img=["data-title",'alt','title']):
-        self.url_mgr = get_url_mgr(url=url,url_mgr=url_mgr)
-        self.url=self.url_mgr.url
-        self.req_mgr = get_req_mgr(req_mgr=req_mgr,url=self.url,url_mgr=self.url_mgr,source_code=source_code)
-        self.source_code = source_code or self.req_mgr.source_code or self.req_mgr.source_code_bytes
-        self.soup_mgr = get_soup_mgr(req_mgr=self.req_mgr,url_mgr=self.url_mgr,source_code = source_code)
+    def __init__(self,
+                 url=None,
+                 source_code=None,
+                 soup=None,
+                 url_mgr=None,
+                 req_mgr=None,
+                 soup_mgr=None,
+                 image_link_tags='img',
+                 img_link_attrs='src',
+                 link_tags='a',
+                 link_attrs='href',
+                 strict_order_tags=False,
+                 img_attr_value_desired=None,
+                 img_attr_value_undesired=None,
+                 link_attr_value_desired=None,
+                 link_attr_value_undesired=None,
+                 associated_data_attr=["data-title",'alt','title'],
+                 get_img=["data-title",'alt','title']
+                 ):
+     
+
+        all_tools = get_soup_tools(
+            url=url,
+            url_mgr=url_mgr,
+            source_code=source_code,
+            req_mgr=req_mgr,
+            soup=soup,
+            soup_mgr=soup_mgr,
+            target_manager = self
+            )
 
+                                                       
+                                                 
+        
         self.strict_order_tags=strict_order_tags
         self.image_link_tags=image_link_tags
         self.img_link_attrs=img_link_attrs
@@ -171,19 +187,25 @@ class linkManager:
                 if url not in domains_ls and comp_domain == domain:
                     domains_ls.append(url)
         return domains_ls
+
     def find_all_desired_links(self,tag='img', attr='src',attr_value_desired=None,strict_order_tags=False,attr_value_undesired=None,associated_data_attr=None,all_desired=None,get_img=None):
         all_desired = all_desired or self.find_all_desired(tag=tag,attr=attr,strict_order_tags=strict_order_tags,attr_value_desired=attr_value_desired,attr_value_undesired=attr_value_undesired,associated_data_attr=associated_data_attr,get_img=get_img)
         assiciated_attrs = all_desired[-1]
         valid_assiciated_attrs = []
         desired_links=[]
         for i,attr in enumerate(all_desired[:-1]):
-            valid_attr=self.url_mgr.make_valid(attr,self.url_mgr.protocol+'://'+self.url_mgr.domain) 
-            if valid_attr:
-                desired_links.append(valid_attr)
-                valid_assiciated_attrs.append(assiciated_attrs[i])
-                valid_assiciated_attrs[-1]["link"]=valid_attr
+
+            self.url_mgr.domain = self.url_mgr.domain or ''
+
+            self.url_mgr.protocol = self.url_mgr.protocol or 'https'
+ 
+            if attr:
+                valid_attr=self.url_mgr.make_valid(attr,self.url_mgr.protocol+'://'+self.url_mgr.domain) 
+                if valid_attr:
+                    desired_links.append(valid_attr)
+                    valid_assiciated_attrs.append(assiciated_attrs[i])
+                    valid_assiciated_attrs[-1]["link"]=valid_attr
         desired_links.append(valid_assiciated_attrs)
         return desired_links
 
-
     

abstract_webtools-0.1.6.140/src/abstract_webtools/managers/middleManager/__init__.py

@@ -0,0 +1 @@
+from .src import *

abstract_webtools-0.1.6.140/src/abstract_webtools/managers/middleManager/imports.py

@@ -0,0 +1,18 @@
+import logging
+from ..urlManager import (
+    urlManager,
+    get_url,
+    get_url_mgr
+    )
+from ..requestManager import (
+    requestManager,
+    get_source,
+    get_req_mgr
+    )
+from ..soupManager import (
+    soupManager,
+    get_soup,
+    get_soup_mgr
+    )
+from bs4 import BeautifulSoup
+logging.basicConfig(level=logging.INFO)

abstract_webtools-0.1.6.140/src/abstract_webtools/managers/middleManager/src/UnifiedWebManager.py

@@ -0,0 +1,135 @@
+from ..imports import *
+
+class UnifiedWebManager:
+    """
+    Unified middleware that ties together URL, request, and soup managers.
+    Lazily initializes components based on provided inputs.
+    
+    Args:
+        url (str or None): The base URL.
+        source_code (str or bytes or None): Pre-fetched source code.
+        url_mgr (urlManager or None): Existing URL manager.
+        req_mgr (requestManager or None): Existing request manager.
+        soup_mgr (soupManager or None): Existing soup manager.
+        parse_type (str): Parser type for BeautifulSoup (default: "html.parser").
+    """
+    def __init__(self, url=None, source_code=None, url_mgr=None, req_mgr=None, soup_mgr=None,soup=None, parse_type="html.parser"):
+        self._url = url
+        self._source_code = source_code
+        self._url_mgr = url_mgr
+        self._req_mgr = req_mgr
+        self._soup_mgr = soup_mgr
+        self._parse_type = parse_type
+        self._soup = None  # Lazy
+
+    @property
+    def url_mgr(self):
+        if self._url_mgr is None:
+            if self._url is None:
+                logging.warning("No URL provided; URL manager cannot be created.")
+                return None
+            self._url_mgr = urlManager(url=self._url)
+        return self._url_mgr
+
+    @property
+    def url(self):
+        if self._url is None and self.url_mgr:
+            self._url = self.url_mgr.url
+        return self._url
+
+    @property
+    def req_mgr(self):
+        if self._req_mgr is None:
+            self._req_mgr = requestManager(
+                url=self.url,
+                url_mgr=self.url_mgr,
+                source_code=self._source_code
+            )
+        return self._req_mgr
+
+    @property
+    def source_code(self):
+        if self._source_code is None and self.req_mgr:
+            self._source_code = self.req_mgr.source_code
+        return self._source_code
+
+    @property
+    def soup_mgr(self):
+        if self._soup_mgr is None:
+            self._soup_mgr = soupManager(
+                url=self.url,
+                url_mgr=self.url_mgr,
+                req_mgr=self.req_mgr,
+                source_code=self.source_code
+            )
+        return self._soup_mgr
+
+    @property
+    def soup(self):
+        if self._soup is None:
+            source = self.source_code
+            if source is None:
+                logging.warning("No source code available; Soup cannot be created.")
+                return None
+            if isinstance(source, bytes):
+                source = source.decode('utf-8', errors='ignore')
+            self._soup = BeautifulSoup(source, self._parse_type)
+        return self._soup
+
+    def update_url(self, url):
+        """Update the URL and reset dependent managers."""
+        self._url = url
+        self._url_mgr = None
+        self._req_mgr = None
+        self._soup_mgr = None
+        self._source_code = None
+        self._soup = None
+
+    def update_source_code(self, source_code):
+        """Update the source code and reset dependent managers."""
+        self._source_code = source_code
+        self._req_mgr = None
+        self._soup_mgr = None
+        self._soup = None
+
+    # Convenience methods for direct access
+    def get_all_tools(self):
+        """Return a dict with all components (similar to original getters)."""
+        return {
+            'url': self.url,
+            'url_mgr': self.url_mgr,
+            'source_code': self.source_code,
+            'req_mgr': self.req_mgr,
+            'soup': self.soup,
+            'soup_mgr': self.soup_mgr
+        }
+    def endow_to_manager(self, target_manager, all_tools=None):
+        """
+        Endow (assign) the attributes from all_tools to the target manager instance.
+        
+        Args:
+            target_manager: The instance (e.g., another manager class) to endow attributes to.
+            all_tools (dict or None): Optional dict of tools/attributes. If None, uses self.get_all_tools().
+        """
+        if all_tools is None:
+            all_tools = self.get_all_tools()
+        for key, value in all_tools.items():
+            setattr(target_manager, key, value)
+# Wrapper functions for backward compatibility
+def get_url_tools(url=None, url_mgr=None):
+    mgr = UnifiedWebManager(url=url, url_mgr=url_mgr)
+    return {'url': mgr.url, 'url_mgr': mgr.url_mgr}
+
+def get_req_tools(url=None, url_mgr=None, source_code=None, req_mgr=None):
+    mgr = UnifiedWebManager(url=url, url_mgr=url_mgr, source_code=source_code, req_mgr=req_mgr)
+    return {'url': mgr.url, 'url_mgr': mgr.url_mgr, 'source_code': mgr.source_code, 'req_mgr': mgr.req_mgr}
+
+def get_soup_tools(url=None, url_mgr=None, source_code=None, req_mgr=None, soup=None, soup_mgr=None,target_manager=None):
+    mgr = UnifiedWebManager(url=url, url_mgr=url_mgr, source_code=source_code, req_mgr=req_mgr, soup_mgr=soup_mgr)
+    if soup is not None:
+        mgr._soup = soup  # Allow overriding
+    if target_manager:
+        mgr.endow_to_manager(target_manager, all_tools=None)
+    return mgr.get_all_tools()
+
+

abstract_webtools-0.1.6.140/src/abstract_webtools/managers/middleManager/src/__init__.py

@@ -0,0 +1,2 @@
+from .UnifiedWebManager import *
+from .legacy_tools import *

abstract_webtools-0.1.6.140/src/abstract_webtools/managers/middleManager/src/legacy_tools.py

@@ -0,0 +1,8 @@
+from ..imports import (
+    get_url,
+    get_soup_mgr,
+    get_source,
+    get_req_mgr,
+    get_soup,
+    get_soup_mgr
+    )