abstract-webtools 0.1.6.121__tar.gz → 0.1.6.122__tar.gz

{abstract_webtools-0.1.6.121 → abstract_webtools-0.1.6.122}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: abstract_webtools
-Version: 0.1.6.121
+Version: 0.1.6.122
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff

{abstract_webtools-0.1.6.121 → abstract_webtools-0.1.6.122}/setup.py

@@ -4,7 +4,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 setuptools.setup(
     name='abstract_webtools',
-    version='0.1.6.121',
+    version='0.1.6.122',
     author='putkoff',
     author_email='partners@abstractendeavors.com',
     description='Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.',

{abstract_webtools-0.1.6.121 → abstract_webtools-0.1.6.122}/src/abstract_webtools/abstract_usurpit.py

@@ -6,7 +6,20 @@ import shutil
 import time
 from abstract_webtools import *
 from abstract_utilities import *
-
+def get_abs_path():
+    return os.path.abspath(__file__)
+
+def get_abs_dir():
+    abs_path = get_abs_path()
+    return os.path.dirname(abs_path)
+def join_abs_path(path):
+    abs_dir = get_abs_dir()
+    return os.path.join(abs_dir,path)
+def get_rel_dir():
+    return os.getcwd()
+def join_rel_path(path):
+    rel_dir = get_rel_dir()
+    return os.path.join(rel_dir,path) 
 # Import your custom classes/functions
 # from your_module import linkManager, get_soup_mgr
 def make_directory(directory=None,path=None):
@@ -16,6 +29,29 @@ def make_directory(directory=None,path=None):
         directory = os.path.join(base_dir,path)
     os.makedirs(directory,exist_ok=True)
     return directory
+def get_paths(*paths):
+    all_paths = []
+    for path in paths:
+        all_paths+=path.split('/')
+    return all_paths
+def makeAllDirs(*paths):
+    full_path= ''
+    paths = get_paths(*paths)
+    for i,path in enumerate(paths):
+        if i == 0:
+            full_path = path
+            if not full_path.startswith('/'):
+                full_path = join_rel_path(full_path)
+        else:
+            full_path = os.path.join(full_path,path)
+        os.makedirs(full_path,exist_ok=True)
+    return full_path
+def currate_full_path(full_path):
+    dirname = os.path.dirname(full_path)
+    basename = os.path.basename(full_path)
+    full_dirname = makeAllDirs(dirname)
+    full_path = os.path.join(full_dirname,basename)
+    return full_path
 def get_domain_name_from_url(url):
     parsed_url = urlparse(url)
     netloc = parsed_url.netloc
@@ -70,9 +106,10 @@ def get_save_page_path(url, output_dir):
 def save_page(url, content,output_dir):
     page_full_path = get_save_page_path(url=url,
                                         output_dir=output_dir)
+    page_full_path = currate_full_path(page_full_path)
     if page_full_path:
         dirname = os.path.dirname(page_full_path)
-        os.makedirs(dirname, exist_ok=True)
+        
 
         with open(page_full_path, 'w', encoding='utf-8') as f:
             f.write(content)
@@ -144,60 +181,56 @@ class usurpManager():
             "Access-Control-Allow-Origin": "*"})
 
     def process_page(self,url, depth, base_domain):
-            """
-            Process a single page: download assets, save HTML, and crawl links.
-            """
-            print(url)
-            if url in self.visited_pages or depth > self.MAX_DEPTH:
-                return
-            self.visited_pages.add(url)
-            
-     
-            # Fetch the page content
-            response = self.session.get(url)
-            #response.raise_for_status()
-            content = response.text
-            page_full_path = get_save_page_path(url=url,
-                                        output_dir=self.OUTPUT_DIR)
-            if not os.path.exists(page_full_path):
-                # Use your get_soup_mgr function to get the soup and attributes
-                soup_mgr = get_soup_mgr(url=url)
-                soup = soup_mgr.soup
-                all_attributes = soup_mgr.get_all_attribute_values()
-                # Now you can use all_attributes as needed
-                get_asset_path(asset_url=full_asset_url,
-                               base_url=self.url,
-                               output_dir=self.OUTPUT_DIR,
-                               downloaded_assets=self.downloaded_assets,
-                               session=self.session)
-                # Update asset links to local paths
-                for tag in soup.find_all(['img', 'script', 'link']):
-                    attr = 'src' if tag.name != 'link' else 'href'
-                    asset_url = tag.get(attr)
-                    if asset_url:
-                        full_asset_url = normalize_url(asset_url, url)
-                        parsed_asset_url = urlparse(full_asset_url)
-
-                        if is_valid_url(full_asset_url, base_domain):
-                            self.downloaded_assets = save_asset(full_asset_url, self.url,self.OUTPUT_DIR,self.downloaded_assets,self.session)
-                            # Update tag to point to the local asset
-                            local_asset_path = '/' + parsed_asset_url.path.lstrip('/')
-                            tag[attr] = local_asset_path
-
-                # Save the modified page
-                save_page(url, str(soup),self.OUTPUT_DIR)
-            else:
-                print(f"skippinng {page_full_path} because it already exists")
-            # Use your linkManager to find all domain links
-            link_mgr = linkManager(url=url)
-            all_domains = link_mgr.find_all_domain()
-            
-            # Process each domain link
-            for link_url in make_list(all_domains):
-                normalized_link = normalize_url(link_url, url)
-                if is_valid_url(normalized_link, base_domain):
-                    time.sleep(self.WAIT_BETWEEN_REQUESTS)
-                    self.process_page(normalized_link, depth + 1, base_domain)
+        """
+        Process a single page: download assets, save HTML, and crawl links.
+        """
+        print(url)
+        if url in self.visited_pages or depth > self.MAX_DEPTH:
+            return
+        self.visited_pages.add(url)
+        
+ 
+        # Fetch the page content
+        response = self.session.get(url)
+        #response.raise_for_status()
+        content = response.text
+                                  
+        # Use your get_soup_mgr function to get the soup and attributes
+        soup_mgr = get_soup_mgr(url=url)
+        soup = soup_mgr.soup
+        all_attributes = soup_mgr.get_all_attribute_values()
+        # Now you can use all_attributes as needed
+
+        # Update asset links to local paths
+        for tag in soup.find_all(['img', 'script', 'link']):
+            attr = 'src' if tag.name != 'link' else 'href'
+            asset_url = tag.get(attr)
+            if asset_url:
+                full_asset_url = normalize_url(asset_url, url)
+                parsed_asset_url = urlparse(full_asset_url)
+
+                if is_valid_url(full_asset_url, base_domain):
+                    self.downloaded_assets = save_asset(full_asset_url,
+                                                        self.url,
+                                                        self.OUTPUT_DIR,
+                                                        self.downloaded_assets,
+                                                        self.session)
+                    # Update tag to point to the local asset
+                    local_asset_path = '/' + parsed_asset_url.path.lstrip('/')
+                    tag[attr] = local_asset_path
+
+        # Save the modified page
+        save_page(url, str(soup),self.OUTPUT_DIR)
+        # Use your linkManager to find all domain links
+        link_mgr = linkManager(url=url)
+        all_domains = link_mgr.find_all_domain()
+        
+        # Process each domain link
+        for link_url in make_list(all_domains):
+            normalized_link = normalize_url(link_url, url)
+            if is_valid_url(normalized_link, base_domain):
+                time.sleep(self.WAIT_BETWEEN_REQUESTS)
+                self.process_page(normalized_link, depth + 1, base_domain)
 
 
     def main(self):
@@ -209,14 +242,9 @@ class usurpManager():
 
         self.process_page(self.BASE_URL, 0, base_domain)
         print("Website copying completed.")
-def test_download(url=None,directory=None):
-    url=url or 'https://www.youtube.com/watch?v=jRGrNDV2mKc&list=RDMMjRGrNDV2mKc&start_radio=1'
-    
-    output_dir= directory or get_directory_from_url(url) or os.path.join(os.getcwd(),'testit')
-    os.makedirs(output_dir,exist_ok=True)
-    site_mgr = usurpManager(url,output_dir)
-    
+
 def usurpit(url,output_dir=None,max_depth=None,wait_between_requests=None,operating_system=None, browser=None, version=None,user_agent=None,website_bot=None):
     output_dir = get_domain_name_from_url(url) or  make_directory(path='usurped')
     site_mgr = usurpManager(url,output_dir=output_dir,max_depth=max_depth,wait_between_requests=wait_between_requests,operating_system=operating_system, browser=browser, version=version,user_agent=user_agent,website_bot=website_bot)
     site_mgr.main()
+

abstract_webtools-0.1.6.122/src/abstract_webtools/url_grabber_new.py

@@ -0,0 +1,95 @@
+from PyQt5 import QtWidgets, QtCore
+from abstract_webtools import urlManager, requestManager, SoupManager, LinkManager
+from abstract_gui import get_user_agents, get_cipher_list
+
+class UrlGrabberWidget(QtWidgets.QWidget):
+    def __init__(self, initial_url="https://example.com", parent=None):
+        super().__init__(parent)
+        self.initial_url = initial_url
+        self.setup_ui()
+        self.setup_logic()
+        self.init_managers()
+
+    def setup_ui(self):
+        layout = QtWidgets.QVBoxLayout(self)
+
+        # URL input and grab button
+        self.url_input = QtWidgets.QLineEdit(self.initial_url)
+        self.status_label = QtWidgets.QLabel("Status: Unknown")
+        self.grab_btn = QtWidgets.QPushButton("Grab URL")
+
+        url_layout = QtWidgets.QHBoxLayout()
+        url_layout.addWidget(QtWidgets.QLabel("URL:"))
+        url_layout.addWidget(self.url_input)
+        url_layout.addWidget(self.grab_btn)
+
+        # User agent input
+        self.user_agent_box = QtWidgets.QComboBox()
+        self.user_agent_box.addItems(get_user_agents())
+
+        # Source output
+        self.source_code_edit = QtWidgets.QPlainTextEdit()
+        self.source_code_edit.setReadOnly(True)
+
+        # Soup output
+        self.soup_result_edit = QtWidgets.QPlainTextEdit()
+
+        # Action buttons
+        self.action_btn = QtWidgets.QPushButton("Parse")
+        self.get_text_btn = QtWidgets.QPushButton("Get All Text")
+        self.send_btn = QtWidgets.QPushButton("Send Soup")
+
+        # Assemble layout
+        layout.addLayout(url_layout)
+        layout.addWidget(self.status_label)
+        layout.addWidget(QtWidgets.QLabel("User-Agent:"))
+        layout.addWidget(self.user_agent_box)
+        layout.addWidget(QtWidgets.QLabel("Source Code:"))
+        layout.addWidget(self.source_code_edit)
+        layout.addWidget(QtWidgets.QLabel("Soup Result:"))
+        layout.addWidget(self.soup_result_edit)
+
+        btn_layout = QtWidgets.QHBoxLayout()
+        btn_layout.addWidget(self.action_btn)
+        btn_layout.addWidget(self.get_text_btn)
+        btn_layout.addWidget(self.send_btn)
+        layout.addLayout(btn_layout)
+
+        self.setLayout(layout)
+
+    def setup_logic(self):
+        self.grab_btn.clicked.connect(self.grab_url)
+        self.action_btn.clicked.connect(self.parse_html)
+        self.get_text_btn.clicked.connect(self.get_all_text)
+        self.send_btn.clicked.connect(self.send_soup)
+
+    def init_managers(self):
+        self.url_mgr = urlManager(url=self.initial_url)
+        self.request_mgr = None
+        self.soup_mgr = None
+        self.link_mgr = None
+
+    def grab_url(self):
+        url = self.url_input.text().strip()
+        self.url_mgr = urlManager(url=url)
+        self.request_mgr = requestManager(url_mgr=self.url_mgr)
+        if self.request_mgr.source_code:
+            self.soup_mgr = SoupManager(url_mgr=self.url_mgr, request_mgr=self.request_mgr)
+            self.link_mgr = LinkManager(url_mgr=self.url_mgr, request_mgr=self.request_mgr, soup_mgr=self.soup_mgr)
+            self.status_label.setText("Status: Success")
+            self.source_code_edit.setPlainText(self.request_mgr.source_code)
+        else:
+            self.status_label.setText("Status: Failed")
+
+    def parse_html(self):
+        if self.soup_mgr:
+            self.soup_result_edit.setPlainText(self.soup_mgr.soup)
+
+    def get_all_text(self):
+        if self.soup_mgr:
+            self.soup_result_edit.setPlainText(self.soup_mgr.extract_text_sections())
+
+    def send_soup(self):
+        soup = self.soup_result_edit.toPlainText()
+        print("Soup sent:", soup[:300])  # or emit a signal
+

{abstract_webtools-0.1.6.121 → abstract_webtools-0.1.6.122}/src/abstract_webtools.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: abstract_webtools
-Version: 0.1.6.121
+Version: 0.1.6.122
 Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
 Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
 Author: putkoff

abstract_webtools-0.1.6.121/src/abstract_webtools/url_grabber_new.py

@@ -1,45 +0,0 @@
-from abstract_gui import AbstractWindowManager,make_component
-from abstract_webtools import UserAgentManager,UrlManager,SafeRequest,SoupManager,LinkManager,CipherManager
-
-class GuiGrabber:
-    def __init__(self,url="www.example.com"):
-        self.window_mgr = AbstractWindowManager()
-        self.window_name = self.window_mgr.add_window(title="Gui_Grabber",layout=[],event_handlers=[self.while_window])
-        self.url = url
-        self.parse_type_choices = ['html.parser', 'lxml', 'html5lib']
-        self.window_mgr.while_window()
-    def layout(event,values,window):
-        # Add a dropdown for selecting BeautifulSoup parsing capabilities
-        make_component("theme",'LightGrey1')
-        layout = [[make_component("Text",'URL:', size=(8, 1)),
-                   make_component("Input",url, key='-URL-',enable_events=True),
-                   make_component("Text",'status:'),
-                   make_component("Text",'200',key="-STATUS_CODE-"),
-                   make_component("Text",f'success: {self.url} is valid',key="-URL_WARNING-"),
-                   make_component("Button",'Grab URL',key='-GRAB_URL-',visible=True)],
-            [make_component("Checkbox",'Custom User-Agent', default=False, key='-CUSTOMUA-', enable_events=True)],
-            [make_component("Text",'User-Agent:', size=(8, 1)),
-             make_component("Combo",get_user_agents(), default_value='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36', key='-USERAGENT-', disabled=False)],
-            [self.get_cypher_checks()],
-            [make_component("Button",'Grab URL'),
-             make_component("Button",'Action'),
-             make_component("Button",'Get All Text')],
-            [make_component("Text",'Parsing Capabilities:', size=(15, 1)),
-             make_component("DropDown",parse_type_choices, default_value='html.parser', key='-parse_type-',enable_events=True)],
-            [get_multi_line({"key":'-SOURCECODE-'})],
-            [make_component("Text",'find soup:'),[[
-                make_component("Checkbox",'',default=True,key='-SOUP_TAG_BOOL-',enable_events=True),
-                make_component("Combo",[], size=(15, 1),key='-SOUP_TAG-',enable_events=True)],
-                                    [make_component("Checkbox",'',default=False,key='-SOUP_ATTRIBUTE_BOOL-',enable_events=True),
-                                     make_component("Combo",[], size=(15, 1),key='-SOUP_ATTRIBUTE-',enable_events=True)],
-                                    [make_component("Checkbox",'',default=False,key='-SOUP_ATTRIBUTE_1_BOOL-',enable_events=True),
-                                     make_component("Combo",[], size=(15, 1),key='-SOUP_ATTRIBUTE_1-',enable_events=True)],
-                                    [make_component("Checkbox",'',default=False,key='-SOUP_ATTRIBUTE_2_BOOL-',enable_events=True),
-                                     make_component("Combo",[], size=(15, 1),key='-SOUP_ATTRIBUTE_2-',enable_events=True)],
-                                    make_component("Input",key='-SOUP_VALUES_INPUT-'),
-                                                  make_component("Button",'get soup'),
-                                                  make_component("Button",'all soup'),
-                                                  make_component("Button",'Send Soup')]],
-                  [get_multi_line({"key":"-FIND_ALL_OUTPUT-"})]]
-        return layout
-GuiGrabber()