abstract-webtools 0.1.6.96__tar.gz → 0.1.6.98__tar.gz

abstract_webtools-0.1.6.98/PKG-INFO

@@ -0,0 +1,196 @@
+Metadata-Version: 2.4
+Name: abstract_webtools
+Version: 0.1.6.98
+Summary: Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.
+Home-page: https://github.com/AbstractEndeavors/abstract_essentials/tree/main/abstract_webtools
+Author: putkoff
+Author-email: partners@abstractendeavors.com
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+Requires-Dist: abstract_utilities>=0.2.2.30
+Requires-Dist: PySimpleGUI>=4.60.5
+Requires-Dist: urllib3>=2.0.4
+Requires-Dist: requests>=2.31.0
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Unknown Package (vUnknown Version)
+
+No description available
+
+## Installation
+
+```bash
+pip install Unknown Package
+```
+
+## Dependencies
+
+None
+
+## Modules
+
+### src/abstract_webtools/url_grabber.py
+
+Description of script based on prompt: You are analyzing a Python script 'url_grabber.py' (mock response)
+
+### src/abstract_webtools/managers/__init__.py
+
+Description of script based on prompt: You are analyzing a Python script '__init__.py' lo (mock response)
+
+### src/abstract_webtools/managers/cipherManager.py
+
+Description of script based on prompt: You are analyzing a Python script 'cipherManager.p (mock response)
+
+### src/abstract_webtools/managers/linkManager/__init__.py
+
+Description of script based on prompt: You are analyzing a Python script '__init__.py' lo (mock response)
+
+### src/abstract_webtools/managers/videoDownloader.py
+
+Description of script based on prompt: You are analyzing a Python script 'videoDownloader (mock response)
+
+### src/abstract_webtools/managers/tlsAdapter.py
+
+Description of script based on prompt: You are analyzing a Python script 'tlsAdapter.py'  (mock response)
+
+### src/abstract_webtools/managers/soupManager/soupManager.py
+
+Description of script based on prompt: You are analyzing a Python script 'soupManager.py' (mock response)
+
+### src/abstract_webtools/managers/urlManager/__init__.py
+
+Description of script based on prompt: You are analyzing a Python script '__init__.py' lo (mock response)
+
+### src/abstract_webtools/url_grabber_new.py
+
+Description of script based on prompt: You are analyzing a Python script 'url_grabber_new (mock response)
+
+### src/abstract_webtools/managers/requestManager/__init__.py
+
+Description of script based on prompt: You are analyzing a Python script '__init__.py' lo (mock response)
+
+### src/abstract_webtools/managers/get_test.py
+
+Description of script based on prompt: You are analyzing a Python script 'get_test.py' lo (mock response)
+
+### src/abstract_webtools/__init__.py
+
+Description of script based on prompt: You are analyzing a Python script '__init__.py' lo (mock response)
+
+### src/abstract_webtools/managers/seleniumManager.py
+
+Description of script based on prompt: You are analyzing a Python script 'seleniumManager (mock response)
+
+### src/abstract_webtools/managers/crawlManager.py
+
+Description of script based on prompt: You are analyzing a Python script 'crawlManager.py (mock response)
+
+### src/abstract_webtools/managers/requestManager/requestManager.py
+
+Description of script based on prompt: You are analyzing a Python script 'requestManager. (mock response)
+
+### src/abstract_webtools/managers/videoDownloader2.py
+
+Description of script based on prompt: You are analyzing a Python script 'videoDownloader (mock response)
+
+### src/abstract_webtools/managers/soupManager/__init__.py
+
+Description of script based on prompt: You are analyzing a Python script '__init__.py' lo (mock response)
+
+### src/abstract_webtools/main.py
+
+Description of script based on prompt: You are analyzing a Python script 'main.py' locate (mock response)
+
+### src/testit.py
+
+Description of script based on prompt: You are analyzing a Python script 'testit.py' loca (mock response)
+
+### src/abstract_webtools/soup_gui.py
+
+Description of script based on prompt: You are analyzing a Python script 'soup_gui.py' lo (mock response)
+
+### src/abstract_webtools/managers/curlMgr.py
+
+Description of script based on prompt: You are analyzing a Python script 'curlMgr.py' loc (mock response)
+
+### src/abstract_webtools/managers/sslManager.py
+
+Description of script based on prompt: You are analyzing a Python script 'sslManager.py'  (mock response)
+
+### src/abstract_webtools/managers/dynamicRateLimiter.py
+
+Description of script based on prompt: You are analyzing a Python script 'dynamicRateLimi (mock response)
+
+### src/abstract_webtools/managers/mySocketClient.py
+
+Description of script based on prompt: You are analyzing a Python script 'mySocketClient. (mock response)
+
+### src/abstract_webtools/managers/linkManager/linkManager.py
+
+Description of script based on prompt: You are analyzing a Python script 'linkManager.py' (mock response)
+
+### src/abstract_webtools/managers/userAgentManager.py
+
+Description of script based on prompt: You are analyzing a Python script 'userAgentManage (mock response)
+
+### src/abstract_webtools/extention_list.py
+
+Description of script based on prompt: You are analyzing a Python script 'extention_list. (mock response)
+
+### src/abstract_webtools/managers/networkManager.py
+
+Description of script based on prompt: You are analyzing a Python script 'networkManager. (mock response)
+
+### src/abstract_webtools/managers/domainManager.py
+
+Description of script based on prompt: You are analyzing a Python script 'domainManager.p (mock response)
+
+### src/abstract_webtools/managers/urlManager/urlManager.py
+
+Description of script based on prompt: You are analyzing a Python script 'urlManager.py'  (mock response)
+
+### src/abstract_webtools/big_user_agent_list.py
+
+Description of script based on prompt: You are analyzing a Python script 'big_user_agent_ (mock response)
+
+### src/abstract_webtools/abstract_usurpit.py
+
+Description of script based on prompt: You are analyzing a Python script 'abstract_usurpi (mock response)
+
+### src/abstract_webtools/managers/soupManager/asoueces.py
+
+Description of script based on prompt: You are analyzing a Python script 'asoueces.py' lo (mock response)
+
+### src/abstract_webtools/managers/crawlmgr2.py
+
+Description of script based on prompt: You are analyzing a Python script 'crawlmgr2.py' l (mock response)
+
+### src/abstract_webtools/abstract_webtools.py
+
+Description of script based on prompt: You are analyzing a Python script 'abstract_webtoo (mock response)
+
+### src/abstract_webtools/find_dirs.py
+
+Description of script based on prompt: You are analyzing a Python script 'find_dirs.py' l (mock response)
+
+### src/abstract_webtools/domain_identifier.py
+
+Description of script based on prompt: You are analyzing a Python script 'domain_identifi (mock response)
+
+### src/abstract_webtools/managers/allss\.py
+
+Description of script based on prompt: You are analyzing a Python script 'allss\.py' loca (mock response)
+

abstract_webtools-0.1.6.98/README.md

@@ -0,0 +1,168 @@
+# Unknown Package (vUnknown Version)
+
+No description available
+
+## Installation
+
+```bash
+pip install Unknown Package
+```
+
+## Dependencies
+
+None
+
+## Modules
+
+### src/abstract_webtools/url_grabber.py
+
+Description of script based on prompt: You are analyzing a Python script 'url_grabber.py' (mock response)
+
+### src/abstract_webtools/managers/__init__.py
+
+Description of script based on prompt: You are analyzing a Python script '__init__.py' lo (mock response)
+
+### src/abstract_webtools/managers/cipherManager.py
+
+Description of script based on prompt: You are analyzing a Python script 'cipherManager.p (mock response)
+
+### src/abstract_webtools/managers/linkManager/__init__.py
+
+Description of script based on prompt: You are analyzing a Python script '__init__.py' lo (mock response)
+
+### src/abstract_webtools/managers/videoDownloader.py
+
+Description of script based on prompt: You are analyzing a Python script 'videoDownloader (mock response)
+
+### src/abstract_webtools/managers/tlsAdapter.py
+
+Description of script based on prompt: You are analyzing a Python script 'tlsAdapter.py'  (mock response)
+
+### src/abstract_webtools/managers/soupManager/soupManager.py
+
+Description of script based on prompt: You are analyzing a Python script 'soupManager.py' (mock response)
+
+### src/abstract_webtools/managers/urlManager/__init__.py
+
+Description of script based on prompt: You are analyzing a Python script '__init__.py' lo (mock response)
+
+### src/abstract_webtools/url_grabber_new.py
+
+Description of script based on prompt: You are analyzing a Python script 'url_grabber_new (mock response)
+
+### src/abstract_webtools/managers/requestManager/__init__.py
+
+Description of script based on prompt: You are analyzing a Python script '__init__.py' lo (mock response)
+
+### src/abstract_webtools/managers/get_test.py
+
+Description of script based on prompt: You are analyzing a Python script 'get_test.py' lo (mock response)
+
+### src/abstract_webtools/__init__.py
+
+Description of script based on prompt: You are analyzing a Python script '__init__.py' lo (mock response)
+
+### src/abstract_webtools/managers/seleniumManager.py
+
+Description of script based on prompt: You are analyzing a Python script 'seleniumManager (mock response)
+
+### src/abstract_webtools/managers/crawlManager.py
+
+Description of script based on prompt: You are analyzing a Python script 'crawlManager.py (mock response)
+
+### src/abstract_webtools/managers/requestManager/requestManager.py
+
+Description of script based on prompt: You are analyzing a Python script 'requestManager. (mock response)
+
+### src/abstract_webtools/managers/videoDownloader2.py
+
+Description of script based on prompt: You are analyzing a Python script 'videoDownloader (mock response)
+
+### src/abstract_webtools/managers/soupManager/__init__.py
+
+Description of script based on prompt: You are analyzing a Python script '__init__.py' lo (mock response)
+
+### src/abstract_webtools/main.py
+
+Description of script based on prompt: You are analyzing a Python script 'main.py' locate (mock response)
+
+### src/testit.py
+
+Description of script based on prompt: You are analyzing a Python script 'testit.py' loca (mock response)
+
+### src/abstract_webtools/soup_gui.py
+
+Description of script based on prompt: You are analyzing a Python script 'soup_gui.py' lo (mock response)
+
+### src/abstract_webtools/managers/curlMgr.py
+
+Description of script based on prompt: You are analyzing a Python script 'curlMgr.py' loc (mock response)
+
+### src/abstract_webtools/managers/sslManager.py
+
+Description of script based on prompt: You are analyzing a Python script 'sslManager.py'  (mock response)
+
+### src/abstract_webtools/managers/dynamicRateLimiter.py
+
+Description of script based on prompt: You are analyzing a Python script 'dynamicRateLimi (mock response)
+
+### src/abstract_webtools/managers/mySocketClient.py
+
+Description of script based on prompt: You are analyzing a Python script 'mySocketClient. (mock response)
+
+### src/abstract_webtools/managers/linkManager/linkManager.py
+
+Description of script based on prompt: You are analyzing a Python script 'linkManager.py' (mock response)
+
+### src/abstract_webtools/managers/userAgentManager.py
+
+Description of script based on prompt: You are analyzing a Python script 'userAgentManage (mock response)
+
+### src/abstract_webtools/extention_list.py
+
+Description of script based on prompt: You are analyzing a Python script 'extention_list. (mock response)
+
+### src/abstract_webtools/managers/networkManager.py
+
+Description of script based on prompt: You are analyzing a Python script 'networkManager. (mock response)
+
+### src/abstract_webtools/managers/domainManager.py
+
+Description of script based on prompt: You are analyzing a Python script 'domainManager.p (mock response)
+
+### src/abstract_webtools/managers/urlManager/urlManager.py
+
+Description of script based on prompt: You are analyzing a Python script 'urlManager.py'  (mock response)
+
+### src/abstract_webtools/big_user_agent_list.py
+
+Description of script based on prompt: You are analyzing a Python script 'big_user_agent_ (mock response)
+
+### src/abstract_webtools/abstract_usurpit.py
+
+Description of script based on prompt: You are analyzing a Python script 'abstract_usurpi (mock response)
+
+### src/abstract_webtools/managers/soupManager/asoueces.py
+
+Description of script based on prompt: You are analyzing a Python script 'asoueces.py' lo (mock response)
+
+### src/abstract_webtools/managers/crawlmgr2.py
+
+Description of script based on prompt: You are analyzing a Python script 'crawlmgr2.py' l (mock response)
+
+### src/abstract_webtools/abstract_webtools.py
+
+Description of script based on prompt: You are analyzing a Python script 'abstract_webtoo (mock response)
+
+### src/abstract_webtools/find_dirs.py
+
+Description of script based on prompt: You are analyzing a Python script 'find_dirs.py' l (mock response)
+
+### src/abstract_webtools/domain_identifier.py
+
+Description of script based on prompt: You are analyzing a Python script 'domain_identifi (mock response)
+
+### src/abstract_webtools/managers/allss\.py
+
+Description of script based on prompt: You are analyzing a Python script 'allss\.py' loca (mock response)
+

{abstract_webtools-0.1.6.96 → abstract_webtools-0.1.6.98}/setup.py

@@ -4,7 +4,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 setuptools.setup(
     name='abstract_webtools',
-    version='0.1.6.96',
+    version='0.1.6.98',
     author='putkoff',
     author_email='partners@abstractendeavors.com',
     description='Abstract Web Tools is a Python package that provides various utility functions for web scraping tasks. It is built on top of popular libraries such as `requests`, `BeautifulSoup`, and `urllib3` to simplify the process of fetching and parsing web content.',

abstract_webtools-0.1.6.98/src/abstract_webtools/k2s_downloader.py

@@ -0,0 +1,129 @@
+import os
+import re
+import time
+import requests
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from abstract_security import *
+from abstract_webtools import *
+DOWNLOAD_DIR = os.path.abspath("./downloads")
+os.makedirs(DOWNLOAD_DIR, exist_ok=True)
+class K2SDownloader:
+    def __init__(self,env_path=None):
+        self.env_path = env_path
+        self.session = requests.Session()
+        self.session.headers.update({
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
+        })
+        self.driver = self._init_driver()
+        self.logged_in = False
+
+    def _init_driver(self):
+        options = webdriver.ChromeOptions()
+        options.add_argument("--disable-blink-features=AutomationControlled")
+        options.add_argument("--headless")
+        return webdriver.Chrome(options=options)
+
+    def login(self):
+        userName = get_env_value('userName',path=self.env_path)
+        passWord = get_env_value('passWord',path=self.env_path)
+
+        self.driver.get("https://k2s.cc/auth/login")
+        time.sleep(3)
+
+        
+        email_input = self.driver.find_element(By.NAME, "email")
+        password_input = self.driver.find_element(By.NAME, "input-password-auto-complete-on")
+        email_input.send_keys(userName)
+        password_input.send_keys(passWord)
+        password_input.send_keys(Keys.RETURN)
+
+        #WebDriverWait(self.driver, 20).until(
+        #    EC.presence_of_element_located((By.XPATH, "//a[contains(text(), 'Logout')]"))
+        #)
+        self.logged_in = True
+        print("Login successful")
+        #except Exception as e:
+        #    print(f"Login failed: {e}")
+        #    with open('login_error.html', 'w', encoding='utf-8') as f:
+        #        f.write(self.driver.page_source)
+
+    def download_file(self, url):
+        if not self.logged_in:
+            self.login()
+
+        print(f"Navigating to: {url}")
+        self.driver.get(url)
+        time.sleep(5)
+
+        if 'captcha' in self.driver.page_source.lower():
+            print("CAPTCHA detected. Manual intervention required.")
+            return
+
+        try:
+            download_button = WebDriverWait(self.driver, 30).until(
+                EC.element_to_be_clickable((By.CSS_SELECTOR, 'a[href*="/download"], button[class*="download"]'))
+            )
+            print("Download button found; attempting to click or fetch URL")
+            download_url = download_button.get_attribute('href')
+
+            if download_url:
+                response = self.session.get(download_url, stream=True)
+                file_name = self._extract_filename(response, download_url)
+                file_path = os.path.join(DOWNLOAD_DIR, file_name)
+
+                with open(file_path, 'wb') as f:
+                    for chunk in response.iter_content(chunk_size=8192):
+                        f.write(chunk)
+                print(f"Downloaded: {file_path}")
+                return file_path
+            else:
+                download_button.click()
+                print("Button clicked. Waiting for download...")
+                time.sleep(30)  # adjust as needed
+        except Exception as e:
+            print(f"Download failed for {url}: {e}")
+
+    def _extract_filename(self, response, url):
+        cd = response.headers.get('Content-Disposition', '')
+        if 'filename=' in cd:
+            return cd.split('filename=')[-1].strip('"')
+        return url.split('/')[-1].split('?')[0]
+
+class dlsManager:
+    def __init__(self, downloader):
+        self.downloader = downloader
+        self.all_dls = []
+
+    def is_prev_dl(self, data):
+        k2s_link = data.get('k2s')
+        for prev_data in self.all_dls:
+            if prev_data.get('k2s') == k2s_link:
+                return True
+        self.all_dls.append(data)
+        return False
+
+    def dl_k2s_link(self, k2s_link):
+        if k2s_link:
+            print(f"Downloading: {k2s_link}")
+            self.downloader.download_file(k2s_link)
+            time.sleep(10)
+
+
+def get_soup(url):
+    try:
+        resp = requests.get(url)
+        resp.raise_for_status()
+        return BeautifulSoup(resp.text, 'html.parser')
+    except Exception as e:
+        print(f"Failed to fetch soup for {url}: {e}")
+        return None
+
+def get_k2s_link(soup):
+    match = re.search(r'https://k2s\.cc/file/[^"<]+', str(soup))
+    return match.group(0) if match else None