orbweaver-tools 1.1.0__tar.gz

orbweaver_tools-1.1.0/LICENSE

@@ -0,0 +1,22 @@
+
+MIT License
+
+Copyright (c) 2025 Tom Freeman
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

orbweaver_tools-1.1.0/PKG-INFO

@@ -0,0 +1,124 @@
+Metadata-Version: 2.1
+Name: orbweaver-tools
+Version: 1.1.0
+Summary: Stealth-oriented utilities for web scraping with rotating user agents, proxies, and Selenium helpers.
+Home-page: https://github.com/Tom3man/orb-weaver
+License: MIT
+Keywords: web-scraping,selenium,proxies,automation
+Author: Tom Freeman
+Requires-Python: >=3.10,<4.0
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Software Development :: Libraries
+Requires-Dist: bs4 (>=0.0.1,<0.0.2)
+Requires-Dist: fake-useragent (>=1.4.0,<2.0.0)
+Requires-Dist: pandas (>=2.1.4,<3.0.0)
+Requires-Dist: requests (>=2.31.0,<3.0.0)
+Requires-Dist: selenium (>=4.16.0,<5.0.0)
+Requires-Dist: webdriver-manager (>=4.0.1,<5.0.0)
+Project-URL: Repository, https://github.com/Tom3man/orb-weaver
+Description-Content-Type: text/markdown
+
+# Orb Weaver
+
+Utilities for stealth-oriented web scraping with Selenium helpers, rotating headers, and proxy support.
+
+## Features
+
+- Selenium driver bootstrap via `OrbDriver`
+- Randomized user-agent and request headers
+- Proxy harvesting and validation utilities
+- Human-like browser interaction helpers (`slow_type`, random clicking, scroll, viewport changes)
+- Optional PIA VPN integration for IP rotation
+
+## Installation
+
+From PyPI (after publish):
+
+```bash
+pip install orbweaver-tools
+```
+
+From source with Poetry:
+
+```bash
+poetry install
+```
+
+## Quick Start
+
+### Build a Selenium driver
+
+```python
+from orb.spinner.core.driver import OrbDriver
+
+orb_driver = OrbDriver(use_pia=False)
+driver = orb_driver.get_webdriver(url="https://example.com")
+```
+
+### Send spoofed requests
+
+```python
+from bs4 import BeautifulSoup
+from orb.scraper.utils import spoof_request
+
+response = spoof_request("https://example.com", use_proxies=False)
+soup = BeautifulSoup(response.content, "html.parser")
+```
+
+### Human-like interactions
+
+```python
+from selenium.webdriver.common.by import By
+from orb.spinner.utils import slow_type, human_clicking
+
+input_box = driver.find_element(By.ID, "search")
+slow_type(input_box, "hello world", send_keys=True)
+
+button = driver.find_element(By.ID, "submit")
+human_clicking(driver, button)
+```
+
+## Development
+
+Run tests:
+
+```bash
+poetry run pytest -q
+```
+
+Build package artifacts:
+
+```bash
+poetry build
+```
+
+## Publish to PyPI
+
+1. Create an account on [PyPI](https://pypi.org/).
+2. Create an API token and configure Poetry credentials:
+
+```bash
+poetry config pypi-token.pypi <your-token>
+```
+
+3. Publish:
+
+```bash
+poetry publish --build
+```
+
+## License
+
+MIT. See [`LICENSE`](LICENSE).
+
+## Notes
+
+Use these tools responsibly and only against systems where you have permission to automate or scrape.
+

orbweaver_tools-1.1.0/README.md

@@ -0,0 +1,96 @@
+# Orb Weaver
+
+Utilities for stealth-oriented web scraping with Selenium helpers, rotating headers, and proxy support.
+
+## Features
+
+- Selenium driver bootstrap via `OrbDriver`
+- Randomized user-agent and request headers
+- Proxy harvesting and validation utilities
+- Human-like browser interaction helpers (`slow_type`, random clicking, scroll, viewport changes)
+- Optional PIA VPN integration for IP rotation
+
+## Installation
+
+From PyPI (after publish):
+
+```bash
+pip install orbweaver-tools
+```
+
+From source with Poetry:
+
+```bash
+poetry install
+```
+
+## Quick Start
+
+### Build a Selenium driver
+
+```python
+from orb.spinner.core.driver import OrbDriver
+
+orb_driver = OrbDriver(use_pia=False)
+driver = orb_driver.get_webdriver(url="https://example.com")
+```
+
+### Send spoofed requests
+
+```python
+from bs4 import BeautifulSoup
+from orb.scraper.utils import spoof_request
+
+response = spoof_request("https://example.com", use_proxies=False)
+soup = BeautifulSoup(response.content, "html.parser")
+```
+
+### Human-like interactions
+
+```python
+from selenium.webdriver.common.by import By
+from orb.spinner.utils import slow_type, human_clicking
+
+input_box = driver.find_element(By.ID, "search")
+slow_type(input_box, "hello world", send_keys=True)
+
+button = driver.find_element(By.ID, "submit")
+human_clicking(driver, button)
+```
+
+## Development
+
+Run tests:
+
+```bash
+poetry run pytest -q
+```
+
+Build package artifacts:
+
+```bash
+poetry build
+```
+
+## Publish to PyPI
+
+1. Create an account on [PyPI](https://pypi.org/).
+2. Create an API token and configure Poetry credentials:
+
+```bash
+poetry config pypi-token.pypi <your-token>
+```
+
+3. Publish:
+
+```bash
+poetry publish --build
+```
+
+## License
+
+MIT. See [`LICENSE`](LICENSE).
+
+## Notes
+
+Use these tools responsibly and only against systems where you have permission to automate or scrape.

orbweaver_tools-1.1.0/orb/__init__.py

@@ -0,0 +1,14 @@
+import logging
+import os
+from importlib.metadata import PackageNotFoundError, version
+
+MODULE_PATH = os.path.dirname(os.path.realpath(__file__))
+REPO_PATH = os.path.dirname(MODULE_PATH)
+
+try:
+    __version__ = version("orbweaver-tools")
+except PackageNotFoundError:
+    __version__ = "0.0.0"
+
+log = logging.getLogger("orb")
+log.addHandler(logging.NullHandler())

orbweaver_tools-1.1.0/orb/common/design/welcome_page.py

@@ -0,0 +1,99 @@
+import os
+import socket
+from typing import Optional
+
+import requests
+from selenium.webdriver.remote.webdriver import WebDriver
+
+from orb import REPO_PATH
+from orb.spinner.utils import get_user_agent
+
+
+def create_welcome_page(
+    proxy_info: Optional[str] = None,
+    user_agent: Optional[str] = None,
+) -> str:
+    """
+    Create a welcome page HTML content with IP, proxy, and user agent information.
+
+    Args:
+        proxy_info (str, optional): Proxy information. Defaults to None.
+        user_agent (str, optional): User agent information. Defaults to None.
+
+    Returns:
+        str: The HTML content of the welcome page.
+    """
+    ip_information = f"""
+        <h2>IP Information</h2>
+        <p>Local IP Address: {get_local_ip()}</p>
+        <p>Public IP Address: {get_public_ip()}</p>
+    """
+
+    proxy_info_section = f"""
+        <h2>Proxy Information</h2>
+        <p>Proxy: {proxy_info}</p>
+    """ if proxy_info else ""
+
+    user_agent_section = f"""
+        <h2>User Agent Information</h2>
+        <p>User Agent: {user_agent}</p>
+    """ if user_agent else ""
+
+    page_content = f"""
+    <html>
+    <body>
+        <h1>Welcome to the Orb Weaver Project!</h1>
+        {ip_information}
+        {proxy_info_section}
+        {user_agent_section}
+    </body>
+    </html>
+    """
+    return page_content
+
+
+def get_local_ip() -> str:
+    """
+    Get the local IP address of the machine.
+
+    Returns:
+        str: The local IP address.
+    """
+    return socket.gethostbyname(socket.gethostname())
+
+
+def get_public_ip() -> str:
+    """
+    Get the public IP address of the machine.
+
+    Returns:
+        str: The public IP address.
+    """
+    response = requests.get('https://api.ipify.org')
+    return response.text
+
+
+def build_welcome_page(
+    driver: WebDriver,
+    proxy_info: Optional[str] = None,
+) -> None:
+    """
+    Build a welcome page using the provided WebDriver and proxy info.
+    Webpage is then rendered with the active driver and then deleted to not store any sensitive information locally.
+    Args:
+        driver (WebDriver): The WebDriver instance.
+        proxy_info (str): Proxy information.
+        user_agent (str, optional): User agent information. Defaults to None.
+    """
+
+    page_content = create_welcome_page(
+        user_agent=get_user_agent(driver=driver),
+        proxy_info=proxy_info,
+    )
+
+    file_path = f"{REPO_PATH}/proxy_info.html"
+    with open(file_path, 'w') as file:
+        file.write(page_content)
+
+    driver.get(f'file://{file_path}')
+    os.remove(file_path)

orbweaver_tools-1.1.0/orb/common/proxies/get_proxies.py

@@ -0,0 +1,111 @@
+import logging
+from datetime import datetime
+from typing import Dict
+
+import pandas as pd
+import requests
+from bs4 import BeautifulSoup, Tag
+
+from orb.common.proxies.test_proxies import test_proxy
+
+log = logging.getLogger(__name__)
+
+
+class GetProxies:
+    """
+    A class for retrieving and working with proxy information.
+    """
+
+    PROXY_SITE = "https://free-proxy-list.net/"
+
+    def __init__(self) -> None:
+        """
+        Initializes the GetProxies object and sets the current date and time.
+        """
+        todays_datetime = datetime.now()
+        self.date_now = todays_datetime.strftime("%Y-%m-%d")
+        self.time_now = todays_datetime.strftime("%H:%M")
+
+    def request_proxies(self) -> requests.Response:
+        """
+        Sends a request to the proxy URL and returns the response object.
+
+        Returns:
+            requests.Response: The response object from the request.
+        """
+        return requests.get(self.PROXY_SITE)
+
+    def parse_requests(self) -> BeautifulSoup:
+        """
+        Parses the returned request from request_proxies and returns a BeautifulSoup object.
+
+        Returns:
+            BeautifulSoup: The BeautifulSoup object representing the parsed HTML.
+        """
+        return BeautifulSoup(self.request_proxies().content, 'html.parser')
+
+    def extract_table_html(self) -> Tag:
+        """
+        Extracts the raw HTML of the table containing the proxy information.
+
+        Returns:
+            BeautifulSoup: The BeautifulSoup object representing the extracted HTML.
+        """
+        return self.parse_requests().find('table')
+
+    def return_proxy_table(self, https_only: bool = True) -> pd.DataFrame:
+        """
+        Iterates through the HTML table to build a pandas DataFrame of proxies.
+
+        Args:
+            https_only (bool, optional): Whether to return only proxies with HTTPS support.
+
+        Returns:
+            pd.DataFrame: The pandas DataFrame containing the proxy information.
+        """
+        df = pd.DataFrame()
+        headers = None
+        for tr in self.extract_table_html().find_all('tr'):
+            if not headers:
+                headers = [
+                    td.text.upper().replace(" ", "_") for td in tr.find_all(['th', 'td'])
+                ]
+                df = pd.DataFrame(columns=headers)
+                continue
+
+            df.loc[len(df)] = [td.text for td in tr.find_all(['th', 'td'])]
+
+        if https_only:
+            return df[df['HTTPS'] == 'yes']
+        return df
+
+    def build_proxy_dict(self) -> None:
+        """
+        Builds a dictionary of HTTP and HTTPS proxy values from the proxy table DataFrame.
+        """
+        proxy_table = self.return_proxy_table(https_only=True)
+        proxy_row = proxy_table.sample(1)
+        ip_address = proxy_row['IP_ADDRESS'].values[0]
+        port = proxy_row['PORT'].values[0]
+
+        self.proxies = {
+            "http": f"{ip_address}:{port}",
+            "https": f"{ip_address}:{port}",
+        }
+
+    @property
+    def proxy_dict(self) -> Dict[str, str]:
+        """
+        Property that returns the proxy dictionary.
+
+        Returns:
+            Dict[str, str]: The dictionary containing HTTP and HTTPS proxy values.
+
+        Raises:
+            RuntimeError: If a working proxy cannot be found after maximum retries.
+        """
+
+        self.build_proxy_dict()
+        if test_proxy(proxies=self.proxies):
+            return self.proxies
+        raise RuntimeError("Failed to find a working proxy.")

orbweaver_tools-1.1.0/orb/common/proxies/test_proxies.py

@@ -0,0 +1,31 @@
+import logging
+from typing import Dict
+
+import requests
+
+log = logging.getLogger(__name__)
+__test__ = False
+
+
+def test_proxy(proxies: Dict[str, str]) -> bool:
+    """
+    Test the functionality of a proxy by making a request to a sample URL.
+
+    Args:
+        proxies (Dict[str, str]): Dictionary containing HTTP and HTTPS proxies.
+
+    Returns:
+        bool: True if the proxy is working, False otherwise.
+    """
+    try:
+        url = 'http://www.example.com'
+        response = requests.get(url, proxies=proxies, timeout=5)
+        if response.status_code == 200:
+            log.info(f"{proxies['https']} Proxy is working!")
+            return True
+        else:
+            log.error(f"{proxies['https']} Proxy is NOT working!")
+            return False
+    except requests.exceptions.RequestException:
+        log.error("Unable to connect to the proxy.")
+        return False

orbweaver_tools-1.1.0/orb/common/user_agents/user_agents.py

@@ -0,0 +1,62 @@
+import random
+from typing import Dict
+
+from fake_useragent import UserAgent
+
+
+class GetUserAgent:
+    """
+    A class for retrieving random User-Agent headers for web scraping.
+    """
+
+    def __init__(self) -> None:
+        """
+        Initializes the GetUserAgent object.
+        Sets up the UserAgent instance.
+        """
+        self.user_agent = UserAgent()
+        self.accept_languages = ["en-US", "en-GB", "fr-FR", "es-ES"]
+        self.referer_urls = [
+            "https://www.google.com",
+            "https://www.bing.com",
+            "https://search.yahoo.com",
+            "https://en.wikipedia.org",
+            "https://www.reddit.com",
+            "https://twitter.com",
+            "https://www.facebook.com",
+            "https://www.youtube.com",
+            "https://www.amazon.com",
+            "https://www.cnn.com",
+            "https://www.bbc.com",
+            "https://www.reuters.com",
+        ]
+
+    def get_random_referer(self) -> str:
+        """
+        Returns a random referer URL.
+
+        Returns:
+            str: A randomly chosen referer URL.
+        """
+        return random.choice(self.referer_urls)
+
+    @property
+    def headers_dict(self) -> Dict[str, "UserAgent"]:
+        """
+        Returns a dictionary containing the User-Agent header.
+
+        Returns:
+            dict: A dictionary with the 'User-Agent' key, random user agent, and a randomly chosen referer URL.
+        """
+        return {
+            "User-Agent": self.user_agent.random,
+            "Accept-Language": random.choice(self.accept_languages),
+            "Referer": self.get_random_referer(),
+            "Cache-Control": "max-age=0" if random.random() < 0.5 else "no-cache",
+            "Upgrade-Insecure-Requests": "1" if random.random() < 0.5 else "0",
+            "DNT": "1" if random.random() < 0.3 else "0",
+            "X-Forwarded-For": ".".join(str(random.randint(0, 255)) for _ in range(4)),
+            "X-Requested-With": "XMLHttpRequest" if random.random() < 0.5 else "",
+            "X-Frame-Options": "DENY" if random.random() < 0.5 else "SAMEORIGIN",
+            "Connection": "keep-alive" if random.random() < 0.8 else "close",
+        }

orbweaver_tools-1.1.0/orb/common/vpn/__init__.py

@@ -0,0 +1,5 @@
+from .pia import PiaVpn
+
+__all__ = [
+    PiaVpn,
+]