webscout 8.3.1__py3-none-any.whl → 8.3.3__py3-none-any.whl

webscout/litagent/Readme.md

@@ -1,6 +1,8 @@
 # 🔥 LitAgent - The Lit User Agent Generator
 
-LitAgent is a powerful and modern user agent generator that keeps your requests fresh and undetectable! Built with style and packed with features, it's your go-to solution for managing user agents in your web scraping projects.
+LitAgent is a powerful, modern user agent generator that keeps your requests fresh and undetectable! Built for web scraping, it helps you manage user agents with ease and style.
+
+---
 
 ## 🚀 Quick Start
 
@@ -15,41 +17,46 @@ ua = agent.random()
 print(ua)  # Mozilla/5.0 (Windows NT 11.0) AppleWebKit/537.36 ...
 ```
 
+---
+
 ## 🎯 Features
 
 ### Browser-Specific Agents
 
 ```python
 # Get agents for specific browsers
-chrome_ua = agent.chrome()    # Latest Chrome agent
-firefox_ua = agent.firefox()  # Latest Firefox agent
-safari_ua = agent.safari()    # Latest Safari agent
-edge_ua = agent.edge()       # Latest Edge agent
-opera_ua = agent.opera()     # Latest Opera agent
+chrome_ua = agent.chrome()      # Latest Chrome agent
+firefox_ua = agent.firefox()    # Latest Firefox agent
+safari_ua = agent.safari()      # Latest Safari agent
+edge_ua = agent.edge()          # Latest Edge agent
+opera_ua = agent.opera()        # Latest Opera agent
 ```
 
 ### Device-Specific Agents
 
 ```python
 # Get mobile or desktop agents
-mobile_ua = agent.mobile()    # Mobile device agent
-desktop_ua = agent.desktop()  # Desktop device agent
+mobile_ua = agent.mobile()      # Mobile device agent
+desktop_ua = agent.desktop()    # Desktop device agent
 
 # New - Get agents for specific device types
-tablet_ua = agent.tablet()    # Tablet device agent
-tv_ua = agent.smart_tv()      # Smart TV agent
-console_ua = agent.gaming()   # Gaming console agent
+tablet_ua = agent.tablet()      # Tablet device agent
+tv_ua = agent.smart_tv()        # Smart TV agent
+console_ua = agent.gaming()     # Gaming console agent
+
+# Wearable device user agent support
+wearable_ua = agent.wearable()  # Get a wearable device user agent
 ```
 
 ### OS-Specific Agents
 
 ```python
 # New - Get agents for specific operating systems
-windows_ua = agent.windows()  # Windows agent
-mac_ua = agent.macos()        # macOS agent
-linux_ua = agent.linux()      # Linux agent
-android_ua = agent.android()  # Android agent
-ios_ua = agent.ios()          # iOS agent
+windows_ua = agent.windows()    # Windows agent
+mac_ua = agent.macos()          # macOS agent
+linux_ua = agent.linux()        # Linux agent`
+android_ua = agent.android()    # Android agent
+ios_ua = agent.ios()            # iOS agent
 ```
 
 ### Custom Agent Generation
@@ -83,6 +90,15 @@ ip = agent.rotate_ip()
 print(ip)  # 192.168.1.10 (example)
 ```
 
+### Proxy Pool & Rotation
+
+```python
+agent.set_proxy_pool(["http://proxy1:8080", "http://proxy2:8080"])
+proxy = agent.rotate_proxy()  # Get next proxy from pool
+```
+
+---
+
 ## 💫 Real-World Examples
 
 ### With Requests
@@ -152,47 +168,30 @@ driver = create_driver()
 driver.get('https://example.com')
 ```
 
-### New - With Playwright
-
-```python
-from playwright.sync_api import sync_playwright
-from webscout import LitAgent
-
-agent = LitAgent()
-
-def browse_with_playwright():
-    with sync_playwright() as p:
-        browser_options = {
-            "user_agent": agent.chrome(),
-            "viewport": {"width": 1280, "height": 720}
-        }
-        browser = p.chromium.launch()
-        context = browser.new_context(**browser_options)
-        page = context.new_page()
-        page.goto('https://example.com')
-        # Continue with your scraping logic
-        browser.close()
-```
+---
 
 ## 🌟 Pro Tips
 
-1. **Rotate Agents**: Refresh your agents pool periodically to avoid detection
+1. **Rotate Agents Regularly**  
+   Refresh your agent pool periodically to avoid detection.
+
    ```python
    agent = LitAgent()
-   for _ in range(10):
+   for i in range(10):
        response = requests.get(url, headers={'User-Agent': agent.random()})
-       if _ % 3 == 0:  # Refresh every 3 requests
+       if i % 3 == 0:
            agent.refresh()
    ```
 
-2. **Device-Specific Scraping**: Use appropriate agents for different platforms
+2. **Device-Specific Scraping**  
+   Use device-specific agents for different platforms.
+
    ```python
    # Mobile site scraping
    mobile_response = requests.get(
        'https://m.example.com',
        headers={'User-Agent': agent.mobile()}
    )
-
    # Desktop site scraping
    desktop_response = requests.get(
        'https://example.com',
@@ -200,7 +199,9 @@ def browse_with_playwright():
    )
    ```
 
-3. **Browser Consistency**: Stick to one browser type per session
+3. **Browser Consistency**  
+   Stick to one browser type per session for realism.
+
    ```python
    chrome_agent = agent.chrome()
    headers = {
@@ -210,11 +211,11 @@ def browse_with_playwright():
    # Use these headers for all requests in this session
    ```
 
-4. **New - Browser Fingerprinting & IP Rotation**:
+4. **Browser Fingerprinting & IP Rotation**  
+   Generate consistent browser fingerprints and rotate IPs.
+
    ```python
-   # Create consistent browser fingerprinting
    fingerprint = agent.generate_fingerprint(browser="chrome")
-   
    headers = {
        'User-Agent': fingerprint['user_agent'],
        'Accept-Language': fingerprint['accept_language'],
@@ -223,33 +224,37 @@ def browse_with_playwright():
        'Sec-Ch-Ua-Platform': fingerprint['platform'],
        'X-Forwarded-For': fingerprint['x-forwarded-for']
    }
-   
-   # Use this consistent set for all session requests while rotating IPs
    ```
 
-5. **New - Multi-threading Support**:
+5. **Multi-threading Support**  
+   Use thread-safe mode for concurrent requests.
+
    ```python
    import concurrent.futures
    from webscout import LitAgent
-   
-   agent = LitAgent(thread_safe=True)  # Thread-safe instance
-   
+   import requests
+
+   agent = LitAgent(thread_safe=True)
+
    def fetch_url(url):
        headers = {'User-Agent': agent.random()}
        return requests.get(url, headers=headers).text
-   
+
    urls = ['https://example1.com', 'https://example2.com', 'https://example3.com']
-   
    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
        results = list(executor.map(fetch_url, urls))
    ```
 
+---
+
 ## 🔧 Supported Browsers & Devices
 
 - **Browsers**: Chrome, Firefox, Safari, Edge, Opera, Brave, Vivaldi
 - **Operating Systems**: Windows, macOS, Linux, Android, iOS, Chrome OS
 - **Devices**: Mobile phones, Tablets, Desktops, Game consoles, Smart TVs, Wearables
 
+---
+
 ## 🎨 Why LitAgent?
 
 - 🚀 Modern and up-to-date user agents
@@ -263,7 +268,11 @@ def browse_with_playwright():
 - 🧵 Thread-safe operation
 - 🕰️ Automatic refresh scheduling
 
-## 📊 New - Analytics and Reporting
+---
+
+## 📊 Analytics and Reporting
+
+Get statistics on your agent usage and export your data.
 
 ```python
 # Get statistics on your agent usage
@@ -276,10 +285,14 @@ print(f"Detection avoidance rate: {stats.avoidance_rate}%")
 agent.export_stats('agent_usage.json')
 ```
 
+---
+
 ## 📋 Installation
 
 ```bash
 pip install webscout
 ```
 
+---
+
 Made with 💖 by the HelpingAI team

webscout/litagent/agent.py

@@ -93,15 +93,23 @@ class LitAgent:
                 self._stats["device_usage"][device_type] = self._stats["device_usage"].get(device_type, 0) + 1
 
     def random(self) -> str:
-        """Get a random user agent! 🎲"""
+        """Get a random user agent! 🎲 (with blacklist/whitelist support)"""
+        if hasattr(self, '_whitelist') and self._whitelist:
+            pool = list(self._whitelist)
+        else:
+            pool = [a for a in self.agents if not hasattr(self, '_blacklist') or a not in self._blacklist]
+        if not pool:
+            pool = self.agents
         if self.thread_safe and self.lock:
             with self.lock:
-                agent = random.choice(self.agents)
+                agent = random.choice(pool)
                 self._update_stats()
+                self._add_to_history(agent)
                 return agent
         else:
-            agent = random.choice(self.agents)
+            agent = random.choice(pool)
             self._update_stats()
+            self._add_to_history(agent)
             return agent
 
     def browser(self, name: str) -> str:
@@ -330,7 +338,8 @@ class LitAgent:
         self._update_stats(browser_type=browser, device_type=device_type)
         return agent
 
-    def generate_fingerprint(self, browser: Optional[str] = None) -> Dict[str, str]:
+    @staticmethod
+    def generate_fingerprint(browser: Optional[str] = None) -> Dict[str, str]:
         """
         Generate a consistent browser fingerprint for anti-fingerprinting purposes.
 
@@ -347,13 +356,14 @@ class LitAgent:
             Dict[str, str]: A dictionary containing fingerprinting headers and values.
         """
         # Get a random user agent using the random() method
-        user_agent = self.random()
+        agent = LitAgent()
+        user_agent = agent.random()
 
         # If browser is specified, try to get a matching one
         if browser:
             browser = browser.lower()
             if browser in BROWSERS:
-                user_agent = self.browser(browser)
+                user_agent = agent.browser(browser)
 
         accept_language = random.choice(FINGERPRINTS["accept_language"])
         accept = random.choice(FINGERPRINTS["accept"])
@@ -367,7 +377,7 @@ class LitAgent:
                 sec_ch_ua = FINGERPRINTS["sec_ch_ua"][browser_name].format(version, version)
                 break
 
-        ip = self.rotate_ip()
+        ip = agent.rotate_ip()
         fingerprint = {
             "user_agent": user_agent,
             "accept_language": accept_language,
@@ -379,10 +389,10 @@ class LitAgent:
             "x-client-ip": ip,
             "forwarded": f"for={ip};proto=https",
             "x-forwarded-proto": "https",
-            "x-request-id": self.random_id(8) if hasattr(self, 'random_id') else ''.join(random.choices('0123456789abcdef', k=8)),
+            "x-request-id": agent.random_id(8) if hasattr(agent, 'random_id') else ''.join(random.choices('0123456789abcdef', k=8)),
         }
 
-        self._update_stats(browser_type=browser)
+        agent._update_stats(browser_type=browser)
         return fingerprint
 
     def refresh(self) -> None:
@@ -472,6 +482,63 @@ class LitAgent:
         """Generate a random identifier string."""
         return ''.join(random.choices('0123456789abcdef', k=length)).lower()
 
+    def wearable(self) -> str:
+        """Get a wearable device agent! ⌚"""
+        wearable_type = random.choice(DEVICES['wearable'])
+        # Example user agent for wearables (simplified)
+        if 'Apple Watch' in wearable_type:
+            agent = f"Mozilla/5.0 (AppleWatch; CPU WatchOS like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/9.0 Mobile/13S344 Safari/602.1"
+        elif 'Samsung' in wearable_type:
+            agent = f"Mozilla/5.0 (Linux; Tizen 3.0; {wearable_type}) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/1.0"
+        elif 'Fitbit' in wearable_type:
+            agent = f"Mozilla/5.0 (Linux; {wearable_type}) AppleWebKit/537.36 (KHTML, like Gecko)"
+        elif 'Garmin' in wearable_type:
+            agent = f"Mozilla/5.0 (Linux; {wearable_type}) AppleWebKit/537.36 (KHTML, like Gecko)"
+        else:
+            agent = self.random()
+        self._update_stats(device_type="wearable")
+        return agent
+
+    def set_proxy_pool(self, proxies: List[str]):
+        """Set a pool of proxies for rotation."""
+        self._proxy_pool = proxies
+        self._proxy_index = 0
+
+    def rotate_proxy(self) -> Optional[str]:
+        """Rotate through the proxy pool and return the next proxy."""
+        if not hasattr(self, '_proxy_pool') or not self._proxy_pool:
+            return None
+        proxy = self._proxy_pool[self._proxy_index]
+        self._proxy_index = (self._proxy_index + 1) % len(self._proxy_pool)
+        return proxy
+
+    def add_to_blacklist(self, agent: str):
+        """Add a user agent to the blacklist."""
+        if not hasattr(self, '_blacklist'):
+            self._blacklist = set()
+        self._blacklist.add(agent)
+
+    def add_to_whitelist(self, agent: str):
+        """Add a user agent to the whitelist."""
+        if not hasattr(self, '_whitelist'):
+            self._whitelist = set()
+        self._whitelist.add(agent)
+
+    def _add_to_history(self, agent: str):
+        if not hasattr(self, '_history'):
+            self._history = []
+        self._history.append(agent)
+        if len(self._history) > 50:
+            self._history.pop(0)
+
+    def get_history(self) -> List[str]:
+        """Get the last 50 user agents served."""
+        return getattr(self, '_history', [])
+
+    def validate_agent(self, agent: str) -> bool:
+        """Validate if a user agent string is realistic (basic check)."""
+        return agent.startswith("Mozilla/5.0") and any(b in agent for b in BROWSERS.keys())
+
 if __name__ == "__main__":
     # Test it out! 🧪
     agent = LitAgent()
@@ -484,9 +551,32 @@ if __name__ == "__main__":
     print("Tablet:", agent.tablet())
     print("Smart TV:", agent.smart_tv())
     print("Gaming:", agent.gaming())
+    print("Wearable:", agent.wearable())
 
     # Test custom agent
     print("Custom:", agent.custom(browser="chrome", os="windows", os_version="10.0"))
 
     # Test fingerprinting
     print("Fingerprint:", agent.generate_fingerprint("chrome"))
+
+    # Test proxy rotation
+    agent.set_proxy_pool(["http://proxy1.com", "http://proxy2.com"])
+    print("Proxy 1:", agent.rotate_proxy())
+    print("Proxy 2:", agent.rotate_proxy())
+
+    # Test blacklist/whitelist
+    agent.add_to_blacklist("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36")
+    agent.add_to_whitelist("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
+    print("Blacklisted:", agent.random())
+    print("Whitelisted:", agent.random())
+
+    # Test agent history
+    for _ in range(55):
+        agent.random()
+    print("History:", agent.get_history())
+
+    # Test agent validation
+    print("Valid agent:", agent.validate_agent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"))
+    print("Invalid agent:", agent.validate_agent("InvalidUserAgentString"))
+    ip = agent.rotate_ip()
+    print(ip)  # 192.168.1.10 (example)

webscout/scout/core/scout.py

@@ -265,7 +265,7 @@ class Scout:
 
         return json.dumps(_tag_to_dict(self._soup), indent=indent)
 
-    def find(self, name=None, attrs={}, recursive=True, text=None, **kwargs) -> ScoutSearchResult:
+    def find(self, name=None, attrs={}, recursive=True, text=None, class_=None, **kwargs) -> ScoutSearchResult:
         """
         Find the first matching element.
 
@@ -278,10 +278,10 @@ class Scout:
         Returns:
             ScoutSearchResult: First matching element
         """
-        result = self._soup.find(name, attrs, recursive, text, **kwargs)
+        result = self._soup.find(name, attrs, recursive, text, limit=1, class_=class_, **kwargs)
         return ScoutSearchResult([result]) if result else ScoutSearchResult([])
 
-    def find_all(self, name=None, attrs={}, recursive=True, text=None, limit=None, **kwargs) -> ScoutSearchResult:
+    def find_all(self, name=None, attrs={}, recursive=True, text=None, limit=None, class_=None, **kwargs) -> ScoutSearchResult:
         """
         Find all matching elements.
 
@@ -295,7 +295,7 @@ class Scout:
         Returns:
             ScoutSearchResult: List of matching elements
         """
-        results = self._soup.find_all(name, attrs, recursive, text, limit, **kwargs)
+        results = self._soup.find_all(name, attrs, recursive, text, limit, class_=class_, **kwargs)
         return ScoutSearchResult(results)
 
     def find_parent(self, name=None, attrs={}, **kwargs) -> Optional[Tag]:
@@ -474,6 +474,19 @@ class Scout:
         sentences = tokenizer.tokenize(text)
         return "\n\n".join(sentences)
 
+    def get_text_robust(self, separator=' ', strip=False, types=None, encoding_fallbacks=None) -> str:
+        """Extract text robustly, trying multiple encodings if needed."""
+        try:
+            return self.get_text(separator, strip, types)
+        except UnicodeDecodeError:
+            if encoding_fallbacks:
+                for enc in encoding_fallbacks:
+                    try:
+                        return self._soup.get_text(separator, strip, types).encode(enc).decode(enc)
+                    except Exception:
+                        continue
+            raise
+
     def remove_tags(self, tags: List[str]) -> None:
         """
         Remove specified tags and their contents from the document.
@@ -543,29 +556,19 @@ class Scout:
         """
         old_tag.replace_with(new_tag)
 
-    def encode(self, encoding='utf-8') -> bytes:
-        """
-        Encode the document to a specific encoding.
-
-        Args:
-            encoding (str, optional): Encoding to use
-
-        Returns:
-            bytes: Encoded document
-        """
-        return str(self._soup).encode(encoding)
-
-    def decode(self, encoding='utf-8') -> str:
-        """
-        Decode the document from a specific encoding.
-
-        Args:
-            encoding (str, optional): Encoding to use
+    def encode(self, encoding='utf-8', errors='strict') -> bytes:
+        """Encode the document to a specific encoding with error handling."""
+        try:
+            return str(self._soup).encode(encoding, errors)
+        except Exception:
+            return str(self._soup).encode('utf-8', errors)
 
-        Returns:
-            str: Decoded document
-        """
-        return str(self._soup)
+    def decode(self, encoding='utf-8', errors='strict') -> str:
+        """Decode the document from a specific encoding with error handling."""
+        try:
+            return str(self._soup).decode(encoding, errors)
+        except Exception:
+            return str(self._soup)
 
     def __str__(self) -> str:
         """
@@ -605,3 +608,78 @@ class Scout:
         decoded_markup = re.sub(r'\s+', ' ', decoded_markup)
 
         return decoded_markup
+
+    def wrap(self, wrapper_tag: Tag) -> Tag:
+        """Wrap the root tag in another tag with error handling."""
+        try:
+            return self._soup.wrap(wrapper_tag)
+        except Exception:
+            return wrapper_tag
+
+    def unwrap(self) -> None:
+        """Unwrap the root tag, keeping its contents in the parent, with error handling."""
+        try:
+            self._soup.unwrap()
+        except Exception:
+            pass
+
+    def insert_before(self, new_element: Tag) -> None:
+        """Insert a tag or string immediately before the root tag with error handling."""
+        try:
+            self._soup.insert_before(new_element)
+        except Exception:
+            pass
+
+    def insert_after(self, new_element: Tag) -> None:
+        """Insert a tag or string immediately after the root tag with error handling."""
+        try:
+            self._soup.insert_after(new_element)
+        except Exception:
+            pass
+
+    def append(self, tag: Tag) -> None:
+        """Append a tag to the root tag with error handling."""
+        try:
+            self._soup.append(tag)
+        except Exception:
+            pass
+
+    @property
+    def descendants(self):
+        """Yield all descendants of the root tag in document order."""
+        return self._soup.descendants
+
+    @property
+    def parents(self):
+        """Yield all parents of the root tag up the tree."""
+        return self._soup.parents
+
+    @property
+    def next_element(self):
+        """Return the next element in document order after the root tag."""
+        return self._soup.next_element
+
+    @property
+    def previous_element(self):
+        """Return the previous element in document order before the root tag."""
+        return self._soup.previous_element
+
+    def fetch_and_parse(self, url: str, requests_session=None, **kwargs) -> 'Scout':
+        """Fetch HTML from a URL using requests and parse it with Scout."""
+        import requests
+        session = requests_session or requests.Session()
+        resp = session.get(url, **kwargs)
+        return Scout(resp.content, features=self.features)
+
+    def tables_to_dataframe(self, table_index=0, pandas_module=None):
+        """Convert the nth table in the document to a pandas DataFrame."""
+        import pandas as pd
+        if pandas_module:
+            pd = pandas_module
+        tables = self.find_all('table')
+        if not tables or table_index >= len(tables):
+            return None
+        table = tables[table_index]
+        rows = table.find_all('tr')
+        data = [[cell.get_text(strip=True) for cell in row.find_all(['td', 'th'])] for row in rows]
+        return pd.DataFrame(data)