webscout 8.2.7__py3-none-any.whl → 8.2.9__py3-none-any.whl

webscout/Provider/AISEARCH/Perplexity.py

@@ -0,0 +1,333 @@
+import json
+import random
+from uuid import uuid4
+from typing import Dict, Optional, Generator, Union, Any
+from curl_cffi import requests
+
+from webscout.AIbase import AISearch, SearchResponse
+from webscout import exceptions
+from webscout.litagent import LitAgent
+
+
+class Perplexity(AISearch):
+    """A class to interact with the Perplexity AI search API.
+    
+    Perplexity provides a powerful search interface that returns AI-generated responses
+    based on web content. It supports both streaming and non-streaming responses,
+    multiple search modes, and model selection.
+    
+    Basic Usage:
+        >>> from webscout import Perplexity
+        >>> ai = Perplexity()
+        >>> # Non-streaming example
+        >>> response = ai.search("What is Python?")
+        >>> print(response)
+        Python is a high-level programming language...
+        
+        >>> # Streaming example
+        >>> for chunk in ai.search("Tell me about AI", stream=True):
+        ...     print(chunk, end="", flush=True)
+        Artificial Intelligence is...
+        
+        >>> # Pro search with specific model (requires authentication via cookies)
+        >>> cookies = {"perplexity-user": "your_cookie_value"}
+        >>> ai_pro = Perplexity(cookies=cookies)
+        >>> response = ai_pro.search("Latest AI research", mode="pro", model="gpt-4o")
+        >>> print(response)
+        
+        >>> # Raw response format
+        >>> for chunk in ai.search("Hello", stream=True, raw=True):
+        ...     print(chunk)
+        {'text': 'Hello'}
+        {'text': ' there!'}
+    
+    Args:
+        cookies (dict, optional): Cookies to use for authentication. Defaults to None.
+        timeout (int, optional): Request timeout in seconds. Defaults to 60.
+        proxies (dict, optional): Proxy configuration for requests. Defaults to None.
+    """
+    
+    def __init__(
+        self,
+        cookies: Optional[Dict[str, str]] = None,
+        timeout: int = 60,
+        proxies: Optional[Dict[str, str]] = None
+    ):
+        """
+        Initialize the Perplexity client.
+        
+        Args:
+            cookies (dict, optional): Cookies to use for authentication. Defaults to None.
+            timeout (int, optional): Request timeout in seconds. Defaults to 60.
+            proxies (dict, optional): Proxy configuration for requests. Defaults to None.
+        """
+        self.timeout = timeout
+        self.agent = LitAgent()
+        self.session = requests.Session(headers={
+            'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+            'accept-language': 'en-US,en;q=0.9',
+            'cache-control': 'max-age=0',
+            'dnt': '1',
+            'priority': 'u=0, i',
+            'sec-ch-ua': '"Not;A=Brand";v="24", "Chromium";v="128"',
+            'sec-ch-ua-arch': '"x86"',
+            'sec-ch-ua-bitness': '"64"',
+            'sec-ch-ua-full-version': '"128.0.6613.120"',
+            'sec-ch-ua-full-version-list': '"Not;A=Brand";v="24.0.0.0", "Chromium";v="128.0.6613.120"',
+            'sec-ch-ua-mobile': '?0',
+            'sec-ch-ua-model': '""',
+            'sec-ch-ua-platform': '"Windows"',
+            'sec-ch-ua-platform-version': '"19.0.0"',
+            'sec-fetch-dest': 'document',
+            'sec-fetch-mode': 'navigate',
+            'sec-fetch-site': 'same-origin',
+            'sec-fetch-user': '?1',
+            'upgrade-insecure-requests': '1',
+            'user-agent': self.agent.random(),
+        }, cookies=cookies or {}, impersonate='chrome')
+        
+        # Apply proxies if provided
+        if proxies:
+            self.session.proxies.update(proxies)
+        
+        # Initialize session with socket.io
+        self.timestamp = format(random.getrandbits(32), '08x')
+        
+        # Get socket.io session ID
+        response = self.session.get(f'https://www.perplexity.ai/socket.io/?EIO=4&transport=polling&t={self.timestamp}')
+        self.sid = json.loads(response.text[1:])['sid']
+        
+        # Initialize socket.io connection
+        assert (self.session.post(
+            f'https://www.perplexity.ai/socket.io/?EIO=4&transport=polling&t={self.timestamp}&sid={self.sid}',
+            data='40{"jwt":"anonymous-ask-user"}'
+        )).text == 'OK'
+        
+        # Get session info
+        self.session.get('https://www.perplexity.ai/api/auth/session')
+        
+        # Set default values
+        self.copilot = 0 if not cookies else float('inf')
+        self.file_upload = 0 if not cookies else float('inf')
+    
+    def _extract_answer(self, response):
+        """
+        Extract the answer from the response.
+        
+        Args:
+            response (dict): The response from Perplexity AI.
+            
+        Returns:
+            str: The extracted answer text.
+        """
+        if not response:
+            return ""
+        
+        # Find the FINAL step in the text array
+        final_step = None
+        if 'text' in response and isinstance(response['text'], list):
+            for step in response['text']:
+                if step.get('step_type') == 'FINAL' and 'content' in step and 'answer' in step['content']:
+                    final_step = step
+                    break
+        
+        if not final_step:
+            return ""
+        
+        try:
+            # Parse the answer JSON string
+            answer_json = json.loads(final_step['content']['answer'])
+            return answer_json.get('answer', '')
+        except (json.JSONDecodeError, KeyError):
+            return ""
+    
+    def search(
+        self,
+        prompt: str,
+        mode: str = 'auto',
+        model: Optional[str] = None,
+        sources: Optional[list] = None,
+        stream: bool = False,
+        raw: bool = False,
+        language: str = 'en-US',
+        follow_up: Optional[Dict[str, Any]] = None,
+        incognito: bool = False
+    ) -> Union[SearchResponse, Generator[Union[Dict[str, str], SearchResponse], None, None]]:
+        """Search using the Perplexity API and get AI-generated responses.
+        
+        This method sends a search query to Perplexity and returns the AI-generated response.
+        It supports both streaming and non-streaming modes, as well as raw response format.
+        
+        Args:
+            prompt (str): The search query or prompt to send to the API.
+            mode (str, optional): Search mode. Options: 'auto', 'pro', 'reasoning', 'deep research'. 
+                                Defaults to 'auto'.
+            model (str, optional): Model to use. Available models depend on the mode. Defaults to None.
+            sources (list, optional): Sources to use. Options: 'web', 'scholar', 'social'. 
+                                    Defaults to ['web'].
+            stream (bool, optional): If True, yields response chunks as they arrive.
+                                   If False, returns complete response. Defaults to False.
+            raw (bool, optional): If True, returns raw response dictionaries.
+                                If False, returns Response objects that convert to text automatically.
+                                Defaults to False.
+            language (str, optional): Language to use. Defaults to 'en-US'.
+            follow_up (dict, optional): Follow-up information. Defaults to None.
+            incognito (bool, optional): Whether to use incognito mode. Defaults to False.
+            
+        Returns:
+            If stream=True: Generator yielding response chunks as they arrive
+            If stream=False: Complete response
+            
+        Raises:
+            ValueError: If invalid mode or model is provided
+            exceptions.APIConnectionError: If connection to API fails
+            exceptions.FailedToGenerateResponseError: If response generation fails
+        """
+        if sources is None:
+            sources = ['web']
+        
+        # Validate inputs
+        if mode not in ['auto', 'pro', 'reasoning', 'deep research']:
+            raise ValueError('Search modes -> ["auto", "pro", "reasoning", "deep research"]')
+        
+        if not all([source in ('web', 'scholar', 'social') for source in sources]):
+            raise ValueError('Sources -> ["web", "scholar", "social"]')
+        
+        # Check if model is valid for the selected mode
+        valid_models = {
+            'auto': [None],
+            'pro': [None, 'sonar', 'gpt-4.5', 'gpt-4o', 'claude 3.7 sonnet', 'gemini 2.0 flash', 'grok-2'],
+            'reasoning': [None, 'r1', 'o3-mini', 'claude 3.7 sonnet'],
+            'deep research': [None]
+        }
+        
+        if mode in valid_models and model not in valid_models[mode] and model is not None:
+            raise ValueError(f'Invalid model for {mode} mode. Valid models: {valid_models[mode]}')
+        
+        # Prepare request data
+        json_data = {
+            'query_str': prompt,
+            'params': {
+                'attachments': follow_up['attachments'] if follow_up else [],
+                'frontend_context_uuid': str(uuid4()),
+                'frontend_uuid': str(uuid4()),
+                'is_incognito': incognito,
+                'language': language,
+                'last_backend_uuid': follow_up['backend_uuid'] if follow_up else None,
+                'mode': 'concise' if mode == 'auto' else 'copilot',
+                'model_preference': {
+                    'auto': {
+                        None: 'turbo'
+                    },
+                    'pro': {
+                        None: 'pplx_pro',
+                        'sonar': 'experimental',
+                        'gpt-4.5': 'gpt45',
+                        'gpt-4o': 'gpt4o',
+                        'claude 3.7 sonnet': 'claude2',
+                        'gemini 2.0 flash': 'gemini2flash',
+                        'grok-2': 'grok'
+                    },
+                    'reasoning': {
+                        None: 'pplx_reasoning',
+                        'r1': 'r1',
+                        'o3-mini': 'o3mini',
+                        'claude 3.7 sonnet': 'claude37sonnetthinking'
+                    },
+                    'deep research': {
+                        None: 'pplx_alpha'
+                    }
+                }[mode][model],
+                'source': 'default',
+                'sources': sources,
+                'version': '2.18'
+            }
+        }
+        
+        try:
+            # Make the request
+            resp = self.session.post(
+                'https://www.perplexity.ai/rest/sse/perplexity_ask', 
+                json=json_data, 
+                stream=True,
+                timeout=self.timeout
+            )
+            
+            if resp.status_code != 200:
+                raise exceptions.APIConnectionError(f"API returned status code {resp.status_code}")
+            
+            # Define streaming response handler
+            def stream_response():
+                for chunk in resp.iter_lines(delimiter=b'\r\n\r\n'):
+                    content = chunk.decode('utf-8')
+                    if content.startswith('event: message\r\n'):
+                        content_json = json.loads(content[len('event: message\r\ndata: '):])
+                        if 'text' in content_json:
+                            try:
+                                # If text is a string, try to parse it as JSON
+                                if isinstance(content_json['text'], str):
+                                    content_json['text'] = json.loads(content_json['text'])
+                            except json.JSONDecodeError:
+                                pass
+                        
+                        if raw:
+                            yield content_json
+                        else:
+                            # For non-raw responses, extract text from each chunk
+                            if 'text' in content_json and isinstance(content_json['text'], list):
+                                for step in content_json['text']:
+                                    if step.get('type') == 'answer' and 'value' in step:
+                                        yield SearchResponse(step['value'])
+                                    elif step.get('type') == 'thinking' and 'value' in step:
+                                        yield SearchResponse(step['value'])
+                    elif content.startswith('event: end_of_stream\r\n'):
+                        return
+            
+            # Handle streaming or non-streaming response
+            if stream:
+                return stream_response()
+            else:
+                chunks = []
+                final_response = None
+                
+                for chunk in resp.iter_lines(delimiter=b'\r\n\r\n'):
+                    content = chunk.decode('utf-8')
+                    if content.startswith('event: message\r\n'):
+                        content_json = json.loads(content[len('event: message\r\ndata: '):])
+                        if 'text' in content_json:
+                            try:
+                                # If text is a string, try to parse it as JSON
+                                if isinstance(content_json['text'], str):
+                                    content_json['text'] = json.loads(content_json['text'])
+                            except json.JSONDecodeError:
+                                pass
+                        chunks.append(content_json)
+                        final_response = content_json
+                    elif content.startswith('event: end_of_stream\r\n'):
+                        # Process the final response to extract the answer
+                        if final_response:
+                            answer_text = self._extract_answer(final_response)
+                            return SearchResponse(answer_text) if not raw else final_response
+                        elif chunks:
+                            answer_text = self._extract_answer(chunks[-1])
+                            return SearchResponse(answer_text) if not raw else chunks[-1]
+                        else:
+                            return SearchResponse("") if not raw else {}
+                
+                # If we get here, something went wrong
+                raise exceptions.FailedToGenerateResponseError("Failed to get complete response")
+                
+        except requests.RequestsError as e:
+            raise exceptions.APIConnectionError(f"Connection error: {str(e)}")
+        except json.JSONDecodeError:
+            raise exceptions.FailedToGenerateResponseError("Failed to parse response JSON")
+        except Exception as e:
+            raise exceptions.FailedToGenerateResponseError(f"Error: {str(e)}")
+
+
+if __name__ == "__main__":
+    # Simple test
+    ai = Perplexity()
+    response = ai.search("What is Python?")
+    print(response)
+    

webscout/Provider/AISEARCH/README.md

@@ -0,0 +1,279 @@
+<div align="center">
+  <h1>🔍 Webscout AI Search Providers</h1>
+  <p><strong>Powerful AI-powered search capabilities with multiple provider support</strong></p>
+</div>
+
+> [!NOTE]
+> AI Search Providers leverage advanced language models and search algorithms to deliver high-quality, context-aware responses with web search integration.
+
+## ✨ Features
+
+- **Multiple Search Providers**: Support for 7+ specialized AI search services
+- **Streaming Responses**: Real-time streaming of AI-generated responses
+- **Raw Response Format**: Access to raw response data when needed
+- **Automatic Text Handling**: Smart response formatting and cleaning
+- **Robust Error Handling**: Comprehensive error management
+- **Cross-Platform Compatibility**: Works seamlessly across different environments
+
+## 📦 Supported Search Providers
+
+| Provider | Description | Key Features |
+|----------|-------------|-------------|
+| **DeepFind** | General purpose AI search | Web-based, reference removal, clean formatting |
+| **Felo** | Fast streaming search | Advanced capabilities, real-time streaming |
+| **Isou** | Scientific search | Multiple model selection, citation handling |
+| **Genspark** | Efficient search | Fast response, markdown link removal |
+| **Monica** | Comprehensive search | Related question suggestions, source references |
+| **WebPilotAI** | Web-integrated search | Web page analysis, content extraction |
+| **Scira** | Research-focused search | Multiple models (Grok3, Claude), vision support |
+| **IAsk** | Multi-mode search | Question, Academic, Fast modes, detail levels |
+| **Hika** | General AI search | Simple interface, clean text output |
+| **Perplexity** | Advanced AI search & chat | Multiple modes (Pro, Reasoning), model selection, source control |
+
+## 🚀 Installation
+
+```bash
+pip install -U webscout
+```
+
+## 💻 Quick Start Guide
+
+### Basic Usage Pattern
+
+All AI Search providers follow a consistent usage pattern:
+
+```python
+from webscout import ProviderName
+
+# Initialize the provider
+ai = ProviderName()
+
+# Basic search
+response = ai.search("Your query here")
+print(response)  # Automatically formats the response
+
+# Streaming search
+for chunk in ai.search("Your query here", stream=True):
+    print(chunk, end="", flush=True)  # Print response as it arrives
+```
+
+### Provider Examples
+
+<details>
+<summary><strong>DeepFind Example</strong></summary>
+
+```python
+from webscout import DeepFind
+
+# Initialize the search provider
+ai = DeepFind()
+
+# Basic search
+response = ai.search("What is Python?")
+print(response)
+
+# Streaming search
+for chunk in ai.search("Tell me about AI", stream=True):
+    print(chunk, end="")
+```
+</details>
+
+<details>
+<summary><strong>Scira Example</strong></summary>
+
+```python
+from webscout import Scira
+
+# Initialize with default model (Grok3)
+ai = Scira()
+
+# Basic search
+response = ai.search("What is the impact of climate change?")
+print(response)
+
+# Streaming search with Claude model
+ai = Scira(model="scira-claude")
+for chunk in ai.search("Explain quantum computing", stream=True):
+    print(chunk, end="", flush=True)
+
+# Available models:
+# - scira-default (Grok3)
+# - scira-grok-3-mini (Grok3-mini)
+# - scira-vision (Grok2-Vision)
+# - scira-claude (Sonnet-3.7)
+# - scira-optimus (optimus)
+```
+</details>
+
+<details>
+<summary><strong>Isou Example</strong></summary>
+
+```python
+from webscout import Isou
+
+# Initialize with specific model
+ai = Isou(model="siliconflow:deepseek-ai/DeepSeek-R1-Distill-Qwen-32B")
+
+# Get a response with scientific information
+response = ai.search("Explain the double-slit experiment")
+print(response)
+```
+</details>
+
+<details>
+<summary><strong>Perplexity Example</strong></summary>
+
+```python
+from webscout import Perplexity
+
+# Initialize (optionally pass cookies for authenticated features)
+# cookies = {"perplexity-user": "your_cookie_value"}
+# ai = Perplexity(cookies=cookies)
+ai = Perplexity() # Anonymous access
+
+# Basic search (auto mode)
+response = ai.search("What is the weather in London?")
+print(response)
+
+# Streaming search
+for chunk in ai.search("Explain black holes", stream=True):
+    print(chunk, end="", flush=True)
+
+# Pro search with specific model (requires authentication via cookies)
+# try:
+#     ai_pro = Perplexity(cookies=your_cookies)
+#     response_pro = ai_pro.search("Latest AI research papers", mode='pro', model='gpt-4o', sources=['scholar'])
+#     print(response_pro)
+# except Exception as e:
+#     print(f"Pro search failed: {e}")
+
+# Available modes: 'auto', 'pro', 'reasoning', 'deep research'
+# Available sources: 'web', 'scholar', 'social'
+# Models depend on the mode selected.
+```
+</details>
+
+## 🎛️ Advanced Configuration
+
+<details>
+<summary><strong>Timeout and Proxy Settings</strong></summary>
+
+```python
+# Configure timeout
+ai = DeepFind(timeout=60)  # 60 seconds timeout
+
+# Use with proxy
+proxies = {'http': 'http://proxy.com:8080'}
+ai = Felo(proxies=proxies)
+
+# Configure max tokens (for providers that support it)
+ai = Genspark(max_tokens=800)
+
+# Configure model and group for Scira
+ai = Scira(model="scira-claude", group="web")
+```
+</details>
+
+<details>
+<summary><strong>Response Formats</strong></summary>
+
+```python
+# Get raw response format
+response = ai.search("Hello", stream=True, raw=True)
+# Output: {'text': 'Hello'}, {'text': ' there!'}, etc.
+
+# Get formatted text response
+response = ai.search("Hello", stream=True)
+# Output: Hello there!
+```
+</details>
+
+## 🔧 Provider Capabilities
+
+| Provider | Key Capabilities | Technical Details |
+|----------|-----------------|-------------------|
+| **DeepFind** | • Web-based AI search<br>• Automatic reference removal<br>• Clean response formatting | • Streaming support with progress tracking<br>• JSON response parsing<br>• Error handling |
+| **Felo** | • Advanced search capabilities<br>• Real-time response streaming<br>• JSON-based response parsing | • Automatic text cleaning<br>• Session management<br>• Rate limiting support |
+| **Isou** | • Multiple model selection<br>• Scientific and general category support<br>• Citation handling | • Deep and simple search modes<br>• Specialized model options<br>• Markdown formatting |
+| **Genspark** | • Fast response generation<br>• Markdown link removal<br>• JSON structure normalization | • Session-based API interactions<br>• Efficient content parsing<br>• Streaming optimization |
+| **Monica** | • Comprehensive search responses<br>• Related question suggestions<br>• Source references | • Answer snippets<br>• Clean formatted responses<br>• Web content integration |
+| **WebPilotAI** | • Web page analysis<br>• Content extraction<br>• Structured data retrieval | • URL processing<br>• HTML parsing<br>• Metadata extraction |
+| **Scira** | • Research-focused search<br>• Multiple model options<br>• Vision support | • Grok3, Claude, Vision models<br>• Customizable group parameters<br>• Efficient content parsing |
+| **IAsk** | • Multi-mode search (Question, Academic, etc.)<br>• Adjustable detail level<br>• Source citation | • Asynchronous backend (sync wrapper)<br>• WebSocket communication<br>• HTML parsing & formatting |
+| **Hika** | • General AI search<br>• Simple streaming<br>• Basic text cleaning | • SSE streaming<br>• Custom headers for auth<br>• JSON response parsing |
+| **Perplexity** | • Multiple search modes (Pro, Reasoning)<br>• Model selection per mode<br>• Source filtering (web, scholar, social)<br>• Follow-up questions | • `curl_cffi` for Cloudflare bypass<br>• Socket.IO communication<br>• SSE streaming<br>• Requires cookies for Pro features |
+
+## 🛡️ Error Handling
+
+<details>
+<summary><strong>Exception Handling Example</strong></summary>
+
+```python
+from webscout import exceptions
+
+try:
+    response = ai.search("Your query")
+except exceptions.APIConnectionError as e:
+    print(f"API error: {e}")
+except Exception as e:
+    print(f"An error occurred: {e}")
+```
+</details>
+
+## 📝 Response Handling
+
+<details>
+<summary><strong>Working with Response Objects</strong></summary>
+
+```python
+# Response objects automatically convert to text
+response = ai.search("What is AI?")
+print(response)  # Prints formatted text
+
+# Access raw text if needed
+print(response.text)
+```
+</details>
+
+## 🔒 Best Practices
+
+<details>
+<summary><strong>Streaming for Long Responses</strong></summary>
+
+```python
+for chunk in ai.search("Long query", stream=True):
+    print(chunk, end="", flush=True)
+```
+</details>
+
+<details>
+<summary><strong>Error Handling</strong></summary>
+
+```python
+try:
+    response = ai.search("Query")
+except exceptions.APIConnectionError:
+    # Handle connection errors
+    pass
+```
+</details>
+
+<details>
+<summary><strong>Provider Selection Guide</strong></summary>
+
+| Use Case | Recommended Provider |
+|----------|----------------------|
+| General purpose search | **DeepFind**, **Hika** |
+| Fast streaming responses | **Felo** |
+| Scientific or specialized queries | **Isou**, **Scira** |
+| Clean and efficient responses | **Genspark** |
+| Comprehensive answers with sources | **Monica**, **IAsk** |
+| Web page interaction/analysis | **WebPilotAI** |
+| Advanced control (modes, models) | **Perplexity**, **Scira**, **Isou** |
+| Research-focused | **Scira**, **Isou**, **Perplexity** (with scholar source) |
+
+</details>
+
+## 🤝 Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.

webscout/Provider/AISEARCH/__init__.py

@@ -0,0 +1,9 @@
+from .felo_search import *
+from .DeepFind import *
+from .genspark_search import *
+from .monica_search import *
+from .webpilotai_search import *
+from .hika_search import *
+from .scira_search import *
+from .iask_search import *
+from .Perplexity import *