webscout 8.2.2__py3-none-any.whl → 8.2.7__py3-none-any.whl

webscout/Provider/AISEARCH/iask_search.py

@@ -1,436 +0,0 @@
-import aiohttp
-import asyncio
-import lxml.html
-import re
-import urllib.parse
-from markdownify import markdownify as md
-from typing import Dict, Optional, Generator, Union, AsyncIterator, Literal
-
-from webscout.AIbase import AISearch
-from webscout import exceptions
-from webscout.scout import Scout
-
-
-class Response:
-    """A wrapper class for IAsk API responses.
-
-    This class automatically converts response objects to their text representation
-    when printed or converted to string.
-
-    Attributes:
-        text (str): The text content of the response
-
-    Example:
-        >>> response = Response("Hello, world!")
-        >>> print(response)
-        Hello, world!
-        >>> str(response)
-        'Hello, world!'
-    """
-    def __init__(self, text: str):
-        self.text = text
-
-    def __str__(self):
-        return self.text
-
-    def __repr__(self):
-        return self.text
-
-
-def cache_find(diff: Union[dict, list]) -> Optional[str]:
-    """Find HTML content in a nested dictionary or list structure.
-
-    Args:
-        diff (Union[dict, list]): The nested structure to search
-
-    Returns:
-        Optional[str]: The found HTML content, or None if not found
-    """
-    values = diff if isinstance(diff, list) else diff.values()
-    for value in values:
-        if isinstance(value, (list, dict)):
-            cache = cache_find(value)
-            if cache:
-                return cache
-        if isinstance(value, str) and re.search(r"<p>.+?</p>", value):
-            return md(value).strip()
-
-    return None
-
-
-ModeType = Literal["question", "academic", "fast", "forums", "wiki", "advanced"]
-DetailLevelType = Literal["concise", "detailed", "comprehensive"]
-
-
-class IAsk(AISearch):
-    """A class to interact with the IAsk AI search API.
-
-    IAsk provides a powerful search interface that returns AI-generated responses
-    based on web content. It supports both streaming and non-streaming responses,
-    as well as different search modes and detail levels.
-
-    Basic Usage:
-        >>> from webscout import IAsk
-        >>> ai = IAsk()
-        >>> # Non-streaming example
-        >>> response = ai.search("What is Python?")
-        >>> print(response)
-        Python is a high-level programming language...
-
-        >>> # Streaming example
-        >>> for chunk in ai.search("Tell me about AI", stream=True):
-        ...     print(chunk, end="", flush=True)
-        Artificial Intelligence is...
-
-        >>> # With specific mode and detail level
-        >>> response = ai.search("Climate change", mode="academic", detail_level="detailed")
-        >>> print(response)
-        Climate change refers to...
-
-    Args:
-        timeout (int, optional): Request timeout in seconds. Defaults to 30.
-        proxies (dict, optional): Proxy configuration for requests. Defaults to None.
-        mode (ModeType, optional): Default search mode. Defaults to "question".
-        detail_level (DetailLevelType, optional): Default detail level. Defaults to None.
-    """
-
-    def __init__(
-        self,
-        timeout: int = 30,
-        proxies: Optional[dict] = None,
-        mode: ModeType = "question",
-        detail_level: Optional[DetailLevelType] = None,
-    ):
-        """Initialize the IAsk API client.
-
-        Args:
-            timeout (int, optional): Request timeout in seconds. Defaults to 30.
-            proxies (dict, optional): Proxy configuration for requests. Defaults to None.
-            mode (ModeType, optional): Default search mode. Defaults to "question".
-            detail_level (DetailLevelType, optional): Default detail level. Defaults to None.
-        """
-        self.timeout = timeout
-        self.proxies = proxies or {}
-        self.default_mode = mode
-        self.default_detail_level = detail_level
-        self.api_endpoint = "https://iask.ai/"
-        self.last_response = {}
-
-    def create_url(self, query: str, mode: ModeType = "question", detail_level: Optional[DetailLevelType] = None) -> str:
-        """Create a properly formatted URL with mode and detail level parameters.
-
-        Args:
-            query (str): The search query.
-            mode (ModeType, optional): Search mode. Defaults to "question".
-            detail_level (DetailLevelType, optional): Detail level. Defaults to None.
-
-        Returns:
-            str: Formatted URL with query parameters.
-
-        Example:
-            >>> ai = IAsk()
-            >>> url = ai.create_url("Climate change", mode="academic", detail_level="detailed")
-            >>> print(url)
-            https://iask.ai/?mode=academic&q=Climate+change&options%5Bdetail_level%5D=detailed
-        """
-        # Create a dictionary of parameters with flattened structure
-        params = {
-            "mode": mode,
-            "q": query
-        }
-
-        # Add detail_level if provided using the flattened format
-        if detail_level:
-            params["options[detail_level]"] = detail_level
-
-        # Encode the parameters and build the URL
-        query_string = urllib.parse.urlencode(params)
-        url = f"{self.api_endpoint}?{query_string}"
-
-        return url
-
-    def format_html(self, html_content: str) -> str:
-        """Format HTML content into a more readable text format.
-
-        Args:
-            html_content (str): The HTML content to format.
-
-        Returns:
-            str: Formatted text.
-        """
-        scout = Scout(html_content, features='html.parser')
-        output_lines = []
-
-        for child in scout.find_all(['h1', 'h2', 'h3', 'p', 'ol', 'ul', 'div']):
-            if child.name in ["h1", "h2", "h3"]:
-                output_lines.append(f"\n**{child.get_text().strip()}**\n")
-            elif child.name == "p":
-                text = child.get_text().strip()
-                text = re.sub(r"^According to Ask AI & Question AI www\.iAsk\.ai:\s*", "", text).strip()
-                # Remove footnote markers
-                text = re.sub(r'\[\d+\]\(#fn:\d+ \'see footnote\'\)', '', text)
-                output_lines.append(text + "\n")
-            elif child.name in ["ol", "ul"]:
-                for li in child.find_all("li"):
-                    output_lines.append("- " + li.get_text().strip() + "\n")
-            elif child.name == "div" and "footnotes" in child.get("class", []):
-                output_lines.append("\n**Authoritative Sources**\n")
-                for li in child.find_all("li"):
-                    link = li.find("a")
-                    if link:
-                        output_lines.append(f"- {link.get_text().strip()} ({link.get('href')})\n")
-
-        return "".join(output_lines)
-
-    def search(
-        self,
-        prompt: str,
-        stream: bool = False,
-        raw: bool = False,
-        mode: Optional[ModeType] = None,
-        detail_level: Optional[DetailLevelType] = None,
-    ) -> Union[Response, Generator[Union[Dict[str, str], Response], None, None]]:
-        """Search using the IAsk API and get AI-generated responses.
-
-        This method sends a search query to IAsk and returns the AI-generated response.
-        It supports both streaming and non-streaming modes, as well as raw response format.
-
-        Args:
-            prompt (str): The search query or prompt to send to the API.
-            stream (bool, optional): If True, yields response chunks as they arrive.
-                                   If False, returns complete response. Defaults to False.
-            raw (bool, optional): If True, returns raw response dictionaries with 'text' key.
-                                If False, returns Response objects that convert to text automatically.
-                                Defaults to False.
-            mode (ModeType, optional): Search mode to use. Defaults to None (uses instance default).
-            detail_level (DetailLevelType, optional): Detail level to use. Defaults to None (uses instance default).
-
-        Returns:
-            Union[Response, Generator[Union[Dict[str, str], Response], None, None]]:
-                - If stream=False: Returns complete response as Response object
-                - If stream=True: Yields response chunks as either Dict or Response objects
-
-        Raises:
-            APIConnectionError: If the API request fails
-
-        Examples:
-            Basic search:
-            >>> ai = IAsk()
-            >>> response = ai.search("What is Python?")
-            >>> print(response)
-            Python is a programming language...
-
-            Streaming response:
-            >>> for chunk in ai.search("Tell me about AI", stream=True):
-            ...     print(chunk, end="")
-            Artificial Intelligence...
-
-            Raw response format:
-            >>> for chunk in ai.search("Hello", stream=True, raw=True):
-            ...     print(chunk)
-            {'text': 'Hello'}
-            {'text': ' there!'}
-
-            With specific mode and detail level:
-            >>> response = ai.search("Climate change", mode="academic", detail_level="detailed")
-            >>> print(response)
-            Climate change refers to...
-        """
-        # Use provided parameters or fall back to instance defaults
-        search_mode = mode or self.default_mode
-        search_detail_level = detail_level or self.default_detail_level
-
-        # For non-streaming, run the async search and return the complete response
-        if not stream:
-            # Create a new event loop for this request
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
-            try:
-                result = loop.run_until_complete(
-                    self._async_search(prompt, False, raw, search_mode, search_detail_level)
-                )
-                return result
-            finally:
-                loop.close()
-
-        # For streaming, use a simpler approach with a single event loop
-        # that stays open until the generator is exhausted
-        buffer = ""
-
-        def sync_generator():
-            nonlocal buffer
-            # Create a new event loop for this generator
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
-
-            try:
-                # Get the async generator
-                async_gen_coro = self._async_search(prompt, True, raw, search_mode, search_detail_level)
-                async_gen = loop.run_until_complete(async_gen_coro)
-
-                # Process chunks one by one
-                while True:
-                    try:
-                        # Get the next chunk
-                        chunk_coro = async_gen.__anext__()
-                        chunk = loop.run_until_complete(chunk_coro)
-
-                        # Update buffer and yield the chunk
-                        if isinstance(chunk, dict) and 'text' in chunk:
-                            buffer += chunk['text']
-                        elif isinstance(chunk, Response):
-                            buffer += chunk.text
-                        else:
-                            buffer += str(chunk)
-
-                        yield chunk
-                    except StopAsyncIteration:
-                        break
-                    except Exception as e:
-                        print(f"Error in generator: {e}")
-                        break
-            finally:
-                # Store the final response and close the loop
-                self.last_response = {"text": buffer}
-                loop.close()
-
-        return sync_generator()
-
-    async def _async_search(
-        self,
-        prompt: str,
-        stream: bool = False,
-        raw: bool = False,
-        mode: ModeType = "question",
-        detail_level: Optional[DetailLevelType] = None,
-    ) -> Union[Response, AsyncIterator[Union[Dict[str, str], Response]]]:
-        """Internal async implementation of the search method."""
-
-        async def stream_generator() -> AsyncIterator[str]:
-            async with aiohttp.ClientSession() as session:
-                # Prepare parameters
-                params = {"mode": mode, "q": prompt}
-                if detail_level:
-                    params["options[detail_level]"] = detail_level
-
-                try:
-                    async with session.get(
-                        self.api_endpoint,
-                        params=params,
-                        proxy=self.proxies.get('http') if self.proxies else None,
-                        timeout=self.timeout
-                    ) as response:
-                        if not response.ok:
-                            raise exceptions.APIConnectionError(
-                                f"Failed to generate response - ({response.status_code}, {response.reason}) - {await response.text()}"
-                            )
-
-                        etree = lxml.html.fromstring(await response.text())
-                        phx_node = etree.xpath('//*[starts-with(@id, "phx-")]').pop()
-                        csrf_token = (
-                            etree.xpath('//*[@name="csrf-token"]').pop().get("content")
-                        )
-
-                    async with session.ws_connect(
-                        f"{self.api_endpoint}live/websocket",
-                        params={
-                            "_csrf_token": csrf_token,
-                            "vsn": "2.0.0",
-                        },
-                        proxy=self.proxies.get('http') if self.proxies else None,
-                        timeout=self.timeout
-                    ) as wsResponse:
-                        await wsResponse.send_json(
-                            [
-                                None,
-                                None,
-                                f"lv:{phx_node.get('id')}",
-                                "phx_join",
-                                {
-                                    "params": {"_csrf_token": csrf_token},
-                                    "url": str(response.url),
-                                    "session": phx_node.get("data-phx-session"),
-                                },
-                            ]
-                        )
-                        while True:
-                            json_data = await wsResponse.receive_json()
-                            if not json_data:
-                                break
-                            diff: dict = json_data[4]
-                            try:
-                                chunk: str = diff["e"][0][1]["data"]
-                                # Check if the chunk contains HTML content
-                                if re.search(r"<[^>]+>", chunk):
-                                    formatted_chunk = self.format_html(chunk)
-                                    yield formatted_chunk
-                                else:
-                                    yield chunk.replace("<br/>", "\n")
-                            except:
-                                cache = cache_find(diff)
-                                if cache:
-                                    if diff.get("response", None):
-                                        # Format the cache content if it contains HTML
-                                        if re.search(r"<[^>]+>", cache):
-                                            formatted_cache = self.format_html(cache)
-                                            yield formatted_cache
-                                        else:
-                                            yield cache
-                                    break
-                except Exception as e:
-                    raise exceptions.APIConnectionError(f"Error connecting to IAsk API: {str(e)}")
-
-        # For non-streaming, collect all chunks and return a single response
-        if not stream:
-            buffer = ""
-            async for chunk in stream_generator():
-                buffer += chunk
-            self.last_response = {"text": buffer}
-            return Response(buffer) if not raw else {"text": buffer}
-
-        # For streaming, create an async generator that yields chunks
-        async def process_stream():
-            buffer = ""
-            async for chunk in stream_generator():
-                buffer += chunk
-                if raw:
-                    yield {"text": chunk}
-                else:
-                    yield Response(chunk)
-            self.last_response = {"text": buffer}
-
-        # Return the async generator
-        return process_stream()
-
-
-if __name__ == "__main__":
-    from rich import print
-
-    ai = IAsk()
-
-    # Example 1: Simple search with default mode
-    print("\n[bold cyan]Example 1: Simple search with default mode[/bold cyan]")
-    response = ai.search("What is Python?", stream=True)
-    for chunk in response:
-        print(chunk, end="", flush=True)
-    print("\n\n[bold green]Response complete.[/bold green]\n")
-
-    # Example 2: Search with academic mode
-    print("\n[bold cyan]Example 2: Search with academic mode[/bold cyan]")
-    response = ai.search("Quantum computing applications", mode="academic", stream=True)
-    for chunk in response:
-        print(chunk, end="", flush=True)
-    print("\n\n[bold green]Response complete.[/bold green]\n")
-
-    # Example 3: Search with advanced mode and detailed level
-    print("\n[bold cyan]Example 3: Search with advanced mode and detailed level[/bold cyan]")
-    response = ai.search("Climate change solutions", mode="advanced", detail_level="detailed", stream=True)
-    for chunk in response:
-        print(chunk, end="", flush=True)
-    print("\n\n[bold green]Response complete.[/bold green]\n")
-
-    # Example 4: Demonstrating the create_url method
-    print("\n[bold cyan]Example 4: Generated URL for browser access[/bold cyan]")
-    url = ai.create_url("Helpingai details", mode="question", detail_level="detailed")
-    print(f"URL: {url}")
-    print("This URL can be used directly in a browser or with other HTTP clients.")

webscout/Provider/AISEARCH/monica_search.py

@@ -1,246 +0,0 @@
-import requests
-import json
-import re
-import uuid
-from typing import Dict, Optional, Generator, Union, Any
-
-from webscout.AIbase import AISearch
-from webscout import exceptions
-from webscout.litagent import LitAgent
-
-
-class Response:
-    """A wrapper class for Monica API responses.
-    
-    This class automatically converts response objects to their text representation
-    when printed or converted to string.
-    
-    Attributes:
-        text (str): The text content of the response
-        
-    Example:
-        >>> response = Response("Hello, world!")
-        >>> print(response)
-        Hello, world!
-        >>> str(response)
-        'Hello, world!'
-    """
-    def __init__(self, text: str):
-        self.text = text
-    
-    def __str__(self):
-        return self.text
-    
-    def __repr__(self):
-        return self.text
-
-
-class Monica(AISearch):
-    """A class to interact with the Monica AI search API.
-    
-    Monica provides a powerful search interface that returns AI-generated responses
-    based on web content. It supports both streaming and non-streaming responses.
-    
-    Basic Usage:
-        >>> from webscout import Monica
-        >>> ai = Monica()
-        >>> # Non-streaming example
-        >>> response = ai.search("What is Python?")
-        >>> print(response)
-        Python is a high-level programming language...
-        
-        >>> # Streaming example
-        >>> for chunk in ai.search("Tell me about AI", stream=True):
-        ...     print(chunk, end="", flush=True)
-        Artificial Intelligence is...
-        
-        >>> # Raw response format
-        >>> for chunk in ai.search("Hello", stream=True, raw=True):
-        ...     print(chunk)
-        {'text': 'Hello'}
-        {'text': ' there!'}
-    
-    Args:
-        timeout (int, optional): Request timeout in seconds. Defaults to 60.
-        proxies (dict, optional): Proxy configuration for requests. Defaults to None.
-    """
-
-    def __init__(
-        self,
-        timeout: int = 60,
-        proxies: Optional[dict] = None,
-    ):
-        """Initialize the Monica API client.
-        
-        Args:
-            timeout (int, optional): Request timeout in seconds. Defaults to 60.
-            proxies (dict, optional): Proxy configuration for requests. Defaults to None.
-        """
-        self.session = requests.Session()
-        self.api_endpoint = "https://monica.so/api/search_v1/search"
-        self.stream_chunk_size = 64
-        self.timeout = timeout
-        self.last_response = {}
-        self.client_id = str(uuid.uuid4())
-        self.session_id = ""
-        
-        self.headers = {
-            "accept": "*/*",
-            "accept-encoding": "gzip, deflate, br, zstd",
-            "accept-language": "en-US,en;q=0.9",
-            "content-type": "application/json",
-            "dnt": "1",
-            "origin": "https://monica.so",
-            "referer": "https://monica.so/answers",
-            "sec-ch-ua": '"Microsoft Edge";v="135", "Not-A.Brand";v="8", "Chromium";v="135"',
-            "sec-ch-ua-mobile": "?0",
-            "sec-ch-ua-platform": '"Windows"',
-            "sec-fetch-dest": "empty",
-            "sec-fetch-mode": "cors",
-            "sec-fetch-site": "same-origin",
-            "sec-gpc": "1",
-            "user-agent": LitAgent().random(),
-            "x-client-id": self.client_id,
-            "x-client-locale": "en",
-            "x-client-type": "web",
-            "x-client-version": "5.4.3",
-            "x-from-channel": "NA",
-            "x-product-name": "Monica-Search",
-            "x-time-zone": "Asia/Calcutta;-330"
-        }
-        
-        self.cookies = {
-            "monica_home_theme": "auto",
-        }
-        
-        self.session.headers.update(self.headers)
-        self.proxies = proxies
-
-    def search(
-        self,
-        prompt: str,
-        stream: bool = False,
-        raw: bool = False,
-    ) -> Union[Response, Generator[Union[Dict[str, str], Response], None, None]]:
-        """Search using the Monica API and get AI-generated responses.
-        
-        This method sends a search query to Monica and returns the AI-generated response.
-        It supports both streaming and non-streaming modes, as well as raw response format.
-        
-        Args:
-            prompt (str): The search query or prompt to send to the API.
-            stream (bool, optional): If True, yields response chunks as they arrive.
-                                   If False, returns complete response. Defaults to False.
-            raw (bool, optional): If True, returns raw response dictionaries with 'text' key.
-                                If False, returns Response objects that convert to text automatically.
-                                Defaults to False.
-        
-        Returns:
-            Union[Response, Generator[Union[Dict[str, str], Response], None, None]]: 
-                - If stream=False: Returns complete response as Response object
-                - If stream=True: Yields response chunks as either Dict or Response objects
-        
-        Raises:
-            APIConnectionError: If the API request fails
-        """
-        task_id = str(uuid.uuid4())
-        payload = {
-            "pro": False,
-            "query": prompt,
-            "round": 1,
-            "session_id": self.session_id,
-            "language": "auto",
-            "task_id": task_id
-        }
-
-        def for_stream():
-            try:
-                with self.session.post(
-                    self.api_endpoint,
-                    json=payload,
-                    stream=True,
-                    cookies=self.cookies,
-                    timeout=self.timeout,
-                    proxies=self.proxies
-                ) as response:
-                    if not response.ok:
-                        raise exceptions.APIConnectionError(
-                            f"Failed to generate response - ({response.status_code}, {response.reason}) - {response.text}"
-                        )
-                    
-                    # Process the Server-Sent Events (SSE) stream
-                    for line in response.iter_lines(decode_unicode=True):
-                        if line and line.startswith("data: "):
-                            try:
-                                data = json.loads(line[6:])  # Remove 'data: ' prefix
-                                
-                                # Save session_id for future requests if present
-                                if "session_id" in data and data["session_id"]:
-                                    self.session_id = data["session_id"]
-                                
-                                # Only process chunks with text content
-                                if "text" in data and data["text"]:
-                                    text_chunk = data["text"]
-                                    
-                                    if raw:
-                                        yield {"text": text_chunk}
-                                    else:
-                                        yield Response(text_chunk)
-                                        
-                                # Check if stream is finished
-                                if "finished" in data and data["finished"]:
-                                    break
-                                    
-                            except json.JSONDecodeError:
-                                continue
-                    
-            except requests.exceptions.RequestException as e:
-                raise exceptions.APIConnectionError(f"Request failed: {e}")
-
-        def for_non_stream():
-            full_response = ""
-            search_results = []
-            
-            for chunk in for_stream():
-                if raw:
-                    yield chunk
-                else:
-                    full_response += str(chunk)
-            
-            if not raw:
-                # Process the full response to clean up formatting
-                formatted_response = self.format_response(full_response)
-                self.last_response = Response(formatted_response)
-                return self.last_response
-
-        return for_stream() if stream else for_non_stream()
-
-    @staticmethod
-    def format_response(text: str) -> str:
-        """Format the response text for better readability.
-        
-        Args:
-            text (str): The raw response text
-        
-        Returns:
-            str: Formatted text
-        """
-        # Clean up markdown formatting
-        cleaned_text = text.replace('**', '')
-        
-        # Remove any empty lines
-        cleaned_text = re.sub(r'\n\s*\n', '\n\n', cleaned_text)
-        
-        # Remove any trailing whitespace
-        cleaned_text = cleaned_text.strip()
-        
-        return cleaned_text
-
-
-if __name__ == "__main__":
-    from rich import print
-
-    ai = Monica()
-    response = ai.search(input(">>> "), stream=True, raw=False)
-    for chunk in response:
-        print(chunk, end="", flush=True)