webscout 6.4__py3-none-any.whl → 6.5__py3-none-any.whl

webscout/Agents/Onlinesearcher.py

@@ -1,182 +0,0 @@
-import json
-import httpx
-from bs4 import BeautifulSoup
-from typing import List, Dict
-from webscout import GoogleS, GEMINIAPI
-import re
-from concurrent.futures import ThreadPoolExecutor, as_completed
-
-
-class WebSearchAgent:
-    def __init__(self):
-        self.webs = GoogleS()
-        self.ai = GEMINIAPI(is_conversation=False, api_key='AIzaSyAYlT5-V0MXZwaLYpXCF1Z-Yvy_tx1jylA')
-
-    def generate_search_queries(self, information: str, num_queries: int = 10) -> List[str]:
-        prompt = f""" Task: Generate exactly {num_queries} optimal search queries based on the given information.  
-Instructions: 
-1. Analyze the provided information thoroughly. 
-2. Identify key concepts, entities, and relationships. 
-3. Formulate {num_queries} concise and specific search queries that will yield relevant and diverse results. 
-4. Each query should focus on a different aspect or angle of the information. 
-5. The queries should be in natural language, not in the form of keywords. 
-6. Avoid unnecessary words or phrases that might limit the search results. 
-7. **Important**: Return the response **ONLY** in JSON format without any additional text or code blocks.  
-Your response must be in the following JSON format: {{
-    "search_queries": [
-        "Your first search query here",
-        "Your second search query here",
-        "...",
-        "Your last search query here"
-    ]
-}}  
-Ensure that: 
-- You provide exactly {num_queries} search queries. 
-- Each query is unique and focuses on a different aspect of the information. 
-- The queries are in plain text, suitable for a web search engine.  
-
-Information to base the search queries on: 
-{information}  
-
-Now, generate the optimal search queries: """
-
-        response = ""
-        for chunk in self.ai.chat(prompt):
-            response += chunk
-
-        json_match = re.search(r'\{.*\}', response, re.DOTALL)
-        if json_match:
-            json_str = json_match.group(0)
-            try:
-                json_response = json.loads(json_str)
-                print(json_response['search_queries'])
-                return json_response["search_queries"]
-            except json.JSONDecodeError:
-                pass
-
-        queries = re.findall(r'"([^"]+)"', response)
-        if len(queries) >= num_queries:
-            return queries[:num_queries]
-        elif queries:
-            return queries
-        else:
-            return [information]
-
-    def search(self, information: str, region: str = 'wt-wt', safe: str = 'off',
-               max_results: int = 10) -> List[Dict]:
-        search_queries = self.generate_search_queries(information, num_queries=10)
-        all_results = []
-
-        for query in search_queries:
-            results = []
-            with self.webs as webs:
-                for result in webs.search(query, region=region, safe=safe,
-                                       max_results=max_results):
-                    results.append(result)
-            all_results.extend(results)
-
-        return all_results
-
-    def extract_urls(self, results: List[Dict]) -> List[str]:
-        urls = [result.get('href') for result in results if result.get('href')]
-        unique_urls = list(set(urls))
-        return unique_urls
-
-    def fetch_webpage(self, url: str) -> Dict[str, str]:
-        try:
-            with httpx.Client(timeout=120) as client:
-                response = client.get(url)
-                if response.status_code == 200:
-                    html = response.text
-                    soup = BeautifulSoup(html, 'html.parser')
-                    paragraphs = soup.find_all('p')
-                    text = ' '.join([p.get_text() for p in paragraphs])
-                    words = text.split()
-                    if len(words) > 600:
-                        text = ' '.join(words[:600]) + '...'
-                    return {"url": url, "content": text}
-                else:
-                    return {"url": url, "content": f"Failed to fetch {url}: HTTP {response.status_code}"}
-        except Exception as e:
-            return {"url": url, "content": f"Error fetching {url}: {str(e)}"}
-
-    def fetch_all_webpages(self, urls: List[str], max_workers: int = 10) -> List[Dict[str, str]]:
-        contents = []
-        with ThreadPoolExecutor(max_workers=max_workers) as executor:
-            future_to_url = {executor.submit(self.fetch_webpage, url): url for url in urls}
-            for future in as_completed(future_to_url):
-                result = future.result()
-                contents.append(result)
-        return contents
-
-
-class OnlineSearcher:
-    def __init__(self):
-        self.agent = WebSearchAgent()
-        self.ai = GEMINIAPI(is_conversation=False, api_key='GOOGLE GEMINI API')
-
-    def answer_question(self, question: str) -> None:
-        search_results = self.agent.search(question, max_results=10)
-        urls = self.agent.extract_urls(search_results)
-        webpage_contents = self.agent.fetch_all_webpages(urls)
-
-        context = "Web search results and extracted content:\n\n"
-        for i, result in enumerate(search_results, 1):
-            title = result.get('title', 'No Title')
-            href = result.get('href', 'No URL')
-            snippet = result.get('body', 'No Snippet')
-            context += f"{i}. **Title:** {title}\n   **URL:** {href}\n   **Snippet:** {snippet}\n\n"
-
-        context += "Extracted webpage contents:\n"
-        for i, webpage in enumerate(webpage_contents, 1):
-            content = webpage['content']
-            content_preview = content[:600] + '...' if len(content) > 600 else content
-            context += f"{i}. **URL:** {webpage['url']}\n   **Content:** {content_preview}\n\n"
-
-        prompt = f""" Task: Provide a comprehensive, insightful, and well-structured answer to the given question based on the provided web search results and your general knowledge.  
-Question: {question}  
-Context: {context}  
-Instructions:  
-1. Carefully analyze the provided web search results and extracted content.  
-2. Synthesize the information to form a coherent and comprehensive answer.  
-3. If the search results contain relevant information, incorporate it into your answer seamlessly.  
-4. Avoid providing irrelevant information, and do not reference "according to web page".  
-5. If the search results don't contain sufficient information, clearly state this and provide the best answer based on your general knowledge.  
-6. Ensure your answer is well-structured, factual, and directly addresses the question.  
-7. Use clear headings, bullet points, or other formatting tools to enhance readability where appropriate.  
-8. Strive for a tone and style similar to that of professional, authoritative sources like Perplexity, ensuring clarity and depth in your response.  
-Your response should be informative, accurate, and properly sourced when possible. Begin your answer now: """
-
-        for chunk in self.ai.chat(prompt, stream=True):
-            print(chunk, end='', flush=True)
-
-
-
-# Usage example
-if __name__ == "__main__":
-    assistant = OnlineSearcher()
-    while True:
-        try:
-            question = input(">>> ")
-            if question.lower() == 'quit':
-                break
-            print("=" * 50)
-            assistant.answer_question(question) 
-            print("=" * 50)
-        except KeyboardInterrupt:
-            print("\nExiting.")
-            break
-        except Exception as e:
-            print(f"An error occurred: {e}")
-
-"""
-def format_prompt(messages: Messages, add_special_tokens=False) -> str:
-
-    if not add_special_tokens and len(messages) <= 1:
-        return messages[0]["content"]
-    formatted = "\n".join([
-        f'{message["role"].capitalize()}: {message["content"]}'
-        for message in messages
-    ])
-    return f"{formatted}\nAssistant:
-"""

webscout/Agents/__init__.py

@@ -1,2 +0,0 @@
-from .Onlinesearcher import *
-from .functioncall import *

webscout/Agents/functioncall.py

@@ -1,248 +0,0 @@
-from datetime import date
-import json
-import logging
-import time
-from typing import Any, Dict, List, Optional, Union, Callable
-from dataclasses import dataclass
-import asyncio
-import requests
-from jinja2 import Template
-from webscout import WEBS, ChatGPTES
-
-@dataclass
-class ToolParameter:
-    name: str
-    type: str
-    description: str
-    required: bool = False
-
-@dataclass
-class Tool:
-    name: str
-    description: str
-    parameters: Dict[str, ToolParameter]
-    function: Callable
-    is_async: bool = False
-
-class ToolRegistry:
-    def __init__(self):
-        self._tools: Dict[str, Tool] = {}
-    
-    def register(self, tool: Tool):
-        self._tools[tool.name] = tool
-    
-    def get_tool(self, name: str) -> Optional[Tool]:
-        return self._tools.get(name)
-    
-    def list_tools(self) -> List[Tool]:
-        return list(self._tools.values())
-
-    def to_schema(self) -> List[Dict]:
-        return [
-            {
-                "type": "function",
-                "function": {
-                    "name": tool.name,
-                    "description": tool.description,
-                    "parameters": {
-                        "type": "object",
-                        "properties": {
-                            param.name: {
-                                "type": param.type,
-                                "description": param.description
-                            } for param in tool.parameters.values()
-                        },
-                        "required": [
-                            param.name for param in tool.parameters.values() 
-                            if param.required
-                        ]
-                    }
-                }
-            }
-            for tool in self._tools.values()
-        ]
-
-class FunctionCallingAgent:
-    SYSTEM_TEMPLATE = Template("""You are an advanced AI assistant tasked with analyzing user requests and determining the most appropriate action. You have access to the following tools:
-
-{{ tools_description }}
-
-Instructions:
-1. Carefully analyze the user's request.
-2. Determine which tools (if any) are necessary to fulfill the request.
-3. You can make multiple tool calls if needed to complete the task.
-4. If you decide to use tool(s), respond ONLY with a JSON array in this format:
-   [
-     {
-       "name": "tool_name",
-       "arguments": {
-         "param1": "value1",
-         "param2": "value2"
-       }
-     },
-     ... (more tool calls as needed)
-   ]
-
-5. If no tool is needed, respond with an empty array: []
-
-The current date is {{ current_date }}. Your knowledge cutoff is {{ knowledge_cutoff }}.
-
-User Request: {{ user_message }}
-
-Your Response (JSON array only):""")
-
-    def __init__(self, registry: ToolRegistry = None):
-        self.ai = ChatGPTES(timeout=300, intro=None)
-        self.registry = registry or ToolRegistry()
-        self.knowledge_cutoff = "September 2022"
-        self.logger = logging.getLogger(__name__)
-
-    def _generate_system_message(self, user_message: str) -> str:
-        tools_description = ""
-        for tool in self.registry.list_tools():
-            tools_description += f"- {tool.name}: {tool.description}\n"
-            tools_description += "    Parameters:\n"
-            for param in tool.parameters.values():
-                tools_description += f"      - {param.name}: {param.description} ({param.type})\n"
-        
-        current_date = date.today().strftime("%B %d, %Y")
-        return self.SYSTEM_TEMPLATE.render(
-            tools_description=tools_description,
-            current_date=current_date,
-            knowledge_cutoff=self.knowledge_cutoff,
-            user_message=user_message
-        )
-
-    async def _execute_tool(self, tool_call: Dict[str, Any]) -> Any:
-        tool_name = tool_call.get("name")
-        arguments = tool_call.get("arguments", {})
-
-        tool = self.registry.get_tool(tool_name)
-        if not tool:
-            raise ValueError(f"Unknown tool: {tool_name}")
-
-        try:
-            if tool.is_async:
-                return await tool.function(**arguments)
-            else:
-                return tool.function(**arguments)
-        except Exception as e:
-            self.logger.error(f"Error executing tool {tool_name}: {str(e)}")
-            raise
-
-    async def process_request(self, message: str) -> List[Any]:
-        """Process a user request and execute any necessary tool calls."""
-        try:
-            system_message = self._generate_system_message(message)
-            response = self.ai.chat(system_message)
-            self.logger.debug(f"Raw AI response: {response}")
-
-            tool_calls = self._parse_tool_calls(response)
-            if not tool_calls:
-                return []
-
-            results = []
-            for tool_call in tool_calls:
-                result = await self._execute_tool(tool_call)
-                results.append(result)
-            return results
-
-        except Exception as e:
-            self.logger.error(f"Error processing request: {str(e)}")
-            raise
-
-    def _parse_tool_calls(self, response: str) -> List[Dict[str, Any]]:
-        """Parse the AI response into a list of tool calls."""
-        try:
-            # Find the JSON array in the response
-            start_idx = response.find("[")
-            end_idx = response.rfind("]") + 1
-
-            if start_idx == -1 or end_idx == -1:
-                return []
-
-            # Extract and parse the JSON array
-            response_json = json.loads(response[start_idx:end_idx])
-            
-            if not isinstance(response_json, list):
-                response_json = [response_json]
-                
-            return response_json
-
-        except (json.JSONDecodeError, ValueError) as e:
-            self.logger.error(f"Error parsing tool calls: {str(e)}")
-            return []
-
-# Example usage
-if __name__ == "__main__":
-    logging.basicConfig(
-        level=logging.INFO,
-        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
-    )
-
-    # Create and configure the tool registry
-    registry = ToolRegistry()
-
-    def web_search(query: str) -> str:
-        # Implement actual web search logic
-        return f"Search results for: {query}"
-
-    def get_user_detail(name: str, age: int) -> Dict[str, Any]:
-        return {"name": name, "age": age}
-
-    # Register tools
-    registry.register(Tool(
-        name="web_search",
-        description="Search the web for current information",
-        parameters={
-            "query": ToolParameter(
-                name="query",
-                type="string",
-                description="The search query to execute",
-                required=True
-            )
-        },
-        function=web_search
-    ))
-
-    registry.register(Tool(
-        name="get_user_detail",
-        description="Get user details",
-        parameters={
-            "name": ToolParameter(
-                name="name",
-                type="string",
-                description="User's name",
-                required=True
-            ),
-            "age": ToolParameter(
-                name="age",
-                type="integer",
-                description="User's age",
-                required=True
-            )
-        },
-        function=get_user_detail
-    ))
-
-    # Create agent
-    agent = FunctionCallingAgent(registry=registry)
-
-    # Test cases
-    test_messages = [
-        "What's the weather like in New York today?",
-        "Get user details name as John and age as 30",
-        "Search for latest news about AI",
-    ]
-
-    async def run_tests():
-        for message in test_messages:
-            print(f"\nProcessing: {message}")
-            try:
-                results = await agent.process_request(message)
-                print(f"Results: {results}")
-            except Exception as e:
-                print(f"Error: {str(e)}")
-
-    # Run test cases
-    asyncio.run(run_tests())

webscout/Bing_search.py

@@ -1,251 +0,0 @@
-from bs4 import BeautifulSoup
-import requests
-from typing import Dict, List, Optional, Union
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from urllib.parse import quote, urlparse, parse_qs
-import base64
-import urllib3
-
-# Disable SSL warnings
-urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-
-class BingS:
-    """A Python interface for Bing search engine.
-    
-The BingS class provides a simple interface to perform searches on Bing.com
-and extract search results programmatically.
-
-Basic Usage:
-    >>> from webscout.Bing_search import BingS
-    >>> searcher = BingS()
-    >>> results = searcher.search("Python programming")
-    >>> for result in results:
-    ...     print(result['title'], result['href'])
-
-Advanced Usage:
-    >>> # With custom headers and proxy
-    >>> headers = {'User-Agent': 'Custom User Agent'}
-    >>> proxy = 'http://proxy.example.com:8080'
-    >>> searcher = BingS(headers=headers, proxy=proxy)
-    >>> results = searcher.search(
-    ...     "AI developments",
-    ...     max_results=5,
-    ...     extract_webpage_text=True,
-    ...     max_extract_characters=1000
-    ... )
-    >>> # Access result fields
-    >>> for result in results:
-    ...     print(f"Title: {result['title']}")
-    ...     print(f"URL: {result['href']}")
-    ...     print(f"Description: {result['abstract']}")
-    ...     if result['visible_text']:
-    ...         print(f"Page Content: {result['visible_text'][:100]}...")
-
-The class supports context management protocol:
-    >>> with BingS() as searcher:
-    ...     results = searcher.search("Python tutorials")
-
-Return Dictionary Format:
-    {
-        'title': str,       # The title of the search result
-        'href': str,        # The URL of the search result
-        'abstract': str,    # Brief description or snippet
-        'index': int,       # Position in search results
-        'type': str,        # Type of result (always 'web')
-        'visible_text': str # Extracted webpage text (if requested)
-    }
-"""
-
-    _executor: ThreadPoolExecutor = ThreadPoolExecutor(max_workers=10)
-
-    def __init__(
-        self,
-        headers: Optional[Dict[str, str]] = None,
-        proxy: Optional[str] = None,
-        timeout: Optional[int] = 10,
-    ) -> None:
-        """Initialize a new BingS instance.
-
-        Args:
-            headers (Optional[Dict[str, str]]): Custom HTTP headers for requests.
-                Defaults to a standard User-Agent if not provided.
-            proxy (Optional[str]): Proxy URL to use for requests.
-                Example: 'http://proxy.example.com:8080'
-            timeout (Optional[int]): Request timeout in seconds. Defaults to 10.
-
-        Example:
-            >>> searcher = BingS(
-            ...     headers={'User-Agent': 'Custom UA'},
-            ...     proxy='http://proxy.example.com:8080',
-            ...     timeout=15
-            ... )
-        """
-        self.proxy: Optional[str] = proxy
-        self.headers = headers if headers else {
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
-        }
-        self.headers["Referer"] = "https://www.bing.com/"
-        self.client = requests.Session()
-        self.client.headers.update(self.headers)
-        self.client.proxies.update({"http": self.proxy, "https": self.proxy})
-        self.timeout = timeout
-
-    def __enter__(self) -> "BingS":
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        self.client.close()
-
-    def _get_url(
-        self,
-        method: str,
-        url: str,
-        params: Optional[Dict[str, str]] = None,
-        data: Optional[Union[Dict[str, str], bytes]] = None,
-    ) -> bytes:
-        try:
-            resp = self.client.request(method, url, params=params, data=data, timeout=self.timeout, verify=False)
-        except Exception as ex:
-            raise Exception(f"{url} {type(ex).__name__}: {ex}") from ex
-        if resp.status_code == 200:
-            return resp.content
-        raise Exception(f"{resp.url} returned status code {resp.status_code}. {params=} {data=}")
-
-    def extract_text_from_webpage(self, html_content, max_characters=None):
-        """Extracts visible text from HTML content using BeautifulSoup."""
-        soup = BeautifulSoup(html_content, "html.parser")
-        # Remove unwanted tags
-        for tag in soup(["script", "style", "header", "footer", "nav"]):
-            tag.extract()
-        # Get the remaining visible text
-        visible_text = soup.get_text(separator=' ', strip=True)
-        if max_characters:
-            visible_text = visible_text[:max_characters]
-        return visible_text
-
-    def search(
-        self,
-        keywords: str,
-        max_results: Optional[int] = 10,
-        extract_webpage_text: bool = False,
-        max_extract_characters: Optional[int] = 100,
-    ) -> List[Dict[str, str]]:
-        """Perform a Bing search and return results.
-
-        Args:
-            keywords (str): Search query string.
-            max_results (Optional[int]): Maximum number of results to return.
-                Defaults to 10.
-            extract_webpage_text (bool): If True, fetches and extracts text from
-                each result webpage. Defaults to False.
-            max_extract_characters (Optional[int]): Maximum number of characters
-                to extract from each webpage. Only used if extract_webpage_text
-                is True. Defaults to 100.
-
-        Returns:
-            List[Dict[str, str]]: List of search results. Each result contains:
-                - title: The title of the search result
-                - href: The URL of the search result
-                - abstract: Brief description or snippet
-                - index: Position in search results
-                - type: Type of result (always 'web')
-                - visible_text: Extracted webpage text (if extract_webpage_text=True)
-
-        Raises:
-            AssertionError: If keywords is empty.
-            Exception: If request fails or returns non-200 status code.
-
-        Example:
-            >>> searcher = BingS()
-            >>> results = searcher.search(
-            ...     "Python tutorials",
-            ...     max_results=5,
-            ...     extract_webpage_text=True
-            ... )
-            >>> for result in results:
-            ...     print(f"Title: {result['title']}")
-            ...     print(f"URL: {result['href']}")
-            ...     print(f"Description: {result['abstract']}")
-            ...     if result['visible_text']:
-            ...         print(f"Content: {result['visible_text'][:100]}...")
-        """
-        assert keywords, "keywords is mandatory"
-
-        results = []
-        futures = []
-        start = 1
-        while len(results) < max_results:
-            params = {
-                "q": keywords,
-                "first": start
-            }
-            futures.append(self._executor.submit(self._get_url, "GET", "https://www.bing.com/search", params=params))
-            start += 10
-
-            for future in as_completed(futures):
-                try:
-                    resp_content = future.result()
-                    soup = BeautifulSoup(resp_content, "html.parser")
-                    result_block = soup.select('li.b_algo')
-
-                    if not result_block:
-                        break
-
-                    for result in result_block:
-                        try:
-                            link = result.select_one('h2 a')
-                            title = link.text if link else ""
-                            url = link['href'] if link else ""
-                            abstract = result.select_one('.b_caption p')
-                            description = abstract.text if abstract else ""
-
-                            # Remove "WEB" from the beginning of the description if it exists
-                            if description.startswith("WEB"):
-                                description = description[3:].strip()
-
-                            visible_text = ""
-                            if extract_webpage_text:
-                                try:
-                                    actual_url = self._decode_bing_url(url)
-                                    page_content = self._get_url("GET", actual_url)
-                                    visible_text = self.extract_text_from_webpage(
-                                        page_content, max_characters=max_extract_characters
-                                    )
-                                except Exception as e:
-                                    print(f"Error extracting text from {url}: {e}")
-
-                            results.append({
-                                "title": title,
-                                "href": url,
-                                "abstract": description,
-                                "index": len(results),
-                                "type": "web",
-                                "visible_text": visible_text,
-                            })
-
-                            if len(results) >= max_results:
-                                return results
-
-                        except Exception as e:
-                            print(f"Error extracting result: {e}")
-
-                except Exception as e:
-                    print(f"Error fetching URL: {e}")
-
-        return results
-
-    def _decode_bing_url(self, url):
-        if 'bing.com/ck/a' in url:
-            parsed_url = urlparse(url)
-            query_params = parse_qs(parsed_url.query)
-            if 'u' in query_params:
-                encoded_url = query_params['u'][0]
-                return base64.b64decode(encoded_url).decode('utf-8')
-        return url
-
-if __name__ == "__main__":
-    from rich import print
-    searcher = BingS()
-    results = searcher.search("Python development tools", max_results=5, extract_webpage_text=True, max_extract_characters=2000)
-    for result in results:
-        print(result)