webscout 6.4__py3-none-any.whl → 6.6__py3-none-any.whl

webscout/Agents/functioncall.py

@@ -1,248 +0,0 @@
-from datetime import date
-import json
-import logging
-import time
-from typing import Any, Dict, List, Optional, Union, Callable
-from dataclasses import dataclass
-import asyncio
-import requests
-from jinja2 import Template
-from webscout import WEBS, ChatGPTES
-
-@dataclass
-class ToolParameter:
-    name: str
-    type: str
-    description: str
-    required: bool = False
-
-@dataclass
-class Tool:
-    name: str
-    description: str
-    parameters: Dict[str, ToolParameter]
-    function: Callable
-    is_async: bool = False
-
-class ToolRegistry:
-    def __init__(self):
-        self._tools: Dict[str, Tool] = {}
-    
-    def register(self, tool: Tool):
-        self._tools[tool.name] = tool
-    
-    def get_tool(self, name: str) -> Optional[Tool]:
-        return self._tools.get(name)
-    
-    def list_tools(self) -> List[Tool]:
-        return list(self._tools.values())
-
-    def to_schema(self) -> List[Dict]:
-        return [
-            {
-                "type": "function",
-                "function": {
-                    "name": tool.name,
-                    "description": tool.description,
-                    "parameters": {
-                        "type": "object",
-                        "properties": {
-                            param.name: {
-                                "type": param.type,
-                                "description": param.description
-                            } for param in tool.parameters.values()
-                        },
-                        "required": [
-                            param.name for param in tool.parameters.values() 
-                            if param.required
-                        ]
-                    }
-                }
-            }
-            for tool in self._tools.values()
-        ]
-
-class FunctionCallingAgent:
-    SYSTEM_TEMPLATE = Template("""You are an advanced AI assistant tasked with analyzing user requests and determining the most appropriate action. You have access to the following tools:
-
-{{ tools_description }}
-
-Instructions:
-1. Carefully analyze the user's request.
-2. Determine which tools (if any) are necessary to fulfill the request.
-3. You can make multiple tool calls if needed to complete the task.
-4. If you decide to use tool(s), respond ONLY with a JSON array in this format:
-   [
-     {
-       "name": "tool_name",
-       "arguments": {
-         "param1": "value1",
-         "param2": "value2"
-       }
-     },
-     ... (more tool calls as needed)
-   ]
-
-5. If no tool is needed, respond with an empty array: []
-
-The current date is {{ current_date }}. Your knowledge cutoff is {{ knowledge_cutoff }}.
-
-User Request: {{ user_message }}
-
-Your Response (JSON array only):""")
-
-    def __init__(self, registry: ToolRegistry = None):
-        self.ai = ChatGPTES(timeout=300, intro=None)
-        self.registry = registry or ToolRegistry()
-        self.knowledge_cutoff = "September 2022"
-        self.logger = logging.getLogger(__name__)
-
-    def _generate_system_message(self, user_message: str) -> str:
-        tools_description = ""
-        for tool in self.registry.list_tools():
-            tools_description += f"- {tool.name}: {tool.description}\n"
-            tools_description += "    Parameters:\n"
-            for param in tool.parameters.values():
-                tools_description += f"      - {param.name}: {param.description} ({param.type})\n"
-        
-        current_date = date.today().strftime("%B %d, %Y")
-        return self.SYSTEM_TEMPLATE.render(
-            tools_description=tools_description,
-            current_date=current_date,
-            knowledge_cutoff=self.knowledge_cutoff,
-            user_message=user_message
-        )
-
-    async def _execute_tool(self, tool_call: Dict[str, Any]) -> Any:
-        tool_name = tool_call.get("name")
-        arguments = tool_call.get("arguments", {})
-
-        tool = self.registry.get_tool(tool_name)
-        if not tool:
-            raise ValueError(f"Unknown tool: {tool_name}")
-
-        try:
-            if tool.is_async:
-                return await tool.function(**arguments)
-            else:
-                return tool.function(**arguments)
-        except Exception as e:
-            self.logger.error(f"Error executing tool {tool_name}: {str(e)}")
-            raise
-
-    async def process_request(self, message: str) -> List[Any]:
-        """Process a user request and execute any necessary tool calls."""
-        try:
-            system_message = self._generate_system_message(message)
-            response = self.ai.chat(system_message)
-            self.logger.debug(f"Raw AI response: {response}")
-
-            tool_calls = self._parse_tool_calls(response)
-            if not tool_calls:
-                return []
-
-            results = []
-            for tool_call in tool_calls:
-                result = await self._execute_tool(tool_call)
-                results.append(result)
-            return results
-
-        except Exception as e:
-            self.logger.error(f"Error processing request: {str(e)}")
-            raise
-
-    def _parse_tool_calls(self, response: str) -> List[Dict[str, Any]]:
-        """Parse the AI response into a list of tool calls."""
-        try:
-            # Find the JSON array in the response
-            start_idx = response.find("[")
-            end_idx = response.rfind("]") + 1
-
-            if start_idx == -1 or end_idx == -1:
-                return []
-
-            # Extract and parse the JSON array
-            response_json = json.loads(response[start_idx:end_idx])
-            
-            if not isinstance(response_json, list):
-                response_json = [response_json]
-                
-            return response_json
-
-        except (json.JSONDecodeError, ValueError) as e:
-            self.logger.error(f"Error parsing tool calls: {str(e)}")
-            return []
-
-# Example usage
-if __name__ == "__main__":
-    logging.basicConfig(
-        level=logging.INFO,
-        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
-    )
-
-    # Create and configure the tool registry
-    registry = ToolRegistry()
-
-    def web_search(query: str) -> str:
-        # Implement actual web search logic
-        return f"Search results for: {query}"
-
-    def get_user_detail(name: str, age: int) -> Dict[str, Any]:
-        return {"name": name, "age": age}
-
-    # Register tools
-    registry.register(Tool(
-        name="web_search",
-        description="Search the web for current information",
-        parameters={
-            "query": ToolParameter(
-                name="query",
-                type="string",
-                description="The search query to execute",
-                required=True
-            )
-        },
-        function=web_search
-    ))
-
-    registry.register(Tool(
-        name="get_user_detail",
-        description="Get user details",
-        parameters={
-            "name": ToolParameter(
-                name="name",
-                type="string",
-                description="User's name",
-                required=True
-            ),
-            "age": ToolParameter(
-                name="age",
-                type="integer",
-                description="User's age",
-                required=True
-            )
-        },
-        function=get_user_detail
-    ))
-
-    # Create agent
-    agent = FunctionCallingAgent(registry=registry)
-
-    # Test cases
-    test_messages = [
-        "What's the weather like in New York today?",
-        "Get user details name as John and age as 30",
-        "Search for latest news about AI",
-    ]
-
-    async def run_tests():
-        for message in test_messages:
-            print(f"\nProcessing: {message}")
-            try:
-                results = await agent.process_request(message)
-                print(f"Results: {results}")
-            except Exception as e:
-                print(f"Error: {str(e)}")
-
-    # Run test cases
-    asyncio.run(run_tests())

webscout/Bing_search.py

@@ -1,251 +0,0 @@
-from bs4 import BeautifulSoup
-import requests
-from typing import Dict, List, Optional, Union
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from urllib.parse import quote, urlparse, parse_qs
-import base64
-import urllib3
-
-# Disable SSL warnings
-urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-
-class BingS:
-    """A Python interface for Bing search engine.
-    
-The BingS class provides a simple interface to perform searches on Bing.com
-and extract search results programmatically.
-
-Basic Usage:
-    >>> from webscout.Bing_search import BingS
-    >>> searcher = BingS()
-    >>> results = searcher.search("Python programming")
-    >>> for result in results:
-    ...     print(result['title'], result['href'])
-
-Advanced Usage:
-    >>> # With custom headers and proxy
-    >>> headers = {'User-Agent': 'Custom User Agent'}
-    >>> proxy = 'http://proxy.example.com:8080'
-    >>> searcher = BingS(headers=headers, proxy=proxy)
-    >>> results = searcher.search(
-    ...     "AI developments",
-    ...     max_results=5,
-    ...     extract_webpage_text=True,
-    ...     max_extract_characters=1000
-    ... )
-    >>> # Access result fields
-    >>> for result in results:
-    ...     print(f"Title: {result['title']}")
-    ...     print(f"URL: {result['href']}")
-    ...     print(f"Description: {result['abstract']}")
-    ...     if result['visible_text']:
-    ...         print(f"Page Content: {result['visible_text'][:100]}...")
-
-The class supports context management protocol:
-    >>> with BingS() as searcher:
-    ...     results = searcher.search("Python tutorials")
-
-Return Dictionary Format:
-    {
-        'title': str,       # The title of the search result
-        'href': str,        # The URL of the search result
-        'abstract': str,    # Brief description or snippet
-        'index': int,       # Position in search results
-        'type': str,        # Type of result (always 'web')
-        'visible_text': str # Extracted webpage text (if requested)
-    }
-"""
-
-    _executor: ThreadPoolExecutor = ThreadPoolExecutor(max_workers=10)
-
-    def __init__(
-        self,
-        headers: Optional[Dict[str, str]] = None,
-        proxy: Optional[str] = None,
-        timeout: Optional[int] = 10,
-    ) -> None:
-        """Initialize a new BingS instance.
-
-        Args:
-            headers (Optional[Dict[str, str]]): Custom HTTP headers for requests.
-                Defaults to a standard User-Agent if not provided.
-            proxy (Optional[str]): Proxy URL to use for requests.
-                Example: 'http://proxy.example.com:8080'
-            timeout (Optional[int]): Request timeout in seconds. Defaults to 10.
-
-        Example:
-            >>> searcher = BingS(
-            ...     headers={'User-Agent': 'Custom UA'},
-            ...     proxy='http://proxy.example.com:8080',
-            ...     timeout=15
-            ... )
-        """
-        self.proxy: Optional[str] = proxy
-        self.headers = headers if headers else {
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
-        }
-        self.headers["Referer"] = "https://www.bing.com/"
-        self.client = requests.Session()
-        self.client.headers.update(self.headers)
-        self.client.proxies.update({"http": self.proxy, "https": self.proxy})
-        self.timeout = timeout
-
-    def __enter__(self) -> "BingS":
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        self.client.close()
-
-    def _get_url(
-        self,
-        method: str,
-        url: str,
-        params: Optional[Dict[str, str]] = None,
-        data: Optional[Union[Dict[str, str], bytes]] = None,
-    ) -> bytes:
-        try:
-            resp = self.client.request(method, url, params=params, data=data, timeout=self.timeout, verify=False)
-        except Exception as ex:
-            raise Exception(f"{url} {type(ex).__name__}: {ex}") from ex
-        if resp.status_code == 200:
-            return resp.content
-        raise Exception(f"{resp.url} returned status code {resp.status_code}. {params=} {data=}")
-
-    def extract_text_from_webpage(self, html_content, max_characters=None):
-        """Extracts visible text from HTML content using BeautifulSoup."""
-        soup = BeautifulSoup(html_content, "html.parser")
-        # Remove unwanted tags
-        for tag in soup(["script", "style", "header", "footer", "nav"]):
-            tag.extract()
-        # Get the remaining visible text
-        visible_text = soup.get_text(separator=' ', strip=True)
-        if max_characters:
-            visible_text = visible_text[:max_characters]
-        return visible_text
-
-    def search(
-        self,
-        keywords: str,
-        max_results: Optional[int] = 10,
-        extract_webpage_text: bool = False,
-        max_extract_characters: Optional[int] = 100,
-    ) -> List[Dict[str, str]]:
-        """Perform a Bing search and return results.
-
-        Args:
-            keywords (str): Search query string.
-            max_results (Optional[int]): Maximum number of results to return.
-                Defaults to 10.
-            extract_webpage_text (bool): If True, fetches and extracts text from
-                each result webpage. Defaults to False.
-            max_extract_characters (Optional[int]): Maximum number of characters
-                to extract from each webpage. Only used if extract_webpage_text
-                is True. Defaults to 100.
-
-        Returns:
-            List[Dict[str, str]]: List of search results. Each result contains:
-                - title: The title of the search result
-                - href: The URL of the search result
-                - abstract: Brief description or snippet
-                - index: Position in search results
-                - type: Type of result (always 'web')
-                - visible_text: Extracted webpage text (if extract_webpage_text=True)
-
-        Raises:
-            AssertionError: If keywords is empty.
-            Exception: If request fails or returns non-200 status code.
-
-        Example:
-            >>> searcher = BingS()
-            >>> results = searcher.search(
-            ...     "Python tutorials",
-            ...     max_results=5,
-            ...     extract_webpage_text=True
-            ... )
-            >>> for result in results:
-            ...     print(f"Title: {result['title']}")
-            ...     print(f"URL: {result['href']}")
-            ...     print(f"Description: {result['abstract']}")
-            ...     if result['visible_text']:
-            ...         print(f"Content: {result['visible_text'][:100]}...")
-        """
-        assert keywords, "keywords is mandatory"
-
-        results = []
-        futures = []
-        start = 1
-        while len(results) < max_results:
-            params = {
-                "q": keywords,
-                "first": start
-            }
-            futures.append(self._executor.submit(self._get_url, "GET", "https://www.bing.com/search", params=params))
-            start += 10
-
-            for future in as_completed(futures):
-                try:
-                    resp_content = future.result()
-                    soup = BeautifulSoup(resp_content, "html.parser")
-                    result_block = soup.select('li.b_algo')
-
-                    if not result_block:
-                        break
-
-                    for result in result_block:
-                        try:
-                            link = result.select_one('h2 a')
-                            title = link.text if link else ""
-                            url = link['href'] if link else ""
-                            abstract = result.select_one('.b_caption p')
-                            description = abstract.text if abstract else ""
-
-                            # Remove "WEB" from the beginning of the description if it exists
-                            if description.startswith("WEB"):
-                                description = description[3:].strip()
-
-                            visible_text = ""
-                            if extract_webpage_text:
-                                try:
-                                    actual_url = self._decode_bing_url(url)
-                                    page_content = self._get_url("GET", actual_url)
-                                    visible_text = self.extract_text_from_webpage(
-                                        page_content, max_characters=max_extract_characters
-                                    )
-                                except Exception as e:
-                                    print(f"Error extracting text from {url}: {e}")
-
-                            results.append({
-                                "title": title,
-                                "href": url,
-                                "abstract": description,
-                                "index": len(results),
-                                "type": "web",
-                                "visible_text": visible_text,
-                            })
-
-                            if len(results) >= max_results:
-                                return results
-
-                        except Exception as e:
-                            print(f"Error extracting result: {e}")
-
-                except Exception as e:
-                    print(f"Error fetching URL: {e}")
-
-        return results
-
-    def _decode_bing_url(self, url):
-        if 'bing.com/ck/a' in url:
-            parsed_url = urlparse(url)
-            query_params = parse_qs(parsed_url.query)
-            if 'u' in query_params:
-                encoded_url = query_params['u'][0]
-                return base64.b64decode(encoded_url).decode('utf-8')
-        return url
-
-if __name__ == "__main__":
-    from rich import print
-    searcher = BingS()
-    results = searcher.search("Python development tools", max_results=5, extract_webpage_text=True, max_extract_characters=2000)
-    for result in results:
-        print(result)