agno 2.0.1__py3-none-any.whl → 2.3.0__py3-none-any.whl

agno/knowledge/reader/url_reader.py

@@ -1,128 +0,0 @@
-from io import BytesIO
-from os.path import basename
-from pathlib import Path
-from typing import List, Optional
-from urllib.parse import urlparse
-
-import httpx
-
-from agno.knowledge.chunking.fixed import FixedSizeChunking
-from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
-from agno.knowledge.document.base import Document
-from agno.knowledge.reader.base import Reader
-from agno.knowledge.reader.csv_reader import CSVReader
-from agno.knowledge.reader.pdf_reader import PDFReader
-from agno.knowledge.types import ContentType
-from agno.utils.http import async_fetch_with_retry, fetch_with_retry
-from agno.utils.log import log_debug
-
-
-class URLReader(Reader):
-    """Reader for general URL content"""
-
-    def __init__(
-        self, chunking_strategy: Optional[ChunkingStrategy] = FixedSizeChunking(), proxy: Optional[str] = None, **kwargs
-    ):
-        super().__init__(chunking_strategy=chunking_strategy, **kwargs)
-        self.proxy = proxy
-
-    @classmethod
-    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
-        """Get the list of supported chunking strategies for URL readers."""
-        return [
-            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
-            ChunkingStrategyType.AGENTIC_CHUNKER,
-            ChunkingStrategyType.DOCUMENT_CHUNKER,
-            ChunkingStrategyType.RECURSIVE_CHUNKER,
-            ChunkingStrategyType.SEMANTIC_CHUNKER,
-        ]
-
-    @classmethod
-    def get_supported_content_types(self) -> List[ContentType]:
-        return [ContentType.URL]
-
-    def read(
-        self, url: str, id: Optional[str] = None, name: Optional[str] = None, password: Optional[str] = None
-    ) -> List[Document]:
-        if not url:
-            raise ValueError("No url provided")
-
-        log_debug(f"Reading: {url}")
-
-        # Retry the request up to 3 times with exponential backoff
-        response = fetch_with_retry(url, proxy=self.proxy)
-
-        documents = self._create_documents(
-            url=url, text=response.text, content=response.content, id=id, name=name, password=password
-        )
-
-        if not self.chunk:
-            return documents
-
-        chunked_documents = []
-        for document in documents:
-            chunked_documents.append(self.chunk_document(document))
-        return [doc for sublist in chunked_documents for doc in sublist]
-
-    async def async_read(
-        self, url: str, id: Optional[str] = None, name: Optional[str] = None, password: Optional[str] = None
-    ) -> List[Document]:
-        """Async version of read method"""
-        if not url:
-            raise ValueError("No url provided")
-
-        log_debug(f"Reading async: {url}")
-        client_args = {"proxy": self.proxy} if self.proxy else {}
-        async with httpx.AsyncClient(**client_args) as client:  # type: ignore
-            response = await async_fetch_with_retry(url, client=client)
-
-        documents = self._create_documents(
-            url=url, text=response.text, content=response.content, id=id, name=name, password=password
-        )
-
-        if not self.chunk:
-            return documents
-
-        return await self.chunk_documents_async(documents)
-
-    def _create_documents(
-        self,
-        url: str,
-        text: str,
-        content: bytes,
-        id: Optional[str] = None,
-        name: Optional[str] = None,
-        password: Optional[str] = None,
-    ) -> List[Document]:
-        """Helper method to create a document from URL content"""
-
-        # Determine file extension from URL
-        parsed_url = urlparse(url)
-        url_path = Path(parsed_url.path)  # type: ignore
-        file_extension = url_path.suffix.lower()
-
-        # Read the document using the appropriate reader
-        if file_extension == ".csv":
-            filename = basename(parsed_url.path) or "data.csv"
-            return CSVReader().read(file=BytesIO(content), name=filename)
-        elif file_extension == ".pdf":
-            if password:
-                return PDFReader().read(pdf=BytesIO(content), name=name, password=password)
-            else:
-                return PDFReader().read(pdf=BytesIO(content), name=name)
-        else:
-            doc_name = name or parsed_url.path.strip("/").replace("/", "_").replace(" ", "_")
-            if not doc_name:
-                doc_name = parsed_url.netloc
-            if not doc_name:
-                doc_name = url
-
-        return [
-            Document(
-                name=doc_name,
-                id=id or doc_name,
-                meta_data={"url": url},
-                content=text,
-                size=len(text),
-            )
-        ]

agno/tools/googlesearch.py

@@ -1,98 +0,0 @@
-import json
-from typing import Any, Dict, List, Optional
-
-from agno.tools import Toolkit
-from agno.utils.log import log_debug
-
-try:
-    from googlesearch import search
-except ImportError:
-    raise ImportError("`googlesearch-python` not installed. Please install using `pip install googlesearch-python`")
-
-try:
-    from pycountry import pycountry
-except ImportError:
-    raise ImportError("`pycountry` not installed. Please install using `pip install pycountry`")
-
-
-class GoogleSearchTools(Toolkit):
-    """
-    GoogleSearch is a Python library for searching Google easily.
-    It uses requests and BeautifulSoup4 to scrape Google.
-
-    Args:
-        fixed_max_results (Optional[int]): A fixed number of maximum results.
-        fixed_language (Optional[str]): Language of the search results.
-        headers (Optional[Any]): Custom headers for the request.
-        proxy (Optional[str]): Proxy settings for the request.
-        timeout (Optional[int]): Timeout for the request, default is 10 seconds.
-        cache_results (bool): Enable caching of search results.
-        cache_ttl (int): Time-to-live for cached results in seconds.
-        cache_dir (Optional[str]): Directory to store cache files.
-        enable_google_search (bool): Enable Google search.
-        all (bool): Enable all tools.
-    """
-
-    def __init__(
-        self,
-        fixed_max_results: Optional[int] = None,
-        fixed_language: Optional[str] = None,
-        headers: Optional[Any] = None,
-        proxy: Optional[str] = None,
-        timeout: Optional[int] = 10,
-        enable_google_search: bool = True,
-        all: bool = False,
-        **kwargs,
-    ):
-        self.fixed_max_results: Optional[int] = fixed_max_results
-        self.fixed_language: Optional[str] = fixed_language
-        self.headers: Optional[Any] = headers
-        self.proxy: Optional[str] = proxy
-        self.timeout: Optional[int] = timeout
-
-        tools = []
-        if all or enable_google_search:
-            tools.append(self.google_search)
-
-        super().__init__(name="google_search_tools", tools=tools, **kwargs)
-
-    def google_search(self, query: str, max_results: int = 5, language: str = "en") -> str:
-        """
-        Use this function to search Google for a specified query.
-
-        Args:
-            query (str): The query to search for.
-            max_results (int, optional): The maximum number of results to return. Default is 5.
-            language (str, optional): The language of the search results. Default is "en".
-
-        Returns:
-            str: A JSON formatted string containing the search results.
-        """
-        max_results = self.fixed_max_results or max_results
-        language = self.fixed_language or language
-
-        # Resolve language to ISO 639-1 code if needed
-        if len(language) != 2:
-            _language = pycountry.languages.lookup(language)
-            if _language:
-                language = _language.alpha_2
-            else:
-                language = "en"
-
-        log_debug(f"Searching Google [{language}] for: {query}")
-
-        # Perform Google search using the googlesearch-python package
-        results = list(search(query, num=max_results, lang=language))
-
-        # Collect the search results
-        res: List[Dict[str, str]] = []
-        for result in results:
-            res.append(
-                {
-                    "title": result.title,
-                    "url": result.url,
-                    "description": result.description,
-                }
-            )
-
-        return json.dumps(res, indent=2)