agno 1.8.1__py3-none-any.whl → 2.0.0a1__py3-none-any.whl

agno/knowledge/reader/web_search_reader.py

@@ -0,0 +1,389 @@
+import asyncio
+import random
+import time
+from dataclasses import dataclass, field
+from typing import Dict, List, Literal, Optional, Set
+from urllib.parse import urlparse
+
+import httpx
+
+from agno.knowledge.chunking.semantic import SemanticChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.reader.url_reader import URLReader
+from agno.knowledge.types import ContentType
+from agno.utils.log import log_debug, logger
+
+try:
+    from bs4 import BeautifulSoup, Tag  # noqa: F401
+except ImportError:
+    raise ImportError("The `bs4` package is not installed. Please install it via `pip install beautifulsoup4`.")
+
+try:
+    from ddgs import DDGS
+except ImportError:
+    raise ImportError("The `ddgs` package is not installed. Please install it via `pip install ddgs`.")
+
+
+@dataclass
+class WebSearchReader(Reader):
+    """Reader that uses web search to find content for a given query"""
+
+    search_timeout: int = 10
+
+    request_timeout: int = 30
+    delay_between_requests: float = 2.0  # Increased default delay
+    max_retries: int = 3
+    user_agent: str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+
+    # Search engine configuration
+    search_engine: Literal["duckduckgo", "google"] = "duckduckgo"
+    search_delay: float = 3.0  # Delay between search requests
+    max_search_retries: int = 2  # Retries for search operations
+
+    # Rate limiting
+    rate_limit_delay: float = 5.0  # Delay when rate limited
+    exponential_backoff: bool = True
+
+    # Internal state
+    _visited_urls: Set[str] = field(default_factory=set)
+    _url_reader: Optional[URLReader] = None
+    _last_search_time: float = field(default=0.0, init=False)
+
+    # Override default chunking strategy
+    chunking_strategy: Optional[ChunkingStrategy] = SemanticChunking()
+
+    def __post_init__(self):
+        """Initialize the URL reader and chunking strategy after dataclass initialization"""
+        self._url_reader = URLReader()
+
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for Web Search readers."""
+        return [
+            ChunkingStrategyType.AGENTIC_CHUNKING,
+            ChunkingStrategyType.DOCUMENT_CHUNKING,
+            ChunkingStrategyType.RECURSIVE_CHUNKING,
+            ChunkingStrategyType.SEMANTIC_CHUNKING,
+            ChunkingStrategyType.FIXED_SIZE_CHUNKING,
+        ]
+
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.URL, ContentType.TEXT]
+
+    def _respect_rate_limits(self):
+        """Ensure we don't exceed rate limits"""
+        current_time = time.time()
+        time_since_last_search = current_time - self._last_search_time
+
+        if time_since_last_search < self.search_delay:
+            sleep_time = self.search_delay - time_since_last_search
+            log_debug(f"Rate limiting: sleeping for {sleep_time:.2f} seconds")
+            time.sleep(sleep_time)
+
+        self._last_search_time = time.time()
+
+    def _perform_duckduckgo_search(self, query: str) -> List[Dict[str, str]]:
+        """Perform web search using DuckDuckGo with rate limiting"""
+        log_debug(f"Performing DuckDuckGo search for: {query}")
+
+        for attempt in range(self.max_search_retries):
+            try:
+                self._respect_rate_limits()
+
+                ddgs = DDGS(timeout=self.search_timeout)
+                search_results = ddgs.text(query=query, max_results=self.max_results)
+
+                # Convert to list and extract relevant fields
+                results = []
+                for result in search_results:
+                    results.append(
+                        {
+                            "title": result.get("title", ""),
+                            "url": result.get("link", ""),
+                            "description": result.get("body", ""),
+                        }
+                    )
+
+                log_debug(f"Found {len(results)} search results")
+                return results
+
+            except Exception as e:
+                logger.warning(f"DuckDuckGo search attempt {attempt + 1} failed: {e}")
+                if "rate limit" in str(e).lower() or "429" in str(e):
+                    # Rate limited - wait longer
+                    wait_time = (
+                        self.rate_limit_delay * (2**attempt) if self.exponential_backoff else self.rate_limit_delay
+                    )
+                    logger.info(f"Rate limited, waiting {wait_time} seconds before retry")
+                    time.sleep(wait_time)
+                elif attempt < self.max_search_retries - 1:
+                    # Other error - shorter wait
+                    time.sleep(self.search_delay)
+                else:
+                    logger.error(f"All DuckDuckGo search attempts failed: {e}")
+                    return []
+        return []
+
+    def _perform_google_search(self, query: str) -> List[Dict[str, str]]:
+        """Perform web search using Google (requires googlesearch-python)"""
+        log_debug(f"Performing Google search for: {query}")
+
+        try:
+            from googlesearch import search
+        except ImportError:
+            logger.error("Google search requires 'googlesearch-python'. Install with: pip install googlesearch-python")
+            return []
+
+        for attempt in range(self.max_search_retries):
+            try:
+                self._respect_rate_limits()
+
+                results = []
+                search_results = search(query, num_results=self.max_results, stop=self.max_results)
+
+                for result in search_results:
+                    results.append(
+                        {
+                            "title": getattr(result, "title", ""),
+                            "url": getattr(result, "url", ""),
+                            "description": getattr(result, "description", ""),
+                        }
+                    )
+
+                log_debug(f"Found {len(results)} Google search results")
+                return results
+
+            except Exception as e:
+                logger.warning(f"Google search attempt {attempt + 1} failed: {e}")
+                if attempt < self.max_search_retries - 1:
+                    time.sleep(self.search_delay)
+                else:
+                    logger.error(f"All Google search attempts failed: {e}")
+                    return []
+
+        return []
+
+    def _perform_web_search(self, query: str) -> List[Dict[str, str]]:
+        """Perform web search using the configured search engine"""
+        if self.search_engine == "duckduckgo":
+            return self._perform_duckduckgo_search(query)
+        elif self.search_engine == "google":
+            return self._perform_google_search(query)
+        else:
+            logger.error(f"Unsupported search engine: {self.search_engine}")
+            return []
+
+    def _is_valid_url(self, url: str) -> bool:
+        """Check if URL is valid and not already visited"""
+        try:
+            parsed = urlparse(url)
+            return bool(parsed.scheme in ["http", "https"] and parsed.netloc and url not in self._visited_urls)
+        except Exception:
+            return False
+
+    def _extract_text_from_html(self, html_content: str, url: str) -> str:
+        """Extract clean text content from HTML"""
+        try:
+            soup = BeautifulSoup(html_content, "html.parser")
+
+            # Remove script and style elements
+            for script in soup(["script", "style"]):
+                script.decompose()
+
+            # Get text content
+            text = soup.get_text()
+
+            # Clean up whitespace
+            lines = (line.strip() for line in text.splitlines())
+            chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
+            text = " ".join(chunk for chunk in chunks if chunk)
+
+            return text
+
+        except Exception as e:
+            logger.warning(f"Error extracting text from {url}: {e}")
+            return html_content
+
+    def _fetch_url_content(self, url: str) -> Optional[str]:
+        """Fetch content from a URL with retry logic"""
+        headers = {"User-Agent": self.user_agent}
+
+        for attempt in range(self.max_retries):
+            try:
+                response = httpx.get(url, headers=headers, timeout=self.request_timeout, follow_redirects=True)
+                response.raise_for_status()
+
+                # Check if it's HTML content
+                content_type = response.headers.get("content-type", "").lower()
+                if "text/html" in content_type:
+                    return self._extract_text_from_html(response.text, url)
+                else:
+                    # For non-HTML content, return as-is
+                    return response.text
+
+            except Exception as e:
+                logger.warning(f"Attempt {attempt + 1} failed for {url}: {e}")
+                if attempt < self.max_retries - 1:
+                    time.sleep(random.uniform(1, 3))  # Random delay between retries
+                continue
+
+        logger.error(f"Failed to fetch content from {url} after {self.max_retries} attempts")
+        return None
+
+    def _create_document_from_url(self, url: str, content: str, search_result: Dict[str, str]) -> Document:
+        """Create a Document object from URL content and search result metadata"""
+        # Use the URL as the document ID
+        doc_id = url
+
+        # Use the search result title as the document name, fallback to URL
+        doc_name = search_result.get("title", urlparse(url).netloc)
+
+        # Create metadata with search information
+        meta_data = {
+            "url": url,
+            "search_title": search_result.get("title", ""),
+            "search_description": search_result.get("description", ""),
+            "source": "web_search",
+            "search_engine": self.search_engine,
+        }
+
+        return Document(id=doc_id, name=doc_name, content=content, meta_data=meta_data)
+
+    def read(self, query: str) -> List[Document]:
+        """Read content for a given query by performing web search and fetching content"""
+        if not query:
+            raise ValueError("Query cannot be empty")
+
+        log_debug(f"Starting web search reader for query: {query}")
+
+        # Perform web search
+        search_results = self._perform_web_search(query)
+        if not search_results:
+            logger.warning(f"No search results found for query: {query}")
+            return []
+
+        documents: List[Document] = []
+
+        for result in search_results:
+            url = result.get("url", "")
+
+            # Skip if URL is invalid or already visited
+            if not self._is_valid_url(url):
+                continue
+
+            # Mark URL as visited
+            self._visited_urls.add(url)
+
+            # Add delay between requests to be respectful
+            if len(documents) > 0:
+                time.sleep(self.delay_between_requests)
+
+            # Fetch content from URL
+            content = self._fetch_url_content(url)
+            if content is None:
+                continue
+
+            # Create document
+            document = self._create_document_from_url(url, content, result)
+
+            # Apply chunking if enabled
+            if self.chunk:
+                chunked_docs = self.chunk_document(document)
+                documents.extend(chunked_docs)
+            else:
+                documents.append(document)
+
+            # Stop if we've reached max_results
+            if len(documents) >= self.max_results:
+                break
+
+        log_debug(f"Created {len(documents)} documents from web search")
+        return documents
+
+    async def async_read(self, query: str) -> List[Document]:
+        """Asynchronously read content for a given query"""
+        if not query:
+            raise ValueError("Query cannot be empty")
+
+        log_debug(f"Starting async web search reader for query: {query}")
+
+        # Perform web search (synchronous operation)
+        search_results = self._perform_web_search(query)
+        if not search_results:
+            logger.warning(f"No search results found for query: {query}")
+            return []
+
+        # Create tasks for fetching content from each URL
+        async def fetch_url_async(result: Dict[str, str]) -> Optional[Document]:
+            url = result.get("url", "")
+
+            # Skip if URL is invalid or already visited
+            if not self._is_valid_url(url):
+                return None
+
+            # Mark URL as visited
+            self._visited_urls.add(url)
+
+            try:
+                # Use the URL reader for async fetching
+                if self._url_reader:
+                    docs = await self._url_reader.async_read(url)
+                    if docs:
+                        # Use the first document and add search metadata
+                        doc = docs[0]
+                        doc.meta_data.update(
+                            {
+                                "search_title": result.get("title", ""),
+                                "search_description": result.get("description", ""),
+                                "source": "web_search",
+                                "search_engine": self.search_engine,
+                            }
+                        )
+                        return doc
+
+                # Fallback to manual async fetching
+                headers = {"User-Agent": self.user_agent}
+                async with httpx.AsyncClient(timeout=self.request_timeout) as client:
+                    response = await client.get(url, headers=headers, follow_redirects=True)
+                    response.raise_for_status()
+
+                    content_type = response.headers.get("content-type", "").lower()
+                    if "text/html" in content_type:
+                        content = self._extract_text_from_html(response.text, url)
+                    else:
+                        content = response.text
+
+                    document = self._create_document_from_url(url, content, result)
+                    return document
+
+            except Exception as e:
+                logger.warning(f"Error fetching {url}: {e}")
+                return None
+
+        # Create tasks for all URLs
+        tasks = [fetch_url_async(result) for result in search_results]
+
+        # Execute all tasks concurrently with delays
+        documents = []
+        for i, task in enumerate(tasks):
+            if i > 0:  # Add delay between requests (except for the first one)
+                await asyncio.sleep(self.delay_between_requests)
+
+            doc = await task
+            if doc is not None:
+                # Apply chunking if enabled
+                if self.chunk:
+                    chunked_docs = await self.chunk_documents_async([doc])
+                    documents.extend(chunked_docs)
+                else:
+                    documents.append(doc)
+
+                # Stop if we've reached max_results
+                if len(documents) >= self.max_results:
+                    break
+
+        log_debug(f"Created {len(documents)} documents from async web search")
+        return documents

agno/{document → knowledge}/reader/website_reader.py

@@ -7,8 +7,11 @@ from urllib.parse import urljoin, urlparse
 
 import httpx
 
-from agno.document.base import Document
-from agno.document.reader.base import Reader
+from agno.knowledge.chunking.semantic import SemanticChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.types import ContentType
 from agno.utils.log import log_debug, logger
 
 try:
@@ -28,9 +31,15 @@ class WebsiteReader(Reader):
     _urls_to_crawl: List[Tuple[str, int]] = field(default_factory=list)
 
     def __init__(
-        self, max_depth: int = 3, max_links: int = 10, timeout: int = 10, proxy: Optional[str] = None, **kwargs
+        self,
+        chunking_strategy: Optional[ChunkingStrategy] = SemanticChunking(),
+        max_depth: int = 3,
+        max_links: int = 10,
+        timeout: int = 10,
+        proxy: Optional[str] = None,
+        **kwargs,
     ):
-        super().__init__(**kwargs)
+        super().__init__(chunking_strategy=chunking_strategy, **kwargs)
         self.max_depth = max_depth
         self.max_links = max_links
         self.proxy = proxy
@@ -39,6 +48,21 @@ class WebsiteReader(Reader):
         self._visited = set()
         self._urls_to_crawl = []
 
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for Website readers."""
+        return [
+            ChunkingStrategyType.AGENTIC_CHUNKING,
+            ChunkingStrategyType.DOCUMENT_CHUNKING,
+            ChunkingStrategyType.RECURSIVE_CHUNKING,
+            ChunkingStrategyType.SEMANTIC_CHUNKING,
+            ChunkingStrategyType.FIXED_SIZE_CHUNKING,
+        ]
+
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.URL]
+
     def delay(self, min_seconds=1, max_seconds=3):
         """
         Introduce a random delay.
@@ -316,7 +340,7 @@ class WebsiteReader(Reader):
 
         return crawler_result
 
-    def read(self, url: str) -> List[Document]:
+    def read(self, url: str, name: Optional[str] = None) -> List[Document]:
         """
         Reads a website and returns a list of documents.
 
@@ -338,7 +362,7 @@ class WebsiteReader(Reader):
                     documents.extend(
                         self.chunk_document(
                             Document(
-                                name=url,
+                                name=name or url,
                                 id=str(crawled_url),
                                 meta_data={"url": str(crawled_url)},
                                 content=crawled_content,
@@ -348,7 +372,7 @@ class WebsiteReader(Reader):
                 else:
                     documents.append(
                         Document(
-                            name=url,
+                            name=name or url,
                             id=str(crawled_url),
                             meta_data={"url": str(crawled_url)},
                             content=crawled_content,
@@ -359,7 +383,7 @@ class WebsiteReader(Reader):
             logger.error(f"Error reading website {url}: {e}")
             raise
 
-    async def async_read(self, url: str) -> List[Document]:
+    async def async_read(self, url: str, name: Optional[str] = None) -> List[Document]:
         """
         Asynchronously reads a website and returns a list of documents.
 
@@ -380,13 +404,16 @@ class WebsiteReader(Reader):
             async def process_document(crawled_url, crawled_content):
                 if self.chunk:
                     doc = Document(
-                        name=url, id=str(crawled_url), meta_data={"url": str(crawled_url)}, content=crawled_content
+                        name=name or url,
+                        id=str(crawled_url),
+                        meta_data={"url": str(crawled_url)},
+                        content=crawled_content,
                     )
                     return self.chunk_document(doc)
                 else:
                     return [
                         Document(
-                            name=url,
+                            name=name or url,
                             id=str(crawled_url),
                             meta_data={"url": str(crawled_url)},
                             content=crawled_content,

agno/knowledge/reader/wikipedia_reader.py

@@ -0,0 +1,59 @@
+from typing import List, Optional
+
+from agno.knowledge.chunking.fixed import FixedSizeChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
+from agno.knowledge.document import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.types import ContentType
+from agno.utils.log import log_debug, log_info
+
+try:
+    import wikipedia  # noqa: F401
+except ImportError:
+    raise ImportError("The `wikipedia` package is not installed. Please install it via `pip install wikipedia`.")
+
+
+class WikipediaReader(Reader):
+    auto_suggest: bool = True
+
+    def __init__(
+        self, chunking_strategy: Optional[ChunkingStrategy] = FixedSizeChunking(), auto_suggest: bool = True, **kwargs
+    ):
+        super().__init__(chunking_strategy=chunking_strategy, **kwargs)
+        self.auto_suggest = auto_suggest
+
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for Wikipedia readers."""
+        return [
+            ChunkingStrategyType.FIXED_SIZE_CHUNKING,
+            ChunkingStrategyType.AGENTIC_CHUNKING,
+            ChunkingStrategyType.DOCUMENT_CHUNKING,
+            ChunkingStrategyType.RECURSIVE_CHUNKING,
+            ChunkingStrategyType.SEMANTIC_CHUNKING,
+        ]
+
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.TOPIC]
+
+    def read(self, topic: str) -> List[Document]:
+        log_debug(f"Reading Wikipedia topic: {topic}")
+        summary = None
+        try:
+            summary = wikipedia.summary(topic, auto_suggest=self.auto_suggest)
+
+        except wikipedia.exceptions.PageError:
+            summary = None
+            log_info("PageError: Page not found.")
+
+        # Only create Document if we successfully got a summary
+        if summary:
+            return [
+                Document(
+                    name=topic,
+                    meta_data={"topic": topic},
+                    content=summary,
+                )
+            ]
+        return []

agno/knowledge/reader/youtube_reader.py

@@ -0,0 +1,78 @@
+import asyncio
+from typing import List, Optional
+
+from agno.knowledge.chunking.recursive import RecursiveChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.types import ContentType
+from agno.utils.log import log_debug, log_error, log_info
+
+try:
+    from youtube_transcript_api import YouTubeTranscriptApi
+except ImportError:
+    raise ImportError(
+        "`youtube_transcript_api` not installed. Please install it via `pip install youtube_transcript_api`."
+    )
+
+
+class YouTubeReader(Reader):
+    """Reader for YouTube video transcripts"""
+
+    def __init__(self, chunking_strategy: Optional[ChunkingStrategy] = RecursiveChunking(), **kwargs):
+        super().__init__(chunking_strategy=chunking_strategy, **kwargs)
+
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for YouTube readers."""
+        return [
+            ChunkingStrategyType.RECURSIVE_CHUNKING,
+            ChunkingStrategyType.AGENTIC_CHUNKING,
+            ChunkingStrategyType.DOCUMENT_CHUNKING,
+            ChunkingStrategyType.SEMANTIC_CHUNKING,
+            ChunkingStrategyType.FIXED_SIZE_CHUNKING,
+        ]
+
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.URL, ContentType.YOUTUBE]
+
+    def read(self, url: str, name: Optional[str] = None) -> List[Document]:
+        try:
+            # Extract video ID from URL
+            video_id = url.split("v=")[-1].split("&")[0]
+            log_info(f"Reading transcript for video: {video_id}")
+
+            # Get transcript
+            log_debug(f"Fetching transcript for video: {video_id}")
+            # Create an instance of YouTubeTranscriptApi
+            ytt_api = YouTubeTranscriptApi()
+            transcript_data = ytt_api.fetch(video_id)
+
+            # Combine transcript segments into full text
+            transcript_text = ""
+            for segment in transcript_data:
+                transcript_text += f"{segment.text} "
+
+            documents = [
+                Document(
+                    name=name or f"youtube_{video_id}",
+                    id=f"youtube_{video_id}",
+                    meta_data={"video_url": url, "video_id": video_id},
+                    content=transcript_text.strip(),
+                )
+            ]
+
+            if self.chunk:
+                chunked_documents = []
+                for document in documents:
+                    chunked_documents.extend(self.chunk_document(document))
+                return chunked_documents
+            return documents
+
+        except Exception as e:
+            log_error(f"Error reading transcript for {url}: {e}")
+            return []
+
+    async def async_read(self, url: str) -> List[Document]:
+        return await asyncio.get_event_loop().run_in_executor(None, self.read, url)