agno 2.0.0rc2__py3-none-any.whl → 2.3.0__py3-none-any.whl

agno/knowledge/reader/text_reader.py

@@ -8,7 +8,7 @@ from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyT
 from agno.knowledge.document.base import Document
 from agno.knowledge.reader.base import Reader
 from agno.knowledge.types import ContentType
-from agno.utils.log import log_info, logger
+from agno.utils.log import log_debug, log_error, log_warning
 
 
 class TextReader(Reader):
@@ -37,14 +37,14 @@ class TextReader(Reader):
             if isinstance(file, Path):
                 if not file.exists():
                     raise FileNotFoundError(f"Could not find file: {file}")
-                log_info(f"Reading: {file}")
+                log_debug(f"Reading: {file}")
                 file_name = name or file.stem
-                file_contents = file.read_text("utf-8")
+                file_contents = file.read_text(self.encoding or "utf-8")
             else:
                 file_name = name or file.name.split(".")[0]
-                log_info(f"Reading uploaded file: {file_name}")
+                log_debug(f"Reading uploaded file: {file_name}")
                 file.seek(0)
-                file_contents = file.read().decode("utf-8")
+                file_contents = file.read().decode(self.encoding or "utf-8")
 
             documents = [
                 Document(
@@ -60,7 +60,7 @@ class TextReader(Reader):
                 return chunked_documents
             return documents
         except Exception as e:
-            logger.error(f"Error reading: {file}: {e}")
+            log_error(f"Error reading: {file}: {e}")
             return []
 
     async def async_read(self, file: Union[Path, IO[Any]], name: Optional[str] = None) -> List[Document]:
@@ -69,22 +69,22 @@ class TextReader(Reader):
                 if not file.exists():
                     raise FileNotFoundError(f"Could not find file: {file}")
 
-                log_info(f"Reading asynchronously: {file}")
+                log_debug(f"Reading asynchronously: {file}")
                 file_name = name or file.stem
 
                 try:
                     import aiofiles
 
-                    async with aiofiles.open(file, "r", encoding="utf-8") as f:
+                    async with aiofiles.open(file, "r", encoding=self.encoding or "utf-8") as f:
                         file_contents = await f.read()
                 except ImportError:
-                    logger.warning("aiofiles not installed, using synchronous file I/O")
-                    file_contents = file.read_text("utf-8")
+                    log_warning("aiofiles not installed, using synchronous file I/O")
+                    file_contents = file.read_text(self.encoding or "utf-8")
             else:
-                log_info(f"Reading uploaded file asynchronously: {file.name}")
+                log_debug(f"Reading uploaded file asynchronously: {file.name}")
                 file_name = name or file.name.split(".")[0]
                 file.seek(0)
-                file_contents = file.read().decode("utf-8")
+                file_contents = file.read().decode(self.encoding or "utf-8")
 
             document = Document(
                 name=file_name,
@@ -96,7 +96,7 @@ class TextReader(Reader):
                 return await self._async_chunk_document(document)
             return [document]
         except Exception as e:
-            logger.error(f"Error reading asynchronously: {file}: {e}")
+            log_error(f"Error reading asynchronously: {file}: {e}")
             return []
 
     async def _async_chunk_document(self, document: Document) -> List[Document]:

agno/knowledge/reader/web_search_reader.py

@@ -37,7 +37,7 @@ class WebSearchReader(Reader):
     user_agent: str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
 
     # Search engine configuration
-    search_engine: Literal["duckduckgo", "google"] = "duckduckgo"
+    search_engine: Literal["duckduckgo"] = "duckduckgo"
     search_delay: float = 3.0  # Delay between search requests
     max_search_retries: int = 2  # Retries for search operations
 
@@ -96,7 +96,7 @@ class WebSearchReader(Reader):
                     results.append(
                         {
                             "title": result.get("title", ""),
-                            "url": result.get("link", ""),
+                            "url": result.get("href", ""),
                             "description": result.get("body", ""),
                         }
                     )
@@ -121,51 +121,10 @@ class WebSearchReader(Reader):
                     return []
         return []
 
-    def _perform_google_search(self, query: str) -> List[Dict[str, str]]:
-        """Perform web search using Google (requires googlesearch-python)"""
-        log_debug(f"Performing Google search for: {query}")
-
-        try:
-            from googlesearch import search
-        except ImportError:
-            logger.error("Google search requires 'googlesearch-python'. Install with: pip install googlesearch-python")
-            return []
-
-        for attempt in range(self.max_search_retries):
-            try:
-                self._respect_rate_limits()
-
-                results = []
-                search_results = search(query, num_results=self.max_results, stop=self.max_results)
-
-                for result in search_results:
-                    results.append(
-                        {
-                            "title": getattr(result, "title", ""),
-                            "url": getattr(result, "url", ""),
-                            "description": getattr(result, "description", ""),
-                        }
-                    )
-
-                log_debug(f"Found {len(results)} Google search results")
-                return results
-
-            except Exception as e:
-                logger.warning(f"Google search attempt {attempt + 1} failed: {e}")
-                if attempt < self.max_search_retries - 1:
-                    time.sleep(self.search_delay)
-                else:
-                    logger.error(f"All Google search attempts failed: {e}")
-                    return []
-
-        return []
-
     def _perform_web_search(self, query: str) -> List[Dict[str, str]]:
         """Perform web search using the configured search engine"""
         if self.search_engine == "duckduckgo":
             return self._perform_duckduckgo_search(query)
-        elif self.search_engine == "google":
-            return self._perform_google_search(query)
         else:
             logger.error(f"Unsupported search engine: {self.search_engine}")
             return []

agno/knowledge/reader/website_reader.py

@@ -12,7 +12,7 @@ from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyT
 from agno.knowledge.document.base import Document
 from agno.knowledge.reader.base import Reader
 from agno.knowledge.types import ContentType
-from agno.utils.log import log_debug, logger
+from agno.utils.log import log_debug, log_error, log_warning
 
 try:
     from bs4 import BeautifulSoup, Tag  # noqa: F401
@@ -106,23 +106,36 @@ class WebsiteReader(Reader):
             """
             Check if the tag matches any of the relevant tags or class names
             """
-            if tag.name in ["article", "main"]:
+            if not isinstance(tag, Tag):
+                return False
+
+            if tag.name in ["article", "main", "section"]:
+                return True
+
+            classes_attr = tag.get("class")
+            classes: List[str] = classes_attr if isinstance(classes_attr, list) else []
+            content_classes = ["content", "main-content", "post-content", "entry-content", "article-body"]
+            if any(cls in content_classes for cls in classes):
                 return True
-            if any(cls in ["content", "main-content", "post-content"] for cls in tag.get("class", [])):  # type: ignore
+
+            # Check for common content IDs
+            tag_id = tag.get("id", "")
+            if tag_id in ["content", "main", "article"]:
                 return True
+
             return False
 
-        # Use a single call to 'find' with a custom function to match tags or classes
+        # Try to find main content element
         element = soup.find(match)
-        if element:
+        if element and hasattr(element, "find_all"):
+            # Remove common unwanted elements from the found content
+            for unwanted in element.find_all(["script", "style", "nav", "header", "footer"]):
+                unwanted.decompose()
             return element.get_text(strip=True, separator=" ")
 
-        # If we only have a div without specific content classes, return empty string
-        if soup.find("div") and not any(
-            soup.find(class_=class_name) for class_name in ["content", "main-content", "post-content"]
-        ):
-            return ""
-
+        # Fallback: get full page content
+        for unwanted in soup.find_all(["script", "style", "nav", "header", "footer"]):
+            unwanted.decompose()
         return soup.get_text(strip=True, separator=" ")
 
     def crawl(self, url: str, starting_depth: int = 1) -> Dict[str, str]:
@@ -164,7 +177,7 @@ class WebsiteReader(Reader):
             if (
                 current_url in self._visited
                 or not urlparse(current_url).netloc.endswith(primary_domain)
-                or current_depth > self.max_depth
+                or (current_depth > self.max_depth and current_url != url)
                 or num_links >= self.max_links
             ):
                 continue
@@ -174,13 +187,14 @@ class WebsiteReader(Reader):
 
             try:
                 log_debug(f"Crawling: {current_url}")
+
                 response = (
-                    httpx.get(current_url, timeout=self.timeout, proxy=self.proxy)
+                    httpx.get(current_url, timeout=self.timeout, proxy=self.proxy, follow_redirects=True)
                     if self.proxy
-                    else httpx.get(current_url, timeout=self.timeout)
+                    else httpx.get(current_url, timeout=self.timeout, follow_redirects=True)
                 )
-
                 response.raise_for_status()
+
                 soup = BeautifulSoup(response.content, "html.parser")
 
                 # Extract main content
@@ -213,19 +227,23 @@ class WebsiteReader(Reader):
 
             except httpx.HTTPStatusError as e:
                 # Log HTTP status errors but continue crawling other pages
-                logger.warning(f"HTTP status error while crawling {current_url}: {e}")
-                # For the initial URL, we should raise the error
-                if current_url == url and not crawler_result:
+                # Skip redirect errors (3xx) as they should be handled by follow_redirects
+                if e.response.status_code >= 300 and e.response.status_code < 400:
+                    log_debug(f"Redirect encountered for {current_url}, skipping: {e}")
+                else:
+                    log_warning(f"HTTP status error while crawling {current_url}: {e}")
+                # For the initial URL, we should raise the error only if it's not a redirect
+                if current_url == url and not crawler_result and not (300 <= e.response.status_code < 400):
                     raise
             except httpx.RequestError as e:
                 # Log request errors but continue crawling other pages
-                logger.warning(f"Request error while crawling {current_url}: {e}")
+                log_warning(f"Request error while crawling {current_url}: {e}")
                 # For the initial URL, we should raise the error
                 if current_url == url and not crawler_result:
                     raise
             except Exception as e:
                 # Log other exceptions but continue crawling other pages
-                logger.warning(f"Failed to crawl {current_url}: {e}")
+                log_warning(f"Failed to crawl {current_url}: {e}")
                 # For the initial URL, we should raise the error
                 if current_url == url and not crawler_result:
                     # Wrap non-HTTP exceptions in a RequestError
@@ -314,19 +332,19 @@ class WebsiteReader(Reader):
 
                 except httpx.HTTPStatusError as e:
                     # Log HTTP status errors but continue crawling other pages
-                    logger.warning(f"HTTP status error while crawling asynchronously {current_url}: {e}")
+                    log_warning(f"HTTP status error while crawling asynchronously {current_url}: {e}")
                     # For the initial URL, we should raise the error
                     if current_url == url and not crawler_result:
                         raise
                 except httpx.RequestError as e:
                     # Log request errors but continue crawling other pages
-                    logger.warning(f"Request error while crawling asynchronously {current_url}: {e}")
+                    log_warning(f"Request error while crawling asynchronously {current_url}: {e}")
                     # For the initial URL, we should raise the error
                     if current_url == url and not crawler_result:
                         raise
                 except Exception as e:
                     # Log other exceptions but continue crawling other pages
-                    logger.warning(f"Failed to crawl asynchronously {current_url}: {e}")
+                    log_warning(f"Failed to crawl asynchronously {current_url}: {e}")
                     # For the initial URL, we should raise the error
                     if current_url == url and not crawler_result:
                         # Wrap non-HTTP exceptions in a RequestError
@@ -380,7 +398,7 @@ class WebsiteReader(Reader):
                     )
             return documents
         except (httpx.HTTPStatusError, httpx.RequestError) as e:
-            logger.error(f"Error reading website {url}: {e}")
+            log_error(f"Error reading website {url}: {e}")
             raise
 
     async def async_read(self, url: str, name: Optional[str] = None) -> List[Document]:
@@ -433,5 +451,5 @@ class WebsiteReader(Reader):
 
             return documents
         except (httpx.HTTPStatusError, httpx.RequestError) as e:
-            logger.error(f"Error reading website asynchronously {url}: {e}")
+            log_error(f"Error reading website asynchronously {url}: {e}")
             raise

agno/knowledge/reranker/__init__.py

@@ -0,0 +1,3 @@
+from agno.knowledge.reranker.base import Reranker
+
+__all__ = ["Reranker"]

agno/knowledge/types.py

@@ -1,4 +1,7 @@
 from enum import Enum
+from typing import Any
+
+from pydantic import BaseModel
 
 
 class ContentType(str, Enum):
@@ -17,6 +20,7 @@ class ContentType(str, Enum):
     MARKDOWN = ".md"
     DOCX = ".docx"
     DOC = ".doc"
+    PPTX = ".pptx"
     JSON = ".json"
 
     # Spreadsheet file extensions
@@ -28,3 +32,8 @@ class ContentType(str, Enum):
 def get_content_type_enum(content_type_str: str) -> ContentType:
     """Convert a content type string to ContentType enum."""
     return ContentType(content_type_str)
+
+
+class KnowledgeFilter(BaseModel):
+    key: str
+    value: Any

agno/knowledge/utils.py

@@ -129,12 +129,32 @@ def get_chunker_info(chunker_key: str) -> Dict:
             class_name = chunker_class.__name__
             docstring = chunker_class.__doc__ or f"{class_name} chunking strategy"
 
+            # Check class __init__ signature for chunk_size and overlap parameters
+            metadata = {}
+            import inspect
+
+            try:
+                sig = inspect.signature(chunker_class.__init__)
+                param_names = set(sig.parameters.keys())
+
+                # If class has chunk_size or max_chunk_size parameter, set default chunk_size
+                if "chunk_size" in param_names or "max_chunk_size" in param_names:
+                    metadata["chunk_size"] = 5000
+
+                # If class has overlap parameter, set default overlap
+                if "overlap" in param_names:
+                    metadata["chunk_overlap"] = 0
+            except Exception:
+                # If we can't inspect, skip metadata
+                pass
+
             return {
                 "key": chunker_key,
                 "class_name": class_name,
                 "name": chunker_key,
                 "description": docstring.strip(),
                 "strategy_type": strategy_type.value,
+                "metadata": metadata,
             }
         except ValueError:
             raise ValueError(f"Unknown chunker key: {chunker_key}")