agno 2.0.0rc1__py3-none-any.whl → 2.0.1__py3-none-any.whl

agno/knowledge/knowledge.py

@@ -5,16 +5,21 @@ import time
 from dataclasses import dataclass
 from enum import Enum
 from functools import cached_property
+from io import BytesIO
+from os.path import basename
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Set, Tuple, Union, cast, overload
 from uuid import uuid4
 
+from httpx import AsyncClient
+
 from agno.db.base import BaseDb
 from agno.db.schemas.knowledge import KnowledgeRow
 from agno.knowledge.content import Content, ContentAuth, ContentStatus, FileData
 from agno.knowledge.document import Document
 from agno.knowledge.reader import Reader, ReaderFactory
 from agno.knowledge.remote_content.remote_content import GCSContent, RemoteContent, S3Content
+from agno.utils.http import async_fetch_with_retry
 from agno.utils.log import log_debug, log_error, log_info, log_warning
 from agno.vectordb import VectorDb
 
@@ -421,20 +426,31 @@ class Knowledge:
         upsert: bool,
         skip_if_exists: bool,
     ):
+        """Load the content in the contextual URL
+
+        1. Set content hash
+        2. Validate the URL
+        3. Read the content
+        4. Prepare and insert the content in the vector database
+        """
         log_info(f"Adding content from URL {content.url}")
         content.file_type = "url"
 
+        if not content.url:
+            raise ValueError("No url provided")
+
         if self.vector_db.__class__.__name__ == "LightRag":
             await self._process_lightrag_content(content, KnowledgeContentOrigin.URL)
             return
 
+        # 1. Set content hash
         content.content_hash = self._build_content_hash(content)
         if self.vector_db and self.vector_db.content_hash_exists(content.content_hash) and skip_if_exists:
             log_info(f"Content {content.content_hash} already exists, skipping")
             return
         self._add_to_contents_db(content)
 
-        # Validate URL
+        # 2. Validate URL
         try:
             from urllib.parse import urlparse
 
@@ -450,61 +466,47 @@ class Knowledge:
             self._update_content(content)
             log_warning(f"Invalid URL: {content.url} - {str(e)}")
 
-        # Determine file type from URL
-        url_path = Path(parsed_url.path)  # type: ignore
-        file_extension = url_path.suffix.lower()
-        read_documents = []
-        try:
-            if content.url.endswith("llms-full.txt") or content.url.endswith("llms.txt"):  # type: ignore
-                log_info("Detected llms, using url reader")
-                reader = content.reader or self.url_reader
-                if reader is not None:
-                    # TODO: We will refactor this to eventually pass authorization to all readers
-                    import inspect
+        # 3. Fetch and load content
+        async with AsyncClient() as client:
+            response = await async_fetch_with_retry(content.url, client=client)
+        bytes_content = BytesIO(response.content)
 
-                    read_signature = inspect.signature(reader.read)
-                    if "password" in read_signature.parameters and content.auth and content.auth.password:
-                        read_documents = reader.read(content.url, name=content.name, password=content.auth.password)
-                    else:
-                        read_documents = reader.read(content.url, name=content.name)
-
-            elif file_extension and file_extension is not None:
-                log_info(f"Detected file type: {file_extension} from URL: {content.url}")
-                if content.reader:
-                    reader = content.reader
-                else:
-                    reader = self._select_url_file_reader(file_extension)
-                if reader is not None:
-                    log_info(f"Selected reader: {reader.__class__.__name__}")
-                    # TODO: We will refactor this to eventually pass authorization to all readers
-                    import inspect
-
-                    read_signature = inspect.signature(reader.read)
-                    if "password" in read_signature.parameters and content.auth and content.auth.password:
-                        read_documents = reader.read(content.url, name=content.name, password=content.auth.password)
-                    else:
-                        read_documents = reader.read(content.url, name=content.name)
-                else:
-                    log_info(f"No reader found for file extension: {file_extension}")
+        # 4. Select reader
+        # If a reader was provided by the user, use it
+        reader = content.reader
+        name = content.name if content.name else content.url
+        # Else select based on file extension
+        if reader is None:
+            url_path = Path(parsed_url.path)
+            file_extension = url_path.suffix.lower()
+            if file_extension == ".csv":
+                name = basename(parsed_url.path) or "data.csv"
+                reader = self.csv_reader
+            elif file_extension == ".pdf":
+                reader = self.pdf_reader
+            elif file_extension == ".docx":
+                reader = self.docx_reader
+            elif file_extension == ".json":
+                reader = self.json_reader
+            elif file_extension == ".markdown":
+                reader = self.markdown_reader
             else:
-                log_info(f"No file extension found for URL: {content.url}, determining website type")
-                if content.reader:
-                    reader = content.reader
-                else:
-                    reader = self._select_url_reader(content.url)  # type: ignore
-                if reader is not None:
-                    log_info(f"Selected reader: {reader.__class__.__name__}")
-                    # TODO: We will refactor this to eventually pass authorization to all readers
-                    import inspect
+                reader = self.text_reader
 
-                    read_signature = inspect.signature(reader.read)
-                    if "password" in read_signature.parameters and content.auth and content.auth.password:
-                        read_documents = reader.read(content.url, name=content.name, password=content.auth.password)
-                    else:
-                        read_documents = reader.read(content.url, name=content.name)
+        # 5. Read content
+        try:
+            read_documents = []
+            if reader is not None:
+                # TODO: We will refactor this to eventually pass authorization to all readers
+                import inspect
+
+                read_signature = inspect.signature(reader.read)
+                if reader.__class__.__name__ == "YouTubeReader":
+                    read_documents = reader.read(content.url, name=name)
+                elif "password" in read_signature.parameters and content.auth and content.auth.password:
+                    read_documents = reader.read(bytes_content, name=name, password=content.auth.password)
                 else:
-                    log_info(f"No reader found for URL: {content.url}")
-
+                    read_documents = reader.read(bytes_content, name=name)
         except Exception as e:
             log_error(f"Error reading URL: {content.url} - {str(e)}")
             content.status = ContentStatus.FAILED
@@ -512,13 +514,17 @@ class Knowledge:
             self._update_content(content)
             return
 
+        # 6. Chunk documents if needed
+        if reader and not reader.chunk:
+            read_documents = await reader.chunk_documents_async(read_documents)
+
+        # 7. Prepare and insert the content in the vector database
         file_size = 0
         if read_documents:
             for read_document in read_documents:
                 if read_document.size:
                     file_size += read_document.size
                 read_document.content_id = content.id
-
         await self._handle_vector_db_insert(content, read_documents, upsert)
 
     async def _load_from_content(
@@ -699,21 +705,23 @@ class Knowledge:
             log_warning(f"Unsupported remote content type: {type(remote_content)}")
 
     async def _load_from_s3(self, content: Content, upsert: bool, skip_if_exists: bool):
-        from agno.aws.resource.s3.object import S3Object  # type: ignore
-
-        if content.reader is None:
-            reader = self.s3_reader
-        else:
-            reader = content.reader
-
-        if reader is None:
-            log_warning("No reader provided for content")
-            return
+        """Load the contextual S3 content.
+
+        1. Identify objects to read
+        2. Setup Content object
+        3. Hash content and add it to the contents database
+        4. Select reader
+        5. Fetch and load the content
+        6. Read the content
+        7. Prepare and insert the content in the vector database
+        8. Remove temporary file if needed
+        """
+        from agno.cloud.aws.s3.object import S3Object
 
         remote_content: S3Content = cast(S3Content, content.remote_content)
 
+        # 1. Identify objects to read
         objects_to_read: List[S3Object] = []
-
         if remote_content.bucket is not None:
             if remote_content.key is not None:
                 _object = S3Object(bucket_name=remote_content.bucket.name, name=remote_content.key)
@@ -725,10 +733,11 @@ class Knowledge:
             else:
                 objects_to_read.extend(remote_content.bucket.get_objects())
 
-        for object in objects_to_read:
+        for s3_object in objects_to_read:
+            # 2. Setup Content object
             id = str(uuid4())
             content_name = content.name or ""
-            content_name += "_" + (object.name or "")
+            content_name += "_" + (s3_object.name or "")
             content_entry = Content(
                 id=id,
                 name=content_name,
@@ -738,63 +747,123 @@ class Knowledge:
                 file_type="s3",
             )
 
+            # 3. Hash content and add it to the contents database
             content_hash = self._build_content_hash(content_entry)
             if self.vector_db and self.vector_db.content_hash_exists(content_hash) and skip_if_exists:
                 log_info(f"Content {content_hash} already exists, skipping")
                 continue
-
             self._add_to_contents_db(content_entry)
 
-            read_documents = reader.read(content_entry.name, object)
+            # 4. Select reader
+            reader = content.reader
+            if reader is None:
+                if s3_object.uri.endswith(".pdf"):
+                    reader = self.pdf_reader
+                elif s3_object.uri.endswith(".csv"):
+                    reader = self.csv_reader
+                elif s3_object.uri.endswith(".docx"):
+                    reader = self.docx_reader
+                elif s3_object.uri.endswith(".json"):
+                    reader = self.json_reader
+                elif s3_object.uri.endswith(".markdown"):
+                    reader = self.markdown_reader
+                else:
+                    reader = self.text_reader
+            reader = cast(Reader, reader)
+
+            # 5. Fetch and load the content
+            temporary_file = None
+            obj_name = content_name or s3_object.name.split("/")[-1]
+            readable_content: Optional[Union[BytesIO, Path]] = None
+            if s3_object.uri.endswith(".pdf"):
+                readable_content = BytesIO(s3_object.get_resource().get()["Body"].read())
+            else:
+                temporary_file = Path("storage").joinpath(obj_name)
+                readable_content = temporary_file
+                s3_object.download(readable_content)  # type: ignore
+
+            # 6. Read the content
+            read_documents = reader.read(readable_content, name=obj_name)
 
+            # 7. Prepare and insert the content in the vector database
             for read_document in read_documents:
                 read_document.content_id = content.id
-
             await self._handle_vector_db_insert(content_entry, read_documents, upsert)
 
-    async def _load_from_gcs(self, content: Content, upsert: bool, skip_if_exists: bool):
-        if content.reader is None:
-            reader = self.gcs_reader
-        else:
-            reader = content.reader
-
-        if reader is None:
-            log_warning("No reader provided for content")
-            return
+            # 8. Remove temporary file if needed
+            if temporary_file:
+                temporary_file.unlink()
 
+    async def _load_from_gcs(self, content: Content, upsert: bool, skip_if_exists: bool):
+        """Load the contextual GCS content.
+
+        1. Identify objects to read
+        2. Setup Content object
+        3. Hash content and add it to the contents database
+        4. Select reader
+        5. Fetch and load the content
+        6. Read the content
+        7. Prepare and insert the content in the vector database
+        """
         remote_content: GCSContent = cast(GCSContent, content.remote_content)
-        objects_to_read = []
 
+        # 1. Identify objects to read
+        objects_to_read = []
         if remote_content.blob_name is not None:
-            objects_to_read.append(remote_content.bucket.blob(remote_content.blob_name))
+            objects_to_read.append(remote_content.bucket.blob(remote_content.blob_name))  # type: ignore
         elif remote_content.prefix is not None:
-            objects_to_read.extend(remote_content.bucket.list_blobs(prefix=remote_content.prefix))
+            objects_to_read.extend(remote_content.bucket.list_blobs(prefix=remote_content.prefix))  # type: ignore
         else:
-            objects_to_read.extend(remote_content.bucket.list_blobs())
+            objects_to_read.extend(remote_content.bucket.list_blobs())  # type: ignore
 
-        for object in objects_to_read:
+        for gcs_object in objects_to_read:
+            # 2. Setup Content object
             id = str(uuid4())
+            name = (content.name or "content") + "_" + gcs_object.name
             content_entry = Content(
                 id=id,
-                name=(content.name or "content") + "_" + object.name,
+                name=name,
                 description=content.description,
                 status=ContentStatus.PROCESSING,
                 metadata=content.metadata,
                 file_type="gcs",
             )
 
+            # 3. Hash content and add it to the contents database
             content_hash = self._build_content_hash(content_entry)
             if self.vector_db and self.vector_db.content_hash_exists(content_hash) and skip_if_exists:
                 log_info(f"Content {content_hash} already exists, skipping")
                 continue
 
+            # 4. Add it to the contents database
             self._add_to_contents_db(content_entry)
 
-            read_documents = reader.read(content_entry.name, object)
+            # 5. Select reader
+            reader = content.reader
+            if reader is None:
+                if gcs_object.name.endswith(".pdf"):
+                    reader = self.pdf_reader
+                elif gcs_object.name.endswith(".csv"):
+                    reader = self.csv_reader
+                elif gcs_object.name.endswith(".docx"):
+                    reader = self.docx_reader
+                elif gcs_object.name.endswith(".json"):
+                    reader = self.json_reader
+                elif gcs_object.name.endswith(".markdown"):
+                    reader = self.markdown_reader
+                else:
+                    reader = self.text_reader
+            reader = cast(Reader, reader)
+
+            # 5. Fetch and load the content
+            readable_content = BytesIO(gcs_object.download_as_bytes())
+
+            # 6. Read the content
+            read_documents = reader.read(readable_content, name=name)
 
+            # 7. Prepare and insert the content in the vector database
             for read_document in read_documents:
                 read_document.content_id = content.id
-
             await self._handle_vector_db_insert(content_entry, read_documents, upsert)
 
     async def _handle_vector_db_insert(self, content, read_documents, upsert):
@@ -1006,7 +1075,7 @@ class Knowledge:
         elif content_type == KnowledgeContentOrigin.URL:
             log_info(f"Uploading file to LightRAG from URL: {content.url}")
             try:
-                reader = self.url_reader
+                reader = content.reader or self.website_reader
                 if reader is None:
                     log_error("No URL reader available")
                     content.status = ContentStatus.FAILED
@@ -1354,14 +1423,6 @@ class Knowledge:
         log_info(f"Selecting reader for extension: {extension}")
         return ReaderFactory.get_reader_for_extension(extension)
 
-    def _select_url_reader(self, url: str) -> Reader:
-        """Select the appropriate reader for a URL."""
-        return ReaderFactory.get_reader_for_url(url)
-
-    def _select_url_file_reader(self, extension: str) -> Reader:
-        """Select the appropriate reader for a URL file extension."""
-        return ReaderFactory.get_reader_for_url_file(extension)
-
     def get_filters(self) -> List[str]:
         return [
             "filter_tag_1",
@@ -1484,32 +1545,7 @@ class Knowledge:
         """Firecrawl reader - lazy loaded via factory."""
         return self._get_reader("firecrawl")
 
-    @property
-    def url_reader(self) -> Optional[Reader]:
-        """URL reader - lazy loaded via factory."""
-        return self._get_reader("url")
-
-    @property
-    def pdf_url_reader(self) -> Optional[Reader]:
-        """PDF URL reader - lazy loaded via factory."""
-        return self._get_reader("pdf_url")
-
     @property
     def youtube_reader(self) -> Optional[Reader]:
         """YouTube reader - lazy loaded via factory."""
         return self._get_reader("youtube")
-
-    @property
-    def csv_url_reader(self) -> Optional[Reader]:
-        """CSV URL reader - lazy loaded via factory."""
-        return self._get_reader("csv_url")
-
-    @property
-    def s3_reader(self) -> Optional[Reader]:
-        """S3 reader - lazy loaded via factory."""
-        return self._get_reader("s3")
-
-    @property
-    def gcs_reader(self) -> Optional[Reader]:
-        """GCS reader - lazy loaded via factory."""
-        return self._get_reader("gcs")

agno/knowledge/reader/arxiv_reader.py

@@ -20,11 +20,11 @@ class ArxivReader(Reader):
     def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
         """Get the list of supported chunking strategies for Arxiv readers."""
         return [
-            ChunkingStrategyType.FIXED_SIZE_CHUNKING,
-            ChunkingStrategyType.AGENTIC_CHUNKING,
-            ChunkingStrategyType.DOCUMENT_CHUNKING,
-            ChunkingStrategyType.RECURSIVE_CHUNKING,
-            ChunkingStrategyType.SEMANTIC_CHUNKING,
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
         ]
 
     @classmethod

agno/knowledge/reader/csv_reader.py

@@ -1,14 +1,10 @@
 import asyncio
 import csv
 import io
-import os
 from pathlib import Path
 from typing import IO, Any, List, Optional, Union
-from urllib.parse import urlparse
 from uuid import uuid4
 
-from agno.utils.http import async_fetch_with_retry, fetch_with_retry
-
 try:
     import aiofiles
 except ImportError:
@@ -32,16 +28,16 @@ class CSVReader(Reader):
     def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
         """Get the list of supported chunking strategies for CSV readers."""
         return [
-            ChunkingStrategyType.ROW_CHUNKING,
-            ChunkingStrategyType.FIXED_SIZE_CHUNKING,
-            ChunkingStrategyType.AGENTIC_CHUNKING,
-            ChunkingStrategyType.DOCUMENT_CHUNKING,
-            ChunkingStrategyType.RECURSIVE_CHUNKING,
+            ChunkingStrategyType.ROW_CHUNKER,
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
         ]
 
     @classmethod
     def get_supported_content_types(self) -> List[ContentType]:
-        return [ContentType.FILE, ContentType.URL, ContentType.CSV, ContentType.XLSX, ContentType.XLS]
+        return [ContentType.CSV, ContentType.XLSX, ContentType.XLS]
 
     def read(
         self, file: Union[Path, IO[Any]], delimiter: str = ",", quotechar: str = '"', name: Optional[str] = None
@@ -168,70 +164,3 @@ class CSVReader(Reader):
                 f"Error reading async: {getattr(file, 'name', str(file)) if isinstance(file, IO) else file}: {e}"
             )
             return []
-
-
-class CSVUrlReader(Reader):
-    """Reader for CSV files"""
-
-    def __init__(
-        self, chunking_strategy: Optional[ChunkingStrategy] = RowChunking(), proxy: Optional[str] = None, **kwargs
-    ):
-        super().__init__(chunking_strategy=chunking_strategy, **kwargs)
-        self.proxy = proxy
-
-    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
-        """Get the list of supported chunking strategies for CSV URL readers."""
-        return [
-            ChunkingStrategyType.ROW_CHUNKING,
-            ChunkingStrategyType.SEMANTIC_CHUNKING,
-            ChunkingStrategyType.FIXED_SIZE_CHUNKING,
-            ChunkingStrategyType.AGENTIC_CHUNKING,
-            ChunkingStrategyType.DOCUMENT_CHUNKING,
-        ]
-
-    def get_supported_content_types(self) -> List[ContentType]:
-        return [ContentType.URL]
-
-    def read(self, url: str, name: Optional[str] = None) -> List[Document]:
-        if not url:
-            raise ValueError("No URL provided")
-
-        logger.info(f"Reading: {url}")
-        # Retry the request up to 3 times with exponential backoff
-        response = fetch_with_retry(url, proxy=self.proxy)
-
-        parsed_url = urlparse(url)
-        filename = os.path.basename(parsed_url.path) or "data.csv"
-
-        file_obj = io.BytesIO(response.content)
-        file_obj.name = filename
-        documents = CSVReader().read(file=file_obj, name=name)
-
-        file_obj.close()
-
-        return documents
-
-    async def async_read(self, url: str, name: Optional[str] = None) -> List[Document]:
-        if not url:
-            raise ValueError("No URL provided")
-
-        import httpx
-
-        logger.info(f"Reading async: {url}")
-
-        client_args = {"proxy": self.proxy} if self.proxy else {}
-        async with httpx.AsyncClient(**client_args) as client:  # type: ignore
-            response = await async_fetch_with_retry(url, client=client)
-
-            parsed_url = urlparse(url)
-            filename = os.path.basename(parsed_url.path) or "data.csv"
-
-            file_obj = io.BytesIO(response.content)
-            file_obj.name = filename
-
-            # Use the async version of CSVReader
-            documents = await CSVReader().async_read(file=file_obj, name=name)
-
-            file_obj.close()
-
-            return documents

agno/knowledge/reader/docx_reader.py

@@ -26,11 +26,11 @@ class DocxReader(Reader):
     def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
         """Get the list of supported chunking strategies for DOCX readers."""
         return [
-            ChunkingStrategyType.DOCUMENT_CHUNKING,
-            ChunkingStrategyType.FIXED_SIZE_CHUNKING,
-            ChunkingStrategyType.SEMANTIC_CHUNKING,
-            ChunkingStrategyType.AGENTIC_CHUNKING,
-            ChunkingStrategyType.RECURSIVE_CHUNKING,
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
         ]
 
     @classmethod

agno/knowledge/reader/firecrawl_reader.py

@@ -46,11 +46,11 @@ class FirecrawlReader(Reader):
     def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
         """Get the list of supported chunking strategies for Firecrawl readers."""
         return [
-            ChunkingStrategyType.SEMANTIC_CHUNKING,
-            ChunkingStrategyType.FIXED_SIZE_CHUNKING,
-            ChunkingStrategyType.AGENTIC_CHUNKING,
-            ChunkingStrategyType.DOCUMENT_CHUNKING,
-            ChunkingStrategyType.RECURSIVE_CHUNKING,
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
         ]
 
     @classmethod

agno/knowledge/reader/json_reader.py

@@ -25,11 +25,11 @@ class JSONReader(Reader):
     def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
         """Get the list of supported chunking strategies for JSON readers."""
         return [
-            ChunkingStrategyType.FIXED_SIZE_CHUNKING,
-            ChunkingStrategyType.AGENTIC_CHUNKING,
-            ChunkingStrategyType.DOCUMENT_CHUNKING,
-            ChunkingStrategyType.RECURSIVE_CHUNKING,
-            ChunkingStrategyType.SEMANTIC_CHUNKING,
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
         ]
 
     @classmethod