agno 0.1.2__py3-none-any.whl → 2.3.13__py3-none-any.whl

agno/knowledge/reader/field_labeled_csv_reader.py

@@ -0,0 +1,290 @@
+import asyncio
+import csv
+import io
+from pathlib import Path
+from typing import IO, Any, List, Optional, Union
+
+try:
+    import aiofiles
+except ImportError:
+    raise ImportError("`aiofiles` not installed. Please install it with `pip install aiofiles`")
+
+from agno.knowledge.chunking.strategy import ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.types import ContentType
+from agno.utils.log import log_debug, log_error, log_warning
+
+
+class FieldLabeledCSVReader(Reader):
+    """Reader for CSV files that converts each row to a field-labeled document."""
+
+    def __init__(
+        self,
+        chunk_title: Optional[Union[str, List[str]]] = None,
+        field_names: Optional[List[str]] = None,
+        format_headers: bool = True,
+        skip_empty_fields: bool = True,
+        **kwargs,
+    ):
+        super().__init__(chunk=False, chunking_strategy=None, **kwargs)
+        self.chunk_title = chunk_title
+        self.field_names = field_names or []
+        self.format_headers = format_headers
+        self.skip_empty_fields = skip_empty_fields
+
+    @classmethod
+    def get_supported_chunking_strategies(cls) -> List[ChunkingStrategyType]:
+        """Chunking is not supported - each row is already a logical document unit."""
+        return []
+
+    @classmethod
+    def get_supported_content_types(cls) -> List[ContentType]:
+        """Get the list of supported content types."""
+        return [ContentType.CSV, ContentType.XLSX, ContentType.XLS]
+
+    def _format_field_name(self, field_name: str) -> str:
+        """Format field name to be more readable."""
+        if not self.format_headers:
+            return field_name.strip()
+
+        # Replace underscores with spaces and title case
+        formatted = field_name.replace("_", " ").strip().title()
+        return formatted
+
+    def _get_title_for_entry(self, entry_index: int) -> Optional[str]:
+        """Get title for a specific entry."""
+        if self.chunk_title is None:
+            return None
+
+        if isinstance(self.chunk_title, str):
+            return self.chunk_title
+
+        if isinstance(self.chunk_title, list) and self.chunk_title:
+            return self.chunk_title[entry_index % len(self.chunk_title)]
+
+        return None
+
+    def _convert_row_to_labeled_text(self, headers: List[str], row: List[str], entry_index: int) -> str:
+        """
+        Convert a CSV row to field-labeled text format.
+
+        Args:
+            headers: Column headers
+            row: Data row values
+            entry_index: Index of this entry (for title rotation)
+
+        Returns:
+            Formatted text with field labels
+        """
+        lines = []
+
+        title = self._get_title_for_entry(entry_index)
+        if title:
+            lines.append(title)
+
+        for i, (header, value) in enumerate(zip(headers, row)):
+            clean_value = value.strip() if value else ""
+
+            if self.skip_empty_fields and not clean_value:
+                continue
+
+            if self.field_names and i < len(self.field_names):
+                field_name = self.field_names[i]
+            else:
+                field_name = self._format_field_name(header)
+
+            lines.append(f"{field_name}: {clean_value}")
+
+        return "\n".join(lines)
+
+    def read(
+        self, file: Union[Path, IO[Any]], delimiter: str = ",", quotechar: str = '"', name: Optional[str] = None
+    ) -> List[Document]:
+        try:
+            if isinstance(file, Path):
+                if not file.exists():
+                    raise FileNotFoundError(f"Could not find file: {file}")
+                log_debug(f"Reading: {file}")
+                file_content = file.open(newline="", mode="r", encoding=self.encoding or "utf-8")
+            else:
+                log_debug(f"Reading retrieved file: {name or file.name}")
+                file.seek(0)
+                file_content = io.StringIO(file.read().decode("utf-8"))  # type: ignore
+
+            csv_name = name or (
+                Path(file.name).stem
+                if isinstance(file, Path)
+                else (getattr(file, "name", "csv_file").split(".")[0] if hasattr(file, "name") else "csv_file")
+            )
+
+            documents = []
+
+            with file_content as csvfile:
+                csv_reader = csv.reader(csvfile, delimiter=delimiter, quotechar=quotechar)
+
+                # Read all rows
+                rows = list(csv_reader)
+
+                if not rows:
+                    log_warning("CSV file is empty")
+                    return []
+
+                # First row is headers
+                headers = [header.strip() for header in rows[0]]
+                log_debug(f"Found {len(headers)} headers: {headers}")
+
+                data_rows = rows[1:] if len(rows) > 1 else []
+                log_debug(f"Processing {len(data_rows)} data rows")
+
+                for row_index, row in enumerate(data_rows):
+                    # Ensure row has same length as headers (pad or truncate)
+                    normalized_row = row[: len(headers)]  # Truncate if too long
+                    while len(normalized_row) < len(headers):  # Pad if too short
+                        normalized_row.append("")
+
+                    # Convert row to labeled text
+                    labeled_text = self._convert_row_to_labeled_text(headers, normalized_row, row_index)
+
+                    if labeled_text.strip():
+                        # Create document for this row
+                        doc_id = f"{csv_name}_row_{row_index + 1}"
+
+                        document = Document(
+                            id=doc_id,
+                            name=csv_name,
+                            meta_data={
+                                "row_index": row_index,
+                                "headers": headers,
+                                "total_rows": len(data_rows),
+                                "source": "field_labeled_csv_reader",
+                            },
+                            content=labeled_text,
+                        )
+
+                        documents.append(document)
+                        log_debug(f"Created document for row {row_index + 1}: {len(labeled_text)} chars")
+
+            log_debug(f"Successfully created {len(documents)} labeled documents from CSV")
+            return documents
+
+        except Exception as e:
+            log_error(f"Error reading: {getattr(file, 'name', str(file)) if isinstance(file, IO) else file}: {e}")
+            return []
+
+    async def async_read(
+        self,
+        file: Union[Path, IO[Any]],
+        delimiter: str = ",",
+        quotechar: str = '"',
+        page_size: int = 1000,
+        name: Optional[str] = None,
+    ) -> List[Document]:
+        try:
+            # Handle file input
+            if isinstance(file, Path):
+                if not file.exists():
+                    raise FileNotFoundError(f"Could not find file: {file}")
+                log_debug(f"Reading async: {file}")
+                async with aiofiles.open(file, mode="r", encoding=self.encoding or "utf-8", newline="") as file_content:
+                    content = await file_content.read()
+                    file_content_io = io.StringIO(content)
+            else:
+                log_debug(f"Reading retrieved file async: {name or file.name}")
+                file.seek(0)
+                file_content_io = io.StringIO(file.read().decode("utf-8"))  # type: ignore
+
+            csv_name = name or (
+                Path(file.name).stem
+                if isinstance(file, Path)
+                else (getattr(file, "name", "csv_file").split(".")[0] if hasattr(file, "name") else "csv_file")
+            )
+
+            file_content_io.seek(0)
+            csv_reader = csv.reader(file_content_io, delimiter=delimiter, quotechar=quotechar)
+            rows = list(csv_reader)
+
+            if not rows:
+                log_warning("CSV file is empty")
+                return []
+
+            # First row is headers
+            headers = [header.strip() for header in rows[0]]
+            log_debug(f"Found {len(headers)} headers: {headers}")
+
+            # Process data rows
+            data_rows = rows[1:] if len(rows) > 1 else []
+            total_rows = len(data_rows)
+            log_debug(f"Processing {total_rows} data rows")
+
+            # For small files, process all at once
+            if total_rows <= 10:
+                documents = []
+                for row_index, row in enumerate(data_rows):
+                    normalized_row = row[: len(headers)]
+                    while len(normalized_row) < len(headers):
+                        normalized_row.append("")
+
+                    labeled_text = self._convert_row_to_labeled_text(headers, normalized_row, row_index)
+
+                    if labeled_text.strip():
+                        document = Document(
+                            id=f"{csv_name}_row_{row_index + 1}",
+                            name=csv_name,
+                            meta_data={
+                                "row_index": row_index,
+                                "headers": headers,
+                                "total_rows": total_rows,
+                                "source": "field_labeled_csv_reader",
+                            },
+                            content=labeled_text,
+                        )
+                        documents.append(document)
+            else:
+                pages = []
+                for i in range(0, total_rows, page_size):
+                    pages.append(data_rows[i : i + page_size])
+
+                async def _process_page(page_number: int, page_rows: List[List[str]]) -> List[Document]:
+                    """Process a page of rows into documents"""
+                    page_documents = []
+                    start_row_index = (page_number - 1) * page_size
+
+                    for i, row in enumerate(page_rows):
+                        row_index = start_row_index + i
+
+                        normalized_row = row[: len(headers)]
+                        while len(normalized_row) < len(headers):
+                            normalized_row.append("")
+
+                        labeled_text = self._convert_row_to_labeled_text(headers, normalized_row, row_index)
+
+                        if labeled_text.strip():
+                            document = Document(
+                                id=f"{csv_name}_row_{row_index + 1}",
+                                name=csv_name,
+                                meta_data={
+                                    "row_index": row_index,
+                                    "headers": headers,
+                                    "total_rows": total_rows,
+                                    "page": page_number,
+                                    "source": "field_labeled_csv_reader",
+                                },
+                                content=labeled_text,
+                            )
+                            page_documents.append(document)
+
+                    return page_documents
+
+                page_results = await asyncio.gather(
+                    *[_process_page(page_number, page) for page_number, page in enumerate(pages, start=1)]
+                )
+
+                documents = [doc for page_docs in page_results for doc in page_docs]
+
+            log_debug(f"Successfully created {len(documents)} labeled documents from CSV")
+            return documents
+
+        except Exception as e:
+            log_error(f"Error reading async: {getattr(file, 'name', str(file)) if isinstance(file, IO) else file}: {e}")
+            return []

agno/knowledge/reader/firecrawl_reader.py

@@ -0,0 +1,201 @@
+import asyncio
+from dataclasses import dataclass
+from typing import Dict, List, Literal, Optional
+
+from agno.knowledge.chunking.semantic import SemanticChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.types import ContentType
+from agno.utils.log import log_debug, logger
+
+try:
+    from firecrawl import FirecrawlApp  # type: ignore[attr-defined]
+except ImportError:
+    raise ImportError("The `firecrawl` package is not installed. Please install it via `pip install firecrawl-py`.")
+
+
+@dataclass
+class FirecrawlReader(Reader):
+    api_key: Optional[str] = None
+    params: Optional[Dict] = None
+    mode: Literal["scrape", "crawl"] = "scrape"
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        params: Optional[Dict] = None,
+        mode: Literal["scrape", "crawl"] = "scrape",
+        chunk: bool = True,
+        chunk_size: int = 5000,
+        chunking_strategy: Optional[ChunkingStrategy] = SemanticChunking(),
+        name: Optional[str] = None,
+        description: Optional[str] = None,
+    ) -> None:
+        # Initialise base Reader (handles chunk_size / strategy)
+        super().__init__(
+            chunk=chunk, chunk_size=chunk_size, chunking_strategy=chunking_strategy, name=name, description=description
+        )
+
+        # Firecrawl-specific attributes
+        self.api_key = api_key
+        self.params = params
+        self.mode = mode
+
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for Firecrawl readers."""
+        return [
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+        ]
+
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.URL]
+
+    def scrape(self, url: str, name: Optional[str] = None) -> List[Document]:
+        """
+        Scrapes a website and returns a list of documents.
+
+        Args:
+            url: The URL of the website to scrape
+
+        Returns:
+            A list of documents
+        """
+
+        log_debug(f"Scraping: {url}")
+
+        app = FirecrawlApp(api_key=self.api_key)
+
+        if self.params:
+            scraped_data = app.scrape_url(url, **self.params)
+        else:
+            scraped_data = app.scrape_url(url)
+        if isinstance(scraped_data, dict):
+            content = scraped_data.get("markdown", "")
+        else:
+            content = getattr(scraped_data, "markdown", "")
+
+        # Debug logging
+        log_debug(f"Received content type: {type(content)}")
+        log_debug(f"Content empty: {not bool(content)}")
+
+        # Ensure content is a string
+        if content is None:
+            content = ""  # or you could use metadata to create a meaningful message
+            logger.warning(f"No content received for URL: {url}")
+
+        documents = []
+        if self.chunk and content:  # Only chunk if there's content
+            documents.extend(self.chunk_document(Document(name=name or url, id=url, content=content)))
+        else:
+            documents.append(Document(name=name or url, id=url, content=content))
+        return documents
+
+    async def async_scrape(self, url: str, name: Optional[str] = None) -> List[Document]:
+        """
+        Asynchronously scrapes a website and returns a list of documents.
+
+        Args:
+            url: The URL of the website to scrape
+
+        Returns:
+            A list of documents
+        """
+        log_debug(f"Async scraping: {url}")
+
+        # Use asyncio.to_thread to run the synchronous scrape in a thread
+        return await asyncio.to_thread(self.scrape, url, name)
+
+    def crawl(self, url: str, name: Optional[str] = None) -> List[Document]:
+        """
+        Crawls a website and returns a list of documents.
+
+        Args:
+            url: The URL of the website to crawl
+
+        Returns:
+            A list of documents
+        """
+        log_debug(f"Crawling: {url}")
+
+        app = FirecrawlApp(api_key=self.api_key)
+
+        if self.params:
+            crawl_result = app.crawl_url(url, **self.params)
+        else:
+            crawl_result = app.crawl_url(url)
+        documents = []
+
+        if isinstance(crawl_result, dict):
+            results_data = crawl_result.get("data", [])
+        else:
+            results_data = getattr(crawl_result, "data", [])
+        for result in results_data:
+            # Get markdown content, default to empty string if not found
+            if isinstance(result, dict):
+                content = result.get("markdown", "")
+            else:
+                content = getattr(result, "markdown", "")
+
+            if content:  # Only create document if content exists
+                if self.chunk:
+                    documents.extend(self.chunk_document(Document(name=name or url, id=url, content=content)))
+                else:
+                    documents.append(Document(name=name or url, id=url, content=content))
+
+        return documents
+
+    async def async_crawl(self, url: str, name: Optional[str] = None) -> List[Document]:
+        """
+        Asynchronously crawls a website and returns a list of documents.
+
+        Args:
+            url: The URL of the website to crawl
+
+        Returns:
+            A list of documents
+        """
+        log_debug(f"Async crawling: {url}")
+
+        # Use asyncio.to_thread to run the synchronous crawl in a thread
+        return await asyncio.to_thread(self.crawl, url, name)
+
+    def read(self, url: str, name: Optional[str] = None) -> List[Document]:
+        """
+        Reads from a URL based on the mode setting.
+
+        Args:
+            url: The URL of the website to process
+
+        Returns:
+            A list of documents
+        """
+        if self.mode == "scrape":
+            return self.scrape(url, name)
+        elif self.mode == "crawl":
+            return self.crawl(url, name)
+        else:
+            raise NotImplementedError(f"Mode {self.mode} not implemented")
+
+    async def async_read(self, url: str, name: Optional[str] = None) -> List[Document]:
+        """
+        Asynchronously reads from a URL based on the mode setting.
+
+        Args:
+            url: The URL of the website to process
+
+        Returns:
+            A list of documents
+        """
+        if self.mode == "scrape":
+            return await self.async_scrape(url, name)
+        elif self.mode == "crawl":
+            return await self.async_crawl(url, name)
+        else:
+            raise NotImplementedError(f"Mode {self.mode} not implemented")

agno/knowledge/reader/json_reader.py

@@ -0,0 +1,88 @@
+import asyncio
+import json
+from io import BytesIO
+from pathlib import Path
+from typing import IO, Any, List, Optional, Union
+from uuid import uuid4
+
+from agno.knowledge.chunking.fixed import FixedSizeChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.types import ContentType
+from agno.utils.log import log_debug, log_error
+
+
+class JSONReader(Reader):
+    """Reader for JSON files"""
+
+    chunk: bool = False
+
+    def __init__(self, chunking_strategy: Optional[ChunkingStrategy] = FixedSizeChunking(), **kwargs):
+        super().__init__(chunking_strategy=chunking_strategy, **kwargs)
+
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for JSON readers."""
+        return [
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
+        ]
+
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.JSON]
+
+    def read(self, path: Union[Path, IO[Any]], name: Optional[str] = None) -> List[Document]:
+        try:
+            if isinstance(path, Path):
+                if not path.exists():
+                    raise FileNotFoundError(f"Could not find file: {path}")
+                log_debug(f"Reading: {path}")
+                json_name = name or path.name.split(".")[0]
+                json_contents = json.loads(path.read_text(self.encoding or "utf-8"))
+
+            elif isinstance(path, BytesIO):
+                json_name = name or path.name.split(".")[0]
+                log_debug(f"Reading uploaded file: {json_name}")
+                path.seek(0)
+                json_contents = json.load(path)
+
+            else:
+                raise ValueError("Unsupported file type. Must be Path or BytesIO.")
+
+            if isinstance(json_contents, dict):
+                json_contents = [json_contents]
+
+            documents = [
+                Document(
+                    name=json_name,
+                    id=str(uuid4()),
+                    meta_data={"page": page_number},
+                    content=json.dumps(content),
+                )
+                for page_number, content in enumerate(json_contents, start=1)
+            ]
+            if self.chunk:
+                chunked_documents = []
+                for document in documents:
+                    chunked_documents.extend(self.chunk_document(document))
+                return chunked_documents
+            return documents
+        except Exception as e:
+            log_error(f"Error reading: {path}: {e}")
+            raise
+
+    async def async_read(self, path: Union[Path, IO[Any]], name: Optional[str] = None) -> List[Document]:
+        """Asynchronously read JSON files.
+
+        Args:
+            path (Union[Path, IO[Any]]): Path to a JSON file or a file-like object
+
+        Returns:
+            List[Document]: List of documents from the JSON file
+        """
+        return await asyncio.to_thread(self.read, path, name)