ai-parrot 0.17.2__cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl

parrot/loaders/epubloader.py

@@ -0,0 +1,316 @@
+from typing import List, Optional, Union, Tuple, Dict
+from pathlib import PurePath
+from collections.abc import Callable
+from ..stores.models import Document
+from .abstract import AbstractLoader
+
+# Optional deps: install via
+# pip install ebooklib beautifulsoup4 markdownify
+try:
+    from ebooklib import epub
+    EBOOKLIB_AVAILABLE = True
+    try:
+        ITEM_DOCUMENT = epub.ITEM_DOCUMENT
+    except AttributeError:
+        try:
+            from ebooklib.epub import ITEM_DOCUMENT
+        except ImportError:
+            ITEM_DOCUMENT = 9  # Known constant value
+except Exception:
+    EBOOKLIB_AVAILABLE = False
+
+try:
+    from bs4 import BeautifulSoup
+    BS4_AVAILABLE = True
+except Exception:
+    BS4_AVAILABLE = False
+
+try:
+    from markdownify import MarkdownConverter
+    MD_AVAILABLE = True
+except Exception:
+    MD_AVAILABLE = False
+
+
+class EpubLoader(AbstractLoader):
+    """
+    EPUB loader that extracts clean Markdown (or plain text) from chapters/sections.
+
+    Features:
+    - Per-chapter documents with titles from TOC/HTML
+    - Optional full-book document (merged)
+    - Clean Markdown conversion (lists, headers, links)
+    - Skips non-document items (css, images, fonts)
+    - Configurable minimum content length
+    """
+
+    extensions: List[str] = ['.epub']
+
+    def __init__(
+        self,
+        source: Optional[Union[str, PurePath, List[PurePath]]] = None,
+        *,
+        tokenizer: Union[str, Callable] = None,
+        text_splitter: Union[str, Callable] = None,
+        source_type: str = 'file',
+
+        # Output controls
+        as_markdown: bool = True,          # emit markdown instead of plain text
+        per_chapter: bool = True,          # True => one Document per chapter; False => single full-book doc
+        include_toc_document: bool = False,# optional separate TOC document
+        min_section_length: int = 50,      # drop tiny/empty sections
+
+        # Markdown conversion tuning
+        heading_style: str = "ATX",        # for markdownify; "ATX" => # Heading
+        strip_whitespace: bool = True,
+
+        **kwargs
+    ):
+        super().__init__(
+            source,
+            tokenizer=tokenizer,
+            text_splitter=text_splitter,
+            source_type=source_type,
+            **kwargs
+        )
+        self.doctype = 'epub'
+        self._source_type = 'ebook'
+
+        # Options
+        self.as_markdown = as_markdown
+        self.per_chapter = per_chapter
+        self.include_toc_document = include_toc_document
+        self.min_section_length = int(min_section_length)
+        self.strip_whitespace = bool(strip_whitespace)
+        self.heading_style = heading_style
+
+        # sanity checks
+        if not EBOOKLIB_AVAILABLE or not BS4_AVAILABLE:
+            missing = []
+            if not EBOOKLIB_AVAILABLE:
+                missing.append("ebooklib")
+            if not BS4_AVAILABLE:
+                missing.append("beautifulsoup4")
+            raise ImportError(
+                f"EpubLoader requires {', '.join(missing)}. "
+                f"Install with: pip install ebooklib beautifulsoup4"
+            )
+
+    def _html_to_markdown(self, html: str) -> str:
+        """Convert XHTML chapter html to Markdown (fallback to plain text)."""
+        soup = BeautifulSoup(html, "html.parser")
+
+        # remove scripts/styles
+        for bad in soup(["script", "style", "noscript"]):
+            bad.decompose()
+
+        if MD_AVAILABLE and self.as_markdown:
+            md = MarkdownConverter(
+                heading_style=self.heading_style,
+                strip=['style', 'script', 'noscript']
+            ).convert_soup(soup)
+            return self._clean(md)
+
+        # plain text fallback
+        text = soup.get_text("\n", strip=True)
+        return self._clean(text)
+
+    def _clean(self, text: str) -> str:
+        if not text:
+            return ""
+        if self.strip_whitespace:
+            # Normalize multiple blank lines; trim trailing spaces
+            lines = [ln.rstrip() for ln in text.splitlines()]
+            # Collapse >2 blank lines to just one
+            cleaned = []
+            blank = 0
+            for ln in lines:
+                if ln.strip():
+                    blank = 0
+                    cleaned.append(ln)
+                else:
+                    blank += 1
+                    if blank <= 1:
+                        cleaned.append("")
+            text = "\n".join(cleaned)
+        return text.strip()
+
+    def _flatten_toc(self, toc) -> List[Tuple[str, str]]:
+        """
+        Flatten ebooklib TOC into a list of (href, title) entries.
+        toc entries are like: Link(title, href) or nested lists/tuples.
+        """
+        flat = []
+
+        def _walk(node):
+            if isinstance(node, (list, tuple)):
+                for child in node:
+                    _walk(child)
+            else:
+                # epub.Link or epub.Section
+                try:
+                    href = getattr(node, "href", None)
+                    title = getattr(node, "title", None)
+                    if href and title:
+                        flat.append((href.split("#", 1)[0], str(title)))
+                except Exception:
+                    pass
+
+        _walk(toc)
+        return flat
+
+    def _toc_title_lookup(self, book: "epub.EpubBook") -> Dict[str, str]:
+        """
+        Build a mapping from href→title using TOC (best effort).
+        Keys are hrefs without fragments; values are strings.
+        """
+        try:
+            flat = self._flatten_toc(book.toc or [])
+            # Normalize: keep last title if duplicates
+            return {href: title for href, title in flat}
+        except Exception:
+            return {}
+
+    def _iter_document_items(self, book: "epub.EpubBook"):
+        """
+        Yield (order_idx, item) for spine items that are HTML documents.
+        """
+        id_to_item = {it.get_id(): it for it in book.get_items()}
+        order = 0
+        for entry in (book.spine or []):
+            if isinstance(entry, tuple) and entry and isinstance(entry[0], str):
+                idref = entry[0]
+                item = id_to_item.get(idref)
+                if item is None:
+                    continue
+                if item.get_type() == ITEM_DOCUMENT:
+                    yield order, item
+                    order += 1
+
+        if order == 0:
+            for i, item in enumerate(book.get_items_of_type(ITEM_DOCUMENT)):
+                yield i, item
+
+    def _derive_title_from_html(self, html: str) -> Optional[str]:
+        soup = BeautifulSoup(html, "html.parser")
+        # Try <title>
+        if soup.title and soup.title.string:
+            t = soup.title.string.strip()
+            if t:
+                return t
+        # Try first heading
+        for tag in ["h1", "h2", "h3"]:
+            h = soup.find(tag)
+            if h and h.get_text(strip=True):
+                return h.get_text(strip=True)
+        return None
+
+    async def _load(self, path: PurePath, **kwargs) -> List[Document]:
+        """
+        Load an EPUB file into Parrot Documents.
+
+        Returns:
+            - Per-chapter Documents (default), or
+            - Single full-book Document if per_chapter=False
+        """
+        self.logger.info(f"Loading EPUB file: {path}")
+
+        docs: List[Document] = []
+        try:
+            book = epub.read_epub(str(path))
+        except Exception as e:
+            self.logger.error(f"Failed to open EPUB {path}: {e}")
+            return docs
+
+        toc_map = self._toc_title_lookup(book)
+
+        # Optionally create a separate TOC document
+        if self.include_toc_document and toc_map:
+            toc_lines = ["# Table of Contents"]
+            for href, title in toc_map.items():
+                toc_lines.append(f"- {title} (Link: {href})")
+            toc_content = "\n".join(toc_lines)
+            toc_meta = self.create_metadata(
+                path=path,
+                doctype="epub",
+                source_type="epub_toc",
+                doc_metadata={
+                    "content_type": "toc",
+                    "entries": len(toc_map)
+                },
+            )
+            docs.append(self.create_document(toc_content, path, toc_meta))
+
+        # Collect per-chapter or full text
+        all_sections = []
+        for order_idx, item in self._iter_document_items(book):
+            try:
+                html = item.get_content().decode("utf-8", errors="ignore")
+            except Exception:
+                continue
+
+            content = self._html_to_markdown(html)
+
+            if len(content) < self.min_section_length:
+                # skip boilerplate/empty stubs
+                continue
+
+            # Derive title from TOC → HTML <title> → filename
+            href = getattr(item, "file_name", "") or ""
+            title = toc_map.get(href) or self._derive_title_from_html(html) or PurePath(href).name or f"Section {order_idx+1}"
+
+            # Track for full-book option
+            all_sections.append((order_idx, title, content, href))
+
+            # Per-chapter Document
+            if self.per_chapter:
+                section_meta = self.create_metadata(
+                    path=path,
+                    doctype="epub",
+                    source_type="epub_section",
+                    doc_metadata={
+                        "section_order": order_idx + 1,
+                        "section_title": title,
+                        "href": href,
+                        "content_type": "chapter",
+                        "output_format": "markdown" if self.as_markdown else "text",
+                        "min_section_length": self.min_section_length
+                    },
+                )
+
+                # Prepend a lightweight context header (like your PPT/PDF style)
+                context = [
+                    f"File Name: {path.name if hasattr(path, 'name') else str(path)}",
+                    f"Section: {order_idx + 1}",
+                    f"Title: {title}",
+                    f"Document Type: epub",
+                    f"Source Type: ebook",
+                ]
+                full_content = "\n".join(context) + "\n======\n\n" + content
+
+                docs.append(self.create_document(full_content, path, section_meta))
+
+        if not all_sections:
+            self.logger.warning(f"No textual sections extracted from {path}")
+            return docs
+
+        # Full-book Document (if requested)
+        if not self.per_chapter:
+            merged = []
+            for order_idx, title, content, href in all_sections:
+                merged.append(f"# {title}\n\n{content}\n")
+            book_text = "\n\n".join(merged).strip()
+
+            full_meta = self.create_metadata(
+                path=path,
+                doctype="epub",
+                source_type="epub_full",
+                doc_metadata={
+                    "sections": len(all_sections),
+                    "content_type": "full_document",
+                    "output_format": "markdown" if self.as_markdown else "text",
+                },
+            )
+            docs.append(self.create_document(book_text, path, full_meta))
+
+        return docs

parrot/loaders/excel.py

@@ -0,0 +1,199 @@
+# parrot/loaders/excel.py
+from typing import List, Optional, Union, Literal, Dict
+from pathlib import PurePath
+from collections.abc import Callable
+import pandas as pd
+from navigator.libs.json import JSONContent
+from ..stores.models import Document
+from .abstract import AbstractLoader
+
+
+class ExcelLoader(AbstractLoader):
+    """
+    Excel loader that converts an Excel workbook (or DataFrame) into per-row Documents.
+
+    - One Document per row per sheet (rows with all-empty values are skipped).
+    - Works for .xlsx / .xlsm / .xls files (pandas engine auto-detects).
+    - Also accepts a pandas.DataFrame (sheet='DataFrame').
+    - Output formats: markdown (default), plain, or json.
+    """
+
+    extensions: List[str] = ['.xlsx', '.xlsm', '.xls']
+
+    def __init__(
+        self,
+        source: Optional[Union[str, PurePath, List[PurePath]]] = None,
+        *,
+        tokenizer: Union[str, Callable] = None,
+        text_splitter: Union[str, Callable] = None,
+        source_type: str = 'file',
+
+        sheets: Optional[Union[str, int, List[Union[str, int]]]] = None,
+        header: Union[int, List[int], None] = 0,
+        usecols: Optional[Union[str, List[Union[int, str]]]] = None,
+        drop_empty_rows: bool = True,
+        max_rows: Optional[int] = None,
+        date_format: str = "%Y-%m-%d",
+        output_format: Literal["markdown", "plain", "json"] = "markdown",
+        min_row_length: int = 1,  # skip rows with < N non-empty fields
+        title_column: Optional[str] = None,
+
+        **kwargs
+    ):
+        super().__init__(
+            source,
+            tokenizer=tokenizer,
+            text_splitter=text_splitter,
+            source_type=source_type,
+            **kwargs
+        )
+        self.doctype = 'excel'
+        self._source_type = source_type
+        self.sheets = sheets
+        self.header = header
+        self.usecols = usecols
+        self.drop_empty_rows = drop_empty_rows
+        self.max_rows = max_rows
+        self.date_format = date_format
+        self.output_format = output_format
+        self.min_row_length = int(min_row_length)
+        self.title_column = title_column
+
+    def _stringify(self, v):
+        if pd.isna(v):
+            return ""
+        if isinstance(v, (pd.Timestamp, )):
+            return v.strftime(self.date_format)
+        return str(v)
+
+    def _row_to_text(self, row: Dict[str, object]) -> str:
+        """Render a single row dict to text in the chosen output_format."""
+        if self.output_format == "json":
+            return JSONContent.dumps(row, indent=2)
+
+        items = [(k, self._stringify(v)) for k, v in row.items()]
+        if self.output_format == "plain":
+            # key: value per line
+            return "\n".join(f"{k}: {v}" for k, v in items if v != "")
+
+        # markdown: list of **key**: value
+        return "\n".join(f"- **{k}**: {v}" for k, v in items if v != "")
+
+    def _row_nonempty_count(self, row: Dict[str, object]) -> int:
+        return sum(1 for v in row.values() if (not pd.isna(v)) and str(v).strip() != "")
+
+    async def _load(self, source: Union[PurePath, str, pd.DataFrame], **kwargs) -> List[Document]:
+        """
+        Load an Excel file (or DataFrame) and return per-row Documents.
+        """
+        docs: List[Document] = []
+
+        # Case A: already a DataFrame (from AbstractLoader.from_dataframe)
+        # (sheet name is synthetic: "DataFrame")
+        if isinstance(source, pd.DataFrame):
+            sheet_name = "DataFrame"
+            docs.extend(await self._docs_from_dataframe(source, sheet_name, path_hint="dataframe"))
+            return docs
+
+        # Case B: excel path
+        path = PurePath(source) if not isinstance(source, PurePath) else source
+        self.logger.info(f"Loading Excel file: {path}")
+
+        # Read one or multiple sheets
+        try:
+            # If sheets=None -> pd returns dict of DataFrames (all sheets)
+            # If sheets is a single name/index -> returns a DataFrame
+            xls = pd.read_excel(
+                str(path),
+                sheet_name=self.sheets if self.sheets is not None else None,
+                header=self.header,
+                usecols=self.usecols,
+                dtype=object  # keep as objects → stringify ourselves
+            )
+        except Exception as e:
+            self.logger.error(f"Failed to read Excel {path}: {e}")
+            return docs
+
+        # Normalize to dict[str, DataFrame]
+        if isinstance(xls, pd.DataFrame):
+            frames = {"Sheet1" if self.sheets is None else str(self.sheets): xls}
+        else:
+            # dict of {sheet_name: df}
+            frames = {str(k): v for k, v in xls.items()}
+
+        for sheet_name, df in frames.items():
+            # Drop fully empty rows
+            if self.drop_empty_rows:
+                df = df.dropna(how="all")
+
+            if self.max_rows is not None:
+                df = df.head(self.max_rows)
+
+            if df.empty:
+                self.logger.info(f"Sheet '{sheet_name}' is empty; skipping.")
+                continue
+
+            # Ensure columns are strings
+            df.columns = [str(c) for c in df.columns]
+            docs.extend(await self._docs_from_dataframe(df, sheet_name, path_hint=path))
+
+        return docs
+
+    async def _docs_from_dataframe(
+        self,
+        df: pd.DataFrame,
+        sheet_name: str,
+        path_hint: Union[str, PurePath]
+    ) -> List[Document]:
+        """Convert a DataFrame into per-row Documents."""
+        docs: List[Document] = []
+
+        # Convert to records for easy iteration
+        records = df.to_dict(orient="records")
+
+        for i, row in enumerate(records, start=1):
+            if self.min_row_length > 1 and self._row_nonempty_count(row) < self.min_row_length:
+                continue
+
+            content_body = self._row_to_text(row)
+
+            # Context header (aligns with PDF/PPT style: header + "======")
+            title_val = None
+            if self.title_column and self.title_column in row:
+                title_val = self._stringify(row[self.title_column]).strip() or None
+
+            context = [
+                f"File Name: {path_hint.name if hasattr(path_hint, 'name') else str(path_hint)}",
+                f"Sheet: {sheet_name}",
+                f"Row: {i}",
+                f"Document Type: excel",
+                f"Source Type: {self._source_type}",
+            ]
+            if title_val:
+                context.append(f"Title: {title_val}")
+
+            full_content = "\n".join(context) + "\n======\n\n" + content_body
+
+            # Metadata
+            doc_meta = {
+                "filename": path_hint.name if hasattr(path_hint, 'name') else str(path_hint),
+                "file_path": str(path_hint),
+                "sheet": sheet_name,
+                "row_index": i,
+                "columns": list(df.columns),
+                "content_type": "row",
+                "output_format": self.output_format,
+            }
+
+            meta = self.create_metadata(
+                path=path_hint,
+                doctype="excel",
+                source_type="excel_row",
+                doc_metadata=doc_meta,
+            )
+
+            docs.append(
+                self.create_document(full_content, path_hint, meta)
+            )
+
+        return docs

parrot/loaders/factory.py

@@ -0,0 +1,55 @@
+####
+# Copyright 2023 Jesus Lara.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#
+# Loaders.
+# Open, extract and load data from different sources.
+#####
+from .pdf import PDFLoader
+from .txt import TextLoader
+from .docx import MSWordLoader
+from .qa import QAFileLoader
+from .html import HTMLLoader
+from .pdfmark import PDFMarkdownLoader
+from .pdftables import PDFTablesLoader
+from .csv import CSVLoader
+from .youtube import YoutubeLoader
+from .web import WebLoader
+from .ppt import PowerPointLoader
+from .markdown import MarkdownLoader
+from .epubloader import EpubLoader
+from .excel import ExcelLoader
+# from .video import VideoLoader
+from .videolocal import VideoLocalLoader
+from .videounderstanding import VideoUnderstandingLoader
+# from .vimeo import VimeoLoader
+from .audio import AudioLoader
+
+AVAILABLE_LOADERS = {
+    '.pdf': PDFLoader,
+    '.txt': TextLoader,
+    '.docx': MSWordLoader,
+    '.qa': QAFileLoader,
+    '.xlsx': ExcelLoader,
+    '.xlsm': ExcelLoader,
+    '.xls': ExcelLoader,
+    '.html': HTMLLoader,
+    '.pdfmd': PDFMarkdownLoader,
+    '.pdftables': PDFTablesLoader,
+    '.csv': CSVLoader,
+    '.youtube': YoutubeLoader,
+    '.web': WebLoader,
+    '.ppt': PowerPointLoader,
+    '.pptx': PowerPointLoader,
+    '.md': MarkdownLoader,
+    '.json': MarkdownLoader,
+    '.xml': MarkdownLoader,
+    '.epub': EpubLoader,
+    '.mp3': AudioLoader,
+    '.wav': AudioLoader,
+    '.avi': VideoUnderstandingLoader,
+    '.mp4': VideoUnderstandingLoader,
+    '.webm': VideoUnderstandingLoader,
+    '.mov': VideoUnderstandingLoader,
+    '.mkv': VideoUnderstandingLoader,
+}

parrot/loaders/files/abstract.py

@@ -0,0 +1,39 @@
+from typing import Optional, Any
+from abc import ABC, abstractmethod
+from navconfig.logging import logging
+
+
+class FilePlugin(ABC):
+    """
+    FilePlugin is a base class for Open Files.
+    It provides a common interface for all opening all kind of iles.
+    Subclasses should implement the `open` method to define
+    the specific file processing logic.
+    """
+
+    def __init__(self, *args, **kwargs):
+        """
+        Initialize the ImagePlugin with an optional image path.
+
+        :param image: Path to the image file.
+        """
+        self.logger = logging.getLogger(
+            f'parrot.FileLoader.{self.__class__.__name__}'
+        )
+
+    async def __aenter__(self):
+        if hasattr(self, "open"):
+            await self.open()
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        if hasattr(self, "close"):
+            await self.close()
+        return True
+
+    @abstractmethod
+    async def read(self):
+        """
+        Return the content of the file, need to be implemented in the subclass.
+        """
+        pass

parrot/loaders/files/html.py

@@ -0,0 +1,26 @@
+from bs4 import BeautifulSoup
+from .text import TextFile
+
+class HTMLFile(TextFile):
+    """
+    A class to handle HTML files asynchronously.
+    """
+    async def read(self) -> str:
+        """
+        Asynchronously read the content of the html file.
+
+        Returns:
+            BeautifulSoup object of HTML File.
+        """
+        if self._file is None:
+            await self.open()
+
+        try:
+            content = await self._file.read()
+            soup = BeautifulSoup(content, 'html.parser')
+            return soup, content
+        except Exception as e:
+            self.logger.error(
+                f"Error reading HTML file {self.path}: {str(e)}"
+            )
+            raise