ai-parrot 0.3.4__cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl

parrot/loaders/vimeo.py

@@ -0,0 +1,106 @@
+from typing import Optional, Union
+from transformers import pipeline
+import torch
+from langchain.docstore.document import Document
+from .youtube import YoutubeLoader
+
+
+class VimeoLoader(YoutubeLoader):
+    """
+    Loader for Vimeo videos.
+    """
+    def load_video(self, url: str, video_title: str, transcript: Optional[Union[str, None]] = None) -> list:
+        metadata = {
+            "source": url,
+            "url": url,
+            "index": url,
+            "filename": video_title,
+            "question": '',
+            "answer": '',
+            'type': 'video_transcript',
+            "source_type": self._source_type,
+            "summary": '',
+            "document_meta": {
+                "language": self._language,
+                "title": video_title,
+                "topic_tags": ""
+            }
+        }
+        if self.topics:
+            metadata['document_meta']['topic_tags'] = self.topics
+        if transcript is None:
+            documents = []
+            docs = []
+            # first: download video
+            try:
+                file_path = self.download_video(url, self._video_path)
+            except Exception:
+                return []
+            if not file_path:
+                self.logger.warning(
+                    f"Error downloading File for video: {self._video_path}"
+                )
+                return []
+            transcript_path = file_path.with_suffix('.vtt')
+            audio_path = file_path.with_suffix('.mp3')
+            # second: extract audio
+            self.extract_audio(file_path, audio_path)
+            # get the Whisper parser
+            transcript_whisper = self.get_whisper_transcript(audio_path)
+            if transcript_whisper:
+                transcript = transcript_whisper['text']
+            else:
+                transcript = ''
+            # Summarize the transcript
+            if transcript:
+                summary = self.get_summary_from_text(transcript)
+                # Create Two Documents, one is for transcript, second is VTT:
+                metadata['summary'] = summary
+                doc = Document(
+                    page_content=transcript,
+                    metadata=metadata
+                )
+                documents.append(doc)
+            if transcript_whisper:
+                # VTT version:
+                transcript = self.transcript_to_vtt(transcript_whisper, transcript_path)
+                doc = Document(
+                    page_content=transcript,
+                    metadata=metadata
+                )
+                documents.append(doc)
+                # Saving every dialog chunk as a separate document
+                dialogs = self.transcript_to_blocks(transcript_whisper)
+                for chunk in dialogs:
+                    _meta = {
+                        "index": f"{video_title}:{chunk['id']}",
+                        "document_meta": {
+                            "start": f"{chunk['start_time']}",
+                            "end": f"{chunk['end_time']}",
+                            "id": f"{chunk['id']}",
+                            "language": self._language,
+                            "title": video_title,
+                            "topic_tags": ""
+                        }
+                    }
+                    _info = {**metadata, **_meta}
+                    doc = Document(
+                        page_content=chunk['text'],
+                        metadata=_info
+                    )
+                    docs.append(doc)
+            documents.extend(docs)
+            return self.split_documents(documents)
+        else:
+            # using the transcript file
+            with open(transcript, 'r') as f:
+                transcript = f.read()
+                summary = self.get_summary_from_text(transcript)
+                transcript_whisper = None
+            metadata['summary'] = f"{summary!s}"
+            # Create Two Documents, one is for transcript, second is VTT:
+            doc = Document(
+                page_content=transcript,
+                metadata=metadata
+            )
+            return self.split_documents([doc])

parrot/loaders/web.py

@@ -0,0 +1,216 @@
+from bs4 import BeautifulSoup
+from markdownify import MarkdownConverter
+from webdriver_manager.chrome import ChromeDriverManager
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from langchain.docstore.document import Document
+from langchain.text_splitter import MarkdownTextSplitter
+from navconfig.logging import logging
+from .abstract import AbstractLoader
+
+
+logging.getLogger(name='selenium.webdriver').setLevel(logging.WARNING)
+logging.getLogger(name='WDM').setLevel(logging.WARNING)
+logging.getLogger(name='matplotlib').setLevel(logging.WARNING)
+
+
+class WebLoader(AbstractLoader):
+    """Class to load web pages and extract text."""
+    chrome_options = [
+        "--headless",
+        "--enable-automation",
+        "--lang=en",
+        "--disable-extensions",
+        "--disable-gpu",
+        "--no-sandbox",
+        "--disable-features=NetworkService",
+        "--disable-dev-shm-usage",
+        "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"
+    ]
+    def __init__(self, urls: dict, source_type: str = 'website', **kwargs):
+        self.urls = urls
+        self._source_type = source_type
+        self._options = Options()
+        self.timeout: int = kwargs.pop('timeout', 60)
+        for option in self.chrome_options:
+            self._options.add_argument(option)
+        self.driver = webdriver.Chrome(
+            service=Service(ChromeDriverManager().install()),
+            options=self._options
+        )
+        self._mark_splitter = MarkdownTextSplitter(
+            chunk_size = 1024,
+            chunk_overlap=10
+        )
+        super().__init__(source_type=source_type, **kwargs)
+
+    def md(self, soup, **options):
+        return MarkdownConverter(**options).convert_soup(soup)
+
+    def clean_html(self, html, tags, objects=[]):
+        soup = BeautifulSoup(html, 'html.parser')
+        page_title = soup.title.string
+        md_text = self.md(soup)
+        # Remove script and style elements
+        for script_or_style in soup(["script", "style", "link"]):
+            script_or_style.decompose()
+        # Extract Content
+        content = []
+        paragraphs = [' '.join(p.get_text().split()) for p in soup.find_all(tags)]
+        # Look for iframe elements and format their src attributes into readable strings
+        iframes = soup.find_all('iframe')
+        for iframe in iframes:
+            video_src = iframe.get('src', '')
+            # You might want to customize the formatting of this string
+            formatted_video = f"Video Link: {video_src}" if video_src else ""
+            content.append(formatted_video)
+        if objects:
+            for obj in objects:
+                (element, args), = obj.items()
+                if 'parse_list' in args:
+                    parse_list = args.pop('parse_list')
+                    # Find the element container:
+                    container = soup.find(element, attrs=args)
+                    # Parse list of objects (UL, LI)
+                    name_type = parse_list.pop('type')
+                    params = parse_list.get('find')
+                    el = params.pop(0)
+                    try:
+                        attrs = params.pop(0)
+                    except IndexError:
+                        attrs = {}
+                    elements = container.find_all(el, attrs=attrs)
+                    structured_text = ''
+                    for element in elements:
+                        title = element.find('span', class_='title').get_text(strip=True)
+                        lists = element.find_all('ul')
+                        if lists:
+                            structured_text += f"\nCategory: {title}\n{name_type}:\n"
+                        for ul in lists:
+                            items = [f"- {li.get_text(strip=True)}" for li in ul.select('li')]
+                            formatted_list = '\n'.join(items)
+                            structured_text += formatted_list
+                        structured_text += "\n"
+                    content.append(structured_text)
+                else:
+                    elements = soup.find_all(element, attrs=args)
+                    for element in elements:
+                        # Handle <a> tags within the current element
+                        links = element.find_all('a')
+                        for link in links:
+                            # Extract link text and href, format them into a readable string
+                            link_text = link.get_text(strip=True)
+                            href = link.get('href', '')
+                            formatted_link = (
+                                f"{link_text} (Link: {href})"
+                                if href
+                                else link_text
+                            )
+                            # Replace the original link text in the element
+                            # with the formatted version
+                            link.replace_with(formatted_link)
+                        # work with UL lists:
+                        lists = element.find_all('ul')
+                        for ul in lists:
+                            items = [li.get_text(strip=True) for li in ul.select('li')]
+                            formatted_list = '\n'.join(items)
+                            content.append(formatted_list)
+                        cleaned_text = ' '.join(element.get_text().split())
+                        content.append(cleaned_text)
+        return (content + paragraphs, md_text, page_title)
+
+    def get(self, address: dict) -> list:
+        (url, args), = address.items()
+        self.logger.info(
+            f'Downloading URL {url} with args {args}'
+        )
+        locator = args.get('locator', (By.TAG_NAME, 'body'))
+        wait = WebDriverWait(self.driver, self.timeout)
+        acookies = args.get('accept_cookies', False)
+        try:
+            self.driver.get(url)
+            # After loading page, accept cookies
+            wait.until(
+                EC.presence_of_element_located(
+                    locator
+                )
+            )
+            if acookies:
+                btn = wait.until(
+                    EC.element_to_be_clickable(
+                        acookies
+                    )
+                )
+                btn.click()
+        except Exception as exc:
+            print(f"Failed to Get {url}: {exc}")
+            self.logger.exception(
+                str(exc), stack_info=True
+            )
+            raise
+        try:
+            extract = args.get('tags', ['p', 'title', 'h1', 'h2', 'section', 'article'])
+            objects = args.get('objects', [])
+            source_type = args.get('source_type', self._source_type)
+            html_content = self.driver.page_source
+            content, md_text, page_title = self.clean_html(
+                html_content,
+                extract,
+                objects
+            )
+            metadata = {
+                "source": url,
+                "index": page_title,
+                "url": url,
+                "filename": page_title,
+                "question": '',
+                "answer": '',
+                "source_type": source_type,
+                'type': 'webpage',
+                'summary': '',
+                "document_meta": {
+                    "language": "en",
+                    "title": page_title,
+                },
+            }
+            docs = []
+            if md_text:
+                docs.append(
+                    Document(
+                        page_content=md_text,
+                        metadata=metadata
+                    )
+                )
+                # for chunk in self._mark_splitter.split_text(md_text):
+                #     docs.append(
+                #         Document(
+                #             page_content=chunk,
+                #             metadata=metadata
+                #         )
+                #     )
+            if content:
+                site_content = [
+                    Document(
+                        page_content=paragraph,
+                        metadata=metadata
+                    ) for paragraph in content
+                ]
+            return docs + site_content
+        except Exception as exc:
+            print(f"Failed to load {url}: {exc}")
+
+    def load(self, **kwargs) -> list:
+        documents = []
+        for address in self.urls:
+            docs = self.get(address)
+            if docs:
+                documents.extend(docs)
+        self.driver.quit()
+        return self.split_by_tokens(documents)
+
+    def parse(self, source):
+        pass

parrot/loaders/web_base.py

@@ -0,0 +1,112 @@
+import asyncio
+import aiohttp
+from markdownify import MarkdownConverter
+from bs4 import BeautifulSoup as bs
+from langchain.text_splitter import MarkdownTextSplitter
+from langchain.docstore.document import Document
+from .abstract import AbstractLoader
+
+
+class WebBaseLoader(AbstractLoader):
+    """Class to load web pages and extract text as Markdown."""
+    def __init__(self, urls: dict, source_type: str = 'website', **kwargs):
+        self.urls = urls
+        self._source_type = source_type
+        self.timeout: int = kwargs.pop('timeout', 60)
+        self._wait: int = kwargs.pop('wait', 60)
+        super().__init__(source_type=source_type, **kwargs)
+        self.md_splitter = MarkdownTextSplitter(
+            chunk_size = 1024,
+            chunk_overlap=10
+        )
+
+    def md(self, soup, **options):
+        return MarkdownConverter(**options).convert_soup(soup)
+
+    async def _fetch(
+        self, url: str, retries: int = 3, cooldown: int = 2, backoff: float = 1.5
+    ) -> str:
+        async with aiohttp.ClientSession() as session:
+            for i in range(retries):
+                try:
+                    async with session.get(url, allow_redirects=True) as response:
+                        if response.status == 200:
+                            return await response.text()
+                        elif response.status == 429:
+                            # Too many requests
+                            if i == retries - 1:
+                                return ''
+                            self.logger.warning(
+                                f"Too many requests to {url}. Waiting {self._wait} seconds."
+                            )
+                            await asyncio.sleep(self._wait)
+                        else:  # Other non-success status codes
+                            response.raise_for_status()  # Raise for other errors
+                except aiohttp.ClientConnectionError as e:
+                    if i == retries - 1:
+                        raise
+                    else:
+                        self.logger.warning(
+                            f"Error fetching {url} with attempt "
+                            f"{i + 1}/{retries}: {e}. Retrying..."
+                        )
+                        await asyncio.sleep(cooldown * backoff**i)
+                except aiohttp.ClientResponseError as e:
+                    self.logger.warning(
+                        f"Request failed (ClientResponseError): {e}"
+                    )
+                    return ''
+        self.logger.warning(f"Failed to fetch {url} after {retries} attempts.")
+        return ''
+
+    async def get_address(self, url: str) -> list:
+        self.logger.info(
+            f'Downloading URL {url}'
+        )
+        html = await self._fetch(url)
+        docs = []
+        if html:
+            soup = bs(html, 'html.parser')
+            md_text = self.md(soup)
+            try:
+                title = soup.title.string
+            except AttributeError:
+                title = None
+            metadata = {
+                "url": url,
+                "source": url,
+                "index": "",
+                "filename": '',
+                "type": 'webpage',
+                "question": '',
+                "answer": '',
+                "source_type": self._source_type,
+                "summary": "",
+                "document_meta": {
+                    "title": title,
+                    "description": soup.find('meta', attrs={'name': 'description'})['content']
+                    if soup.find('meta', attrs={'name': 'description'}) else '',
+                    "keywords": soup.find('meta', attrs={'name': 'keywords'})['content']
+                    if soup.find('meta', attrs={'name': 'keywords'}) else '',
+                }
+            }
+            for chunk in self.md_splitter.split_text(md_text):
+                docs.append(
+                    Document(
+                        page_content=chunk,
+                        metadata=metadata
+                    )
+                )
+        return docs
+
+    async def load(self, **kwargs) -> list:
+        documents = []
+        if self.urls is None:
+            return []
+        for address in self.urls:
+            docs = await self.get_address(address)
+            documents.extend(docs)
+        return self.split_documents(documents)
+
+    def parse(self, source):
+        pass

parrot/loaders/word.py

@@ -0,0 +1,125 @@
+from collections.abc import Callable
+from pathlib import Path, PurePath
+from typing import Union
+import mammoth
+import docx
+from markdownify import markdownify as md
+from langchain_community.document_loaders.word_document import (
+    UnstructuredWordDocumentLoader
+)
+from langchain.text_splitter import MarkdownTextSplitter
+from langchain.docstore.document import Document
+from .abstract import AbstractLoader
+
+
+class MSWordLoader(AbstractLoader):
+    """
+    Loader for Microsoft Word files.
+    """
+    _extension: list = ['.docx', '.doc']
+
+    def __init__(
+        self,
+        path: PurePath,
+        tokenizer: Union[str, Callable] = None,
+        text_splitter: Union[str, Callable] = None,
+        source_type: str = 'document',
+        **kwargs
+    ):
+        self.path = path
+        if isinstance(path, str):
+            path = Path(path)
+        self._use_mammoth: bool = kwargs.pop('use_mammoth', True)
+        self._md_splitter = MarkdownTextSplitter(chunk_size = 1024, chunk_overlap=10)
+        super().__init__(
+            tokenizer=tokenizer,
+            text_splitter=text_splitter,
+            source_type=source_type,
+            **kwargs
+        )
+
+    def _load_document(self, path: PurePath) -> list:
+        if self._check_path(path):
+            docs = []
+            self.logger.info(f"Loading Word file: {path}")
+            if self._use_mammoth:
+                with open(path, "rb") as docx_file:
+                    doc = docx.Document(str(path))
+                    prop = doc.core_properties
+                    result = mammoth.convert_to_html(docx_file)
+                    # result = mammoth.extract_raw_text(docx_file)
+                    html = result.value # The generated HTML
+                    md_text = md(html) # The generated Markdown
+                    try:
+                        summary = self.get_summary_from_text(md_text)
+                    except ValueError:
+                        summary = ''
+                    metadata = {
+                        "url": '',
+                        "source": path.name,
+                        "filename": path.name,
+                        "index": str(path.name),
+                        "type": 'document',
+                        "question": '',
+                        "answer": '',
+                        "source_type": self._source_type,
+                        "data": {},
+                        "summary": summary,
+                        "document_meta": {
+                            "author": prop.author,
+                            "version": prop.version,
+                            "title": prop.title,
+                            "created": prop.created.strftime("%Y-%m-%d %H:%M:%S"),
+                            "last_modified": prop.modified.strftime("%Y-%m-%d %H:%M:%S")
+                        }
+                    }
+                    for chunk in self._md_splitter.split_text(md_text):
+                        _idx = {
+                            **metadata
+                        }
+                        docs.append(
+                            Document(
+                                page_content=chunk,
+                                metadata=_idx
+                            )
+                        )
+                return docs
+            else:
+                word_loader = UnstructuredWordDocumentLoader(
+                    file_path=str(path)
+                )
+                docs = word_loader.load()
+                for doc in docs:
+                    # Fix This
+                    doc.metadata['source_type'] = self._source_type
+        return []
+
+    def load(self) -> list:
+        """
+        Load data from a DOCX file.
+
+        Args:
+            source (str): The path to the DOCX file.
+
+        Returns:
+            list: A list of Langchain Documents.
+        """
+        if not self.path.exists():
+            raise FileNotFoundError(f"DOCX file/directory not found: {self.path}")
+        if self.path.is_dir():
+            documents = []
+            # iterate over the files in the directory
+            for ext in self._extension:
+                for item in self.path.glob(f'*{ext}'):
+                    documents.extend(self._load_document(item))
+        elif self.path.is_file():
+            documents = self._load_document(self.path)
+        else:
+            raise ValueError(
+                f"DOCX Loader: Invalid path: {self.path}"
+            )
+        # return documents
+        return self.split_documents(documents)
+
+    def parse(self, source):
+        pass