ai-parrot 0.3.4__cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl

parrot/loaders/ppt.py

@@ -0,0 +1,30 @@
+from pathlib import PurePath
+from langchain_community.document_loaders import (
+    UnstructuredPowerPointLoader
+)
+from .abstract import AbstractLoader
+
+
+class PPTXLoader(AbstractLoader):
+    """
+    Loader for PPTX files.
+    """
+    _extension: list = ['.pptx']
+
+    def load(self, path: PurePath) -> list:
+        if self._check_path(path):
+            docs = []
+            self.logger.info(f"Loading PPTX file: {path}")
+            ppt_loader = UnstructuredPowerPointLoader(
+                file_path=str(path)
+            )
+            docs += ppt_loader.load()
+            for doc in docs:
+                doc.metadata['source_type'] = self._source_type
+            # Split the documents into chunks
+            return self.split_documents(docs)
+        else:
+            return []
+
+    def parse(self, source):
+        pass

parrot/loaders/qa.py

@@ -0,0 +1,81 @@
+
+from pathlib import Path, PurePath
+from typing import Any
+from collections.abc import Callable
+import pandas as pd
+from langchain.docstore.document import Document
+from .abstract import AbstractLoader
+
+
+class QAFileLoader(AbstractLoader):
+    """
+    Question and Answers File based on Excel.
+    """
+    _extension = ['.xlsx']
+    chunk_size = 768
+
+    def __init__(
+        self,
+        path: PurePath,
+        tokenizer: Callable[..., Any] = None,
+        text_splitter: Callable[..., Any] = None,
+        source_type: str = 'QA',
+        columns: list = ['Question', 'Answer'],
+        **kwargs
+    ):
+        super().__init__(tokenizer, text_splitter, source_type, **kwargs)
+        self.path = path
+        self._columns = columns
+        if isinstance(path, str):
+            self.path = Path(path).resolve()
+        if self.path.is_dir():
+            raise ValueError(
+                f"Currently only accepting single Files."
+            )
+
+    def _load_document(self, path: PurePath) -> list:
+        if path.exists():
+            print('Load QA Excel File: ', path)
+            df = pd.read_excel(path)
+            q = self._columns[0]
+            a = self._columns[1]
+            docs = []
+            for idx, row in df.iterrows():
+                # Question Document
+                doc = Document(
+                    page_content=f"**Question:** {row[q]}: **Answer:** {row[a]}",
+                    metadata={
+                        "url": '',
+                        "index": f"{path.name} #{idx}",
+                        "source": f"{path.name} Row.#{idx}",
+                        "filename": f"{path.name}",
+                        "question": row[q],
+                        "answer": row[a],
+                        "page_number": idx,
+                        "source_type": self._source_type,
+                        "type": "QA",
+                        "summary": f"Question: {row[q]}?: **{row[a]}**",
+                        "document_meta": {
+                            "question": row[q],
+                            "answer": row[a],
+                        }
+                    }
+                )
+                docs.append(doc)
+            return docs
+        return []
+
+    def load(self, **kwargs) -> list:
+        """
+        Load Chapters from a PDF file.
+
+        Returns:
+            list: A list of Langchain Documents.
+        """
+        if self.path.is_file():
+            documents = self._load_document(path=self.path, **kwargs)
+            # after all documents are retrieved, procesed and stored
+            return self.split_documents(documents)
+
+    def parse(self, source):
+        pass

parrot/loaders/repo.py

@@ -0,0 +1,103 @@
+from pathlib import PurePath
+from langchain_core.document_loaders.blob_loaders import Blob
+from langchain_community.document_loaders.generic import GenericLoader
+from langchain_community.document_loaders.parsers import LanguageParser
+from langchain_community.document_loaders import (
+    DirectoryLoader,
+    TextLoader,
+    JSONLoader
+)
+from langchain_text_splitters import Language
+from langchain.text_splitter import (
+    RecursiveCharacterTextSplitter
+)
+from .abstract import AbstractLoader
+
+
+class RepositoryLoader(AbstractLoader):
+    """Repository (Code Directory) loader.
+    """
+    exclude_paths: list = [
+        ".venv/**",
+        ".venv/**/**/*",
+        ".git/**",
+        "node_modules/**",
+        "build/**",
+        "dist/**",
+        "templates/**",
+        "tmp/**"
+    ]
+
+    def load(self, path: PurePath, lang: str = 'python', excludes: list = []) -> list:
+        """
+        Load data from a repository and return it as a Langchain Document.
+        """
+        if isinstance(path, str):
+            path = PurePath(path)
+        if excludes:
+            self.exclude_paths += excludes
+        excludes_path = [
+            str(path.joinpath(p).resolve()) for p in self.exclude_paths
+        ]
+        if lang == 'python':
+            parser = LanguageParser(language=Language.PYTHON, parser_threshold=100)
+            splitter = RecursiveCharacterTextSplitter.from_language(
+                language=Language.PYTHON, chunk_size=1024, chunk_overlap=200
+            )
+            suffixes = [".py", ".pyx"]
+            glob = "**/[!.]*.py?"
+        elif lang == 'javascript':
+            parser = LanguageParser(language=Language.JS, parser_threshold=100)
+            splitter = RecursiveCharacterTextSplitter.from_language(
+                language=Language.JS, chunk_size=1024, chunk_overlap=200
+            )
+            suffixes = [".js", ".jsx", ".json", ".ts", ".tsx"]
+        elif lang == 'typescript':
+            parser = LanguageParser(language=Language.TS, parser_threshold=100)
+            splitter = RecursiveCharacterTextSplitter.from_language(
+                language=Language.TS, chunk_size=1024, chunk_overlap=200
+            )
+            suffixes = [".js", ".jsx", ".json", ".ts", ".tsx"]
+        elif lang == 'json':
+            loader = DirectoryLoader(
+                path,
+                glob="**/*.json",
+                show_progress=True,
+                exclude=excludes_path,
+                silent_errors=True,
+                recursive=True,
+                # loader_cls=TextLoader,
+                loader_cls=JSONLoader,
+                loader_kwargs={
+                    'jq_schema': '.',
+                    'text_content': False
+                }
+            )
+            docs = loader.load()
+            for doc in docs:
+                doc.metadata['url'] = ''
+                doc.metadata['source_type'] = self._source_type
+                doc.metadata['language'] = lang
+            return self.text_splitter.split_documents(docs)
+        else:
+            raise ValueError(
+                f"Language {lang} not supported for Repository"
+            )
+        loader = GenericLoader.from_filesystem(
+            path,
+            glob=glob,
+            suffixes=suffixes,
+            exclude=self.exclude_paths,
+            parser=parser,
+            show_progress=True
+        )
+        docs = loader.load()
+        for doc in docs:
+            doc.metadata['url'] = ''
+            doc.metadata['source_type'] = self._source_type
+            doc.metadata['language'] = lang
+        documents = splitter.split_documents(docs)
+        return documents
+
+    def parse(self, source):
+        raise NotImplementedError("Parser method is not implemented for PDFLoader.")

parrot/loaders/rtd.py

@@ -0,0 +1,65 @@
+from collections.abc import Callable
+from pathlib import PurePath
+from typing import Any
+import re
+from langchain_community.document_loaders import ReadTheDocsLoader as RTLoader
+from .abstract import AbstractLoader
+
+
+class ReadTheDocsLoader(AbstractLoader):
+    """
+    Loading documents from ReadTheDocs.
+    """
+    _extension: list = []
+
+    def __init__(
+        self,
+        path: PurePath,
+        tokenizer: Callable[..., Any] = None,
+        text_splitter: Callable[..., Any] = None,
+        source_type: str = 'documentation',
+        encoding: str = 'utf-8',
+        origin: str = '',
+        **kwargs
+    ):
+        super().__init__(tokenizer, text_splitter, source_type=source_type, **kwargs)
+        self.path = path
+        self.encoding = encoding
+        self.origin = origin
+        self._prefix = ''
+        match = re.search(r'://([^/]+)', origin)
+        if match:
+            self._prefix = match.group(1)
+
+    def load(self) -> list:
+        documents = []
+        self.logger.info(
+            f"Loading ReadTheDocs from: {self.path}"
+        )
+        rt_loader = RTLoader(path=self.path, encoding=self.encoding)
+        documents = rt_loader.load()
+        for doc in documents:
+            src = doc.metadata.get('source')
+            suffix = src.replace(f'{self.path}', '').replace(self._prefix, '')
+            if suffix.startswith('//'):
+                suffix = suffix[1:]
+            url = f"{self.origin}{suffix}"
+            metadata = {
+                "source": url,
+                "url": url,
+                "index": suffix,
+                "filename": src,
+                "question": '',
+                "answer": '',
+                'type': 'documentation',
+                "source_type": self._source_type,
+                "summary": '',
+                "document_meta": {
+                    **doc.metadata
+                }
+            }
+            doc.metadata = metadata
+        return documents
+
+    def parse(self, source):
+        pass

parrot/loaders/txt.py

@@ -0,0 +1,92 @@
+from typing import Any
+from collections.abc import Callable
+from pathlib import Path, PurePath
+from langchain_community.document_loaders import TextLoader
+from langchain.docstore.document import Document
+from .abstract import AbstractLoader
+
+
+class TXTLoader(AbstractLoader):
+    """
+    Loader for PDF files.
+    """
+    _extension = ['.txt']
+
+    def __init__(
+        self,
+        path: PurePath,
+        tokenizer: Callable[..., Any] = None,
+        text_splitter: Callable[..., Any] = None,
+        source_type: str = 'text',
+        **kwargs
+    ):
+        super().__init__(tokenizer, text_splitter, source_type=source_type, **kwargs)
+        self.path = path
+        if isinstance(path, str):
+            self.path = Path(path).resolve()
+
+    def _load_document(self, path: Path) -> list:
+        """
+        Load a TXT file.
+
+        Args:
+            path (Path): The path to the TXT file.
+
+        Returns:
+            list: A list of Langchain Documents.
+        """
+        if self._check_path(path):
+            self.logger.info(f"Loading TXT file: {path}")
+            with open(path, 'r') as file:
+                text = file.read()
+            return [
+                Document(
+                    page_content=text,
+                    metadata={
+                        "url": '',
+                        "index": str(path.name),
+                        "source": str(path),
+                        "filename": str(path.name),
+                        "summary": '',
+                        "question": '',
+                        "answer": '',
+                        'type': 'text',
+                        "source_type": self._source_type,
+                        "document_meta": {}
+                    }
+                )
+            ]
+        return []
+
+    def load(self) -> list:
+        """
+        Load data from a TXT file.
+
+        Args:
+            source (str): The path to the TXT file.
+
+        Returns:
+            list: A list of Langchain Documents.
+        """
+        if not self.path.exists():
+            raise FileNotFoundError(
+                f"File/directory not found: {self.path}"
+            )
+        if self.path.is_dir():
+            documents = []
+            # iterate over the files in the directory
+            for ext in self._extension:
+                for item in self.path.glob(f'*{ext}'):
+                    documents.extend(self._load_document(item))
+        elif self.path.is_file():
+            documents = self._load_document(self.path)
+        else:
+            raise ValueError(
+                f"TXT Loader: Invalid path: {self.path}"
+            )
+        return self.split_documents(documents)
+
+    def parse(self, source):
+        raise NotImplementedError(
+            "Parser method is not implemented for TXTLoader."
+        )

parrot/loaders/utils/__init__.py

@@ -0,0 +1 @@
+from .models import BotData

parrot/loaders/utils/models.py

@@ -0,0 +1,25 @@
+from typing import Optional, Union
+from datetime import datetime
+from pathlib import PurePath
+from datamodel import BaseModel, Field
+
+def uuid_to_str(obj) -> str:
+    return str(obj)
+
+class BotData(BaseModel):
+    chatbot_id: str = Field(primary_key=True, required=True, encoder=uuid_to_str)
+    name: str = Field(required=True)
+    source_type: str = Field(required=True, default='content')
+    category: str = Field(required=True, default='data')
+    tags: Optional[list[str]] = Field(required=False, default_factory=list)
+    document_type: str = Field(required=False, default='document')
+    loader: str = Field(required=True, default='TXTLoader')
+    source_path: Union[str,PurePath] = Field(required=False)
+    extensions: list[str] = Field(required=False)
+    data: Optional[Union[list,dict]] = Field(required=False)
+    arguments: Optional[dict] = Field(default_factory=dict)
+    version: int = Field(required=True, default=1)
+    updated_at: datetime = Field(required=False, default=datetime.now)
+
+    class Meta:
+        name: str = 'chatbots_data'

parrot/loaders/video.py

@@ -0,0 +1,96 @@
+from collections.abc import Callable
+from typing import Any, Union, List
+from abc import abstractmethod
+from pathlib import Path
+import subprocess
+from .basevideo import BaseVideoLoader
+
+
+class VideoLoader(BaseVideoLoader):
+    """
+    Generating Video transcripts from Videos.
+    """
+    _extension = ['.youtube']
+    encoding = 'utf-8'
+    chunk_size = 768
+
+    def __init__(
+        self,
+        urls: List[str],
+        tokenizer: Callable[..., Any] = None,
+        text_splitter: Callable[..., Any] = None,
+        source_type: str = 'video',
+        language: str = "en",
+        video_path: Union[str, Path] = None,
+        **kwargs
+    ):
+        super().__init__(
+            urls,
+            tokenizer,
+            text_splitter,
+            source_type,
+            language=language,
+            video_path=video_path,
+            **kwargs
+        )
+
+    def download_video(self, url: str, path: str) -> Path:
+        """
+        Downloads a video from a URL using yt-dlp.
+
+        Args:
+            video_url (str): The URL of the video to download.
+            output_path (str): The directory where the video will be saved.
+        """
+        command = [
+            "yt-dlp",
+            "--get-filename",
+            url
+        ]
+        try:
+            result = subprocess.run(command, check=True, stdout=subprocess.PIPE, text=True)
+            filename = result.stdout.strip()  # Remove any trailing newline characters
+            file_path = path.joinpath(filename)
+            if file_path.exists():
+                print(f"Video already downloaded: {filename}")
+                return file_path
+            print(f"Downloading video: {filename}")
+            # after extracted filename, download the video
+            command = [
+                "yt-dlp",
+                url,
+                "-o",
+                str(file_path)
+            ]
+            subprocess.run(command, check=True)
+            return file_path
+        except subprocess.CalledProcessError as e:
+            print(f"Error downloading video: {e}")
+
+
+    def load(self) -> list:
+        documents = []
+        for url in self.urls:
+            transcript = None
+            if isinstance(url, dict):
+                path = list(url.keys())[0]
+                parts = url[path]
+                if isinstance(parts, str):
+                    video_title = parts
+                elif isinstance(parts, dict):
+                    video_title = parts['title']
+                    transcript = parts.get('transcript', None)
+                url = path
+            else:
+                video_title = url
+            docs = self.load_video(url, video_title, transcript)
+            documents.extend(docs)
+        # return documents
+        return self.split_documents(documents)
+
+    @abstractmethod
+    def load_video(self, url: str, video_title: str, transcript: str) -> list:
+        pass
+
+    def parse(self, source):
+        pass

parrot/loaders/videolocal.py

@@ -0,0 +1,120 @@
+from typing import Any
+from collections.abc import Callable
+from pathlib import PurePath
+from langchain.docstore.document import Document
+from .basevideo import BaseVideoLoader
+
+
+class VideoLocalLoader(BaseVideoLoader):
+    """
+    Generating Video transcripts from local Videos.
+    """
+    _extension = ['.mp4', '.webm']
+
+    def __init__(
+        self,
+        path: PurePath,
+        tokenizer: Callable[..., Any] = None,
+        text_splitter: Callable[..., Any] = None,
+        source_type: str = 'documentation',
+        encoding: str = 'utf-8',
+        origin: str = '',
+        **kwargs
+    ):
+        super().__init__(tokenizer, text_splitter, source_type=source_type, **kwargs)
+        self.path = path
+
+    def load_video(self, path: PurePath) -> list:
+        metadata = {
+            "source": f"{path}",
+            "url": f"{path.name}",
+            "index": path.stem,
+            "filename": f"{path}",
+            "question": '',
+            "answer": '',
+            'type': 'video_transcript',
+            "source_type": self._source_type,
+            "summary": '',
+            "document_meta": {
+                "language": self._language,
+                "topic_tags": ""
+            }
+        }
+        documents = []
+        transcript_path = path.with_suffix('.vtt')
+        audio_path = path.with_suffix('.mp3')
+        # second: extract audio from File
+        self.extract_audio(path, audio_path)
+        # get the Whisper parser
+        transcript_whisper = self.get_whisper_transcript(audio_path)
+        if transcript_whisper:
+            transcript = transcript_whisper['text']
+        else:
+            transcript = ''
+        # Summarize the transcript
+        if transcript:
+            summary = self.get_summary_from_text(transcript)
+            # Create Two Documents, one is for transcript, second is VTT:
+            metadata['summary'] = summary
+            doc = Document(
+                page_content=transcript,
+                metadata=metadata
+            )
+            documents.append(doc)
+        if transcript_whisper:
+            # VTT version:
+            transcript = self.transcript_to_vtt(transcript_whisper, transcript_path)
+            doc = Document(
+                page_content=transcript,
+                metadata=metadata
+            )
+            documents.append(doc)
+            # Saving every dialog chunk as a separate document
+            dialogs = self.transcript_to_blocks(transcript_whisper)
+            docs = []
+            for chunk in dialogs:
+                _meta = {
+                    "index": f"{path.stem}:{chunk['id']}",
+                    "document_meta": {
+                        "start": f"{chunk['start_time']}",
+                        "end": f"{chunk['end_time']}",
+                        "id": f"{chunk['id']}",
+                        "language": self._language,
+                        "title": f"{path.stem}",
+                        "topic_tags": ""
+                    }
+                }
+                _info = {**metadata, **_meta}
+                doc = Document(
+                    page_content=chunk['text'],
+                    metadata=_info
+                )
+                docs.append(doc)
+            documents.extend(docs)
+        return documents
+
+    def load(self) -> list:
+        documents = []
+        if self.path.is_file():
+            docs = self.load_video(self.path)
+            documents.extend(docs)
+        if self.path.is_dir():
+            # iterate over the files in the directory
+            for ext in self._extension:
+                for item in self.path.glob(f'*{ext}'):
+                    if set(item.parts).isdisjoint(self.skip_directories):
+                        documents.extend(self.load_video(item))
+        return self.split_documents(documents)
+
+    def extract(self) -> list:
+        documents = []
+        if self.path.is_file():
+            docs = self.load_video(self.path)
+            documents.extend(docs)
+        if self.path.is_dir():
+            # iterate over the files in the directory
+            for ext in self._extension:
+                for item in self.path.glob(f'*{ext}'):
+                    if set(item.parts).isdisjoint(self.skip_directories):
+                        documents.extend(self.load_video(item))
+        return documents