ai-parrot 0.1.0__cp311-cp311-manylinux_2_28_x86_64.whl

parrot/loaders/rtd.py

@@ -0,0 +1,65 @@
+from collections.abc import Callable
+from pathlib import PurePath
+from typing import Any
+import re
+from langchain_community.document_loaders import ReadTheDocsLoader as RTLoader
+from .abstract import AbstractLoader
+
+
+class ReadTheDocsLoader(AbstractLoader):
+    """
+    Loading documents from ReadTheDocs.
+    """
+    _extension: list = []
+
+    def __init__(
+        self,
+        path: PurePath,
+        tokenizer: Callable[..., Any] = None,
+        text_splitter: Callable[..., Any] = None,
+        source_type: str = 'documentation',
+        encoding: str = 'utf-8',
+        origin: str = '',
+        **kwargs
+    ):
+        super().__init__(tokenizer, text_splitter, source_type=source_type, **kwargs)
+        self.path = path
+        self.encoding = encoding
+        self.origin = origin
+        self._prefix = ''
+        match = re.search(r'://([^/]+)', origin)
+        if match:
+            self._prefix = match.group(1)
+
+    def load(self) -> list:
+        documents = []
+        self.logger.info(
+            f"Loading ReadTheDocs from: {self.path}"
+        )
+        rt_loader = RTLoader(path=self.path, encoding=self.encoding)
+        documents = rt_loader.load()
+        for doc in documents:
+            src = doc.metadata.get('source')
+            suffix = src.replace(f'{self.path}', '').replace(self._prefix, '')
+            if suffix.startswith('//'):
+                suffix = suffix[1:]
+            url = f"{self.origin}{suffix}"
+            metadata = {
+                "source": url,
+                "url": url,
+                "index": suffix,
+                "filename": src,
+                "question": '',
+                "answer": '',
+                'type': 'documentation',
+                "source_type": self._source_type,
+                "summary": '',
+                "document_meta": {
+                    **doc.metadata
+                }
+            }
+            doc.metadata = metadata
+        return documents
+
+    def parse(self, source):
+        pass

parrot/loaders/txt.py

@@ -0,0 +1,92 @@
+from typing import Any
+from collections.abc import Callable
+from pathlib import Path, PurePath
+from langchain_community.document_loaders import TextLoader
+from langchain.docstore.document import Document
+from .abstract import AbstractLoader
+
+
+class TXTLoader(AbstractLoader):
+    """
+    Loader for PDF files.
+    """
+    _extension = ['.txt']
+
+    def __init__(
+        self,
+        path: PurePath,
+        tokenizer: Callable[..., Any] = None,
+        text_splitter: Callable[..., Any] = None,
+        source_type: str = 'text',
+        **kwargs
+    ):
+        super().__init__(tokenizer, text_splitter, source_type=source_type, **kwargs)
+        self.path = path
+        if isinstance(path, str):
+            self.path = Path(path).resolve()
+
+    def _load_document(self, path: Path) -> list:
+        """
+        Load a TXT file.
+
+        Args:
+            path (Path): The path to the TXT file.
+
+        Returns:
+            list: A list of Langchain Documents.
+        """
+        if self._check_path(path):
+            self.logger.info(f"Loading TXT file: {path}")
+            with open(path, 'r') as file:
+                text = file.read()
+            return [
+                Document(
+                    page_content=text,
+                    metadata={
+                        "url": '',
+                        "index": str(path.name),
+                        "source": str(path),
+                        "filename": str(path.name),
+                        "summary": '',
+                        "question": '',
+                        "answer": '',
+                        'type': 'text',
+                        "source_type": self._source_type,
+                        "document_meta": {}
+                    }
+                )
+            ]
+        return []
+
+    def load(self) -> list:
+        """
+        Load data from a TXT file.
+
+        Args:
+            source (str): The path to the TXT file.
+
+        Returns:
+            list: A list of Langchain Documents.
+        """
+        if not self.path.exists():
+            raise FileNotFoundError(
+                f"File/directory not found: {self.path}"
+            )
+        if self.path.is_dir():
+            documents = []
+            # iterate over the files in the directory
+            for ext in self._extension:
+                for item in self.path.glob(f'*{ext}'):
+                    documents.extend(self._load_document(item))
+        elif self.path.is_file():
+            documents = self._load_document(self.path)
+        else:
+            raise ValueError(
+                f"TXT Loader: Invalid path: {self.path}"
+            )
+        return self.split_documents(documents)
+
+    def parse(self, source):
+        raise NotImplementedError(
+            "Parser method is not implemented for TXTLoader."
+        )

parrot/loaders/utils/__init__.py

@@ -0,0 +1 @@
+from .models import BotData

parrot/loaders/utils/models.py

@@ -0,0 +1,25 @@
+from typing import Optional, Union
+from datetime import datetime
+from pathlib import PurePath
+from datamodel import BaseModel, Field
+
+def uuid_to_str(obj) -> str:
+    return str(obj)
+
+class BotData(BaseModel):
+    chatbot_id: str = Field(primary_key=True, required=True, encoder=uuid_to_str)
+    name: str = Field(required=True)
+    source_type: str = Field(required=True, default='content')
+    category: str = Field(required=True, default='data')
+    tags: Optional[list[str]] = Field(required=False, default_factory=list)
+    document_type: str = Field(required=False, default='document')
+    loader: str = Field(required=True, default='TXTLoader')
+    source_path: Union[str,PurePath] = Field(required=False)
+    extensions: list[str] = Field(required=False)
+    data: Optional[Union[list,dict]] = Field(required=False)
+    arguments: Optional[dict] = Field(default_factory=dict)
+    version: int = Field(required=True, default=1)
+    updated_at: datetime = Field(required=False, default=datetime.now)
+
+    class Meta:
+        name: str = 'chatbots_data'

parrot/loaders/video.py

@@ -0,0 +1,96 @@
+from collections.abc import Callable
+from typing import Any, Union, List
+from abc import abstractmethod
+from pathlib import Path
+import subprocess
+from .basevideo import BaseVideoLoader
+
+
+class VideoLoader(BaseVideoLoader):
+    """
+    Generating Video transcripts from Videos.
+    """
+    _extension = ['.youtube']
+    encoding = 'utf-8'
+    chunk_size = 768
+
+    def __init__(
+        self,
+        urls: List[str],
+        tokenizer: Callable[..., Any] = None,
+        text_splitter: Callable[..., Any] = None,
+        source_type: str = 'video',
+        language: str = "en",
+        video_path: Union[str, Path] = None,
+        **kwargs
+    ):
+        super().__init__(
+            urls,
+            tokenizer,
+            text_splitter,
+            source_type,
+            language=language,
+            video_path=video_path,
+            **kwargs
+        )
+
+    def download_video(self, url: str, path: str) -> Path:
+        """
+        Downloads a video from a URL using yt-dlp.
+
+        Args:
+            video_url (str): The URL of the video to download.
+            output_path (str): The directory where the video will be saved.
+        """
+        command = [
+            "yt-dlp",
+            "--get-filename",
+            url
+        ]
+        try:
+            result = subprocess.run(command, check=True, stdout=subprocess.PIPE, text=True)
+            filename = result.stdout.strip()  # Remove any trailing newline characters
+            file_path = path.joinpath(filename)
+            if file_path.exists():
+                print(f"Video already downloaded: {filename}")
+                return file_path
+            print(f"Downloading video: {filename}")
+            # after extracted filename, download the video
+            command = [
+                "yt-dlp",
+                url,
+                "-o",
+                str(file_path)
+            ]
+            subprocess.run(command, check=True)
+            return file_path
+        except subprocess.CalledProcessError as e:
+            print(f"Error downloading video: {e}")
+
+
+    def load(self) -> list:
+        documents = []
+        for url in self.urls:
+            transcript = None
+            if isinstance(url, dict):
+                path = list(url.keys())[0]
+                parts = url[path]
+                if isinstance(parts, str):
+                    video_title = parts
+                elif isinstance(parts, dict):
+                    video_title = parts['title']
+                    transcript = parts.get('transcript', None)
+                url = path
+            else:
+                video_title = url
+            docs = self.load_video(url, video_title, transcript)
+            documents.extend(docs)
+        # return documents
+        return self.split_documents(documents)
+
+    @abstractmethod
+    def load_video(self, url: str, video_title: str, transcript: str) -> list:
+        pass
+
+    def parse(self, source):
+        pass

parrot/loaders/videolocal.py

@@ -0,0 +1,107 @@
+from typing import Any
+from collections.abc import Callable
+from pathlib import PurePath
+from langchain.docstore.document import Document
+from .basevideo import BaseVideoLoader
+
+
+class VideoLocalLoader(BaseVideoLoader):
+    """
+    Generating Video transcripts from local Videos.
+    """
+    _extension = ['.mp4', '.webm']
+
+    def __init__(
+        self,
+        path: PurePath,
+        tokenizer: Callable[..., Any] = None,
+        text_splitter: Callable[..., Any] = None,
+        source_type: str = 'documentation',
+        encoding: str = 'utf-8',
+        origin: str = '',
+        **kwargs
+    ):
+        super().__init__(tokenizer, text_splitter, source_type=source_type, **kwargs)
+        self.path = path
+
+    def load_video(self, path: PurePath) -> list:
+        metadata = {
+            "source": f"{path}",
+            "url": f"{path.name}",
+            "index": path.stem,
+            "filename": f"{path}",
+            "question": '',
+            "answer": '',
+            'type': 'video_transcript',
+            "source_type": self._source_type,
+            "summary": '',
+            "document_meta": {
+                "language": self._language,
+                "topic_tags": ""
+            }
+        }
+        documents = []
+        transcript_path = path.with_suffix('.vtt')
+        audio_path = path.with_suffix('.mp3')
+        # second: extract audio from File
+        self.extract_audio(path, audio_path)
+        # get the Whisper parser
+        transcript_whisper = self.get_whisper_transcript(audio_path)
+        if transcript_whisper:
+            transcript = transcript_whisper['text']
+        else:
+            transcript = ''
+        # Summarize the transcript
+        if transcript:
+            summary = self.get_summary_from_text(transcript)
+            # Create Two Documents, one is for transcript, second is VTT:
+            metadata['summary'] = summary
+            doc = Document(
+                page_content=transcript,
+                metadata=metadata
+            )
+            documents.append(doc)
+        if transcript_whisper:
+            # VTT version:
+            transcript = self.transcript_to_vtt(transcript_whisper, transcript_path)
+            doc = Document(
+                page_content=transcript,
+                metadata=metadata
+            )
+            documents.append(doc)
+            # Saving every dialog chunk as a separate document
+            dialogs = self.transcript_to_blocks(transcript_whisper)
+            docs = []
+            for chunk in dialogs:
+                _meta = {
+                    "index": f"{path.stem}:{chunk['id']}",
+                    "document_meta": {
+                        "start": f"{chunk['start_time']}",
+                        "end": f"{chunk['end_time']}",
+                        "id": f"{chunk['id']}",
+                        "language": self._language,
+                        "title": f"{path.stem}",
+                        "topic_tags": ""
+                    }
+                }
+                _info = {**metadata, **_meta}
+                doc = Document(
+                    page_content=chunk['text'],
+                    metadata=_info
+                )
+                docs.append(doc)
+            documents.extend(docs)
+        return documents
+
+    def load(self) -> list:
+        documents = []
+        if self.path.is_file():
+            docs = self.load_video(self.path)
+            documents.extend(docs)
+        if self.path.is_dir():
+            # iterate over the files in the directory
+            for ext in self._extension:
+                for item in self.path.glob(f'*{ext}'):
+                    if set(item.parts).isdisjoint(self.skip_directories):
+                        documents.extend(self.load_video(item))
+        return self.split_documents(documents)

parrot/loaders/vimeo.py

@@ -0,0 +1,106 @@
+from typing import Optional, Union
+from transformers import pipeline
+import torch
+from langchain.docstore.document import Document
+from .youtube import YoutubeLoader
+
+
+class VimeoLoader(YoutubeLoader):
+    """
+    Loader for Vimeo videos.
+    """
+    def load_video(self, url: str, video_title: str, transcript: Optional[Union[str, None]] = None) -> list:
+        metadata = {
+            "source": url,
+            "url": url,
+            "index": url,
+            "filename": video_title,
+            "question": '',
+            "answer": '',
+            'type': 'video_transcript',
+            "source_type": self._source_type,
+            "summary": '',
+            "document_meta": {
+                "language": self._language,
+                "title": video_title,
+                "topic_tags": ""
+            }
+        }
+        if self.topics:
+            metadata['document_meta']['topic_tags'] = self.topics
+        if transcript is None:
+            documents = []
+            docs = []
+            # first: download video
+            try:
+                file_path = self.download_video(url, self._video_path)
+            except Exception:
+                return []
+            if not file_path:
+                self.logger.warning(
+                    f"Error downloading File for video: {self._video_path}"
+                )
+                return []
+            transcript_path = file_path.with_suffix('.vtt')
+            audio_path = file_path.with_suffix('.mp3')
+            # second: extract audio
+            self.extract_audio(file_path, audio_path)
+            # get the Whisper parser
+            transcript_whisper = self.get_whisper_transcript(audio_path)
+            if transcript_whisper:
+                transcript = transcript_whisper['text']
+            else:
+                transcript = ''
+            # Summarize the transcript
+            if transcript:
+                summary = self.get_summary_from_text(transcript)
+                # Create Two Documents, one is for transcript, second is VTT:
+                metadata['summary'] = summary
+                doc = Document(
+                    page_content=transcript,
+                    metadata=metadata
+                )
+                documents.append(doc)
+            if transcript_whisper:
+                # VTT version:
+                transcript = self.transcript_to_vtt(transcript_whisper, transcript_path)
+                doc = Document(
+                    page_content=transcript,
+                    metadata=metadata
+                )
+                documents.append(doc)
+                # Saving every dialog chunk as a separate document
+                dialogs = self.transcript_to_blocks(transcript_whisper)
+                for chunk in dialogs:
+                    _meta = {
+                        "index": f"{video_title}:{chunk['id']}",
+                        "document_meta": {
+                            "start": f"{chunk['start_time']}",
+                            "end": f"{chunk['end_time']}",
+                            "id": f"{chunk['id']}",
+                            "language": self._language,
+                            "title": video_title,
+                            "topic_tags": ""
+                        }
+                    }
+                    _info = {**metadata, **_meta}
+                    doc = Document(
+                        page_content=chunk['text'],
+                        metadata=_info
+                    )
+                    docs.append(doc)
+            documents.extend(docs)
+            return self.split_documents(documents)
+        else:
+            # using the transcript file
+            with open(transcript, 'r') as f:
+                transcript = f.read()
+                summary = self.get_summary_from_text(transcript)
+                transcript_whisper = None
+            metadata['summary'] = f"{summary!s}"
+            # Create Two Documents, one is for transcript, second is VTT:
+            doc = Document(
+                page_content=transcript,
+                metadata=metadata
+            )
+            return self.split_documents([doc])