pygpt-net 2.4.47__py3-none-any.whl

pygpt_net/provider/loaders/hub/image_vision/base.py

@@ -0,0 +1,229 @@
+import base64
+from pathlib import Path
+from typing import Dict, List, Optional
+
+from llama_index.core.readers.base import BaseReader
+from llama_index.core.schema import Document, ImageDocument
+from llama_index.core.utils import infer_torch_device
+
+
+class ImageVisionLLMReader(BaseReader):
+    """Image parser.
+
+    Caption image using Blip2 (a multimodal VisionLLM similar to GPT4).
+
+    Based on: https://github.com/run-llama/llama_index/blob/main/llama-index-integrations/readers/llama-index-readers-file/llama_index/readers/file/image_vision_llm/base.py
+
+    """
+
+    def __init__(
+        self,
+        use_local: bool = False,
+        window = None,
+        keep_image: bool = False,
+        api_prompt: str = "Describe what you see in this image",
+        api_model: str = "gpt-4-vision-preview",
+        api_tokens: int = 1000,
+        local_prompt: str = "Question: describe what you see in this image. Answer:",
+    ):
+        """
+        Init params.
+
+        :param use_local: use local model
+        :param window: Window instance
+        :param keep_image: keep image in document
+        :param api_prompt: API prompt
+        :param api_model: API model
+        :param api_tokens: API tokens
+        :param local_prompt: local prompt
+        """
+        self._initialized = False
+        self._use_local = use_local
+        self._api_prompt = api_prompt
+        self._api_model = api_model
+        self._api_tokens = api_tokens
+        self._window = window
+
+        self._parser_config = None
+        self._keep_image = keep_image
+        self._local_prompt = local_prompt
+
+    def _initialize(self) -> None:
+        """Initialize local parser."""
+        parser_config = None
+        if self._parser_config is None:
+            try:
+                import sentencepiece  # noqa
+                import torch
+                from PIL import Image  # noqa
+                from transformers import Blip2ForConditionalGeneration, Blip2Processor
+            except ImportError:
+                raise ImportError(
+                    "Please install extra dependencies that are required for "
+                    "the ImageVisionLLMReader: "
+                    "`pip install torch transformers sentencepiece Pillow`"
+                )
+
+            device = infer_torch_device()
+            dtype = torch.float16 if torch.cuda.is_available() else torch.float32
+            processor = Blip2Processor.from_pretrained("Salesforce/blip2-opt-2.7b")
+            model = Blip2ForConditionalGeneration.from_pretrained(
+                "Salesforce/blip2-opt-2.7b", torch_dtype=dtype
+            )
+            parser_config = {
+                "processor": processor,
+                "model": model,
+                "device": device,
+                "dtype": dtype,
+            }
+
+        self._parser_config = parser_config
+        self._initialized = True
+
+    def load_local(self, file: Path, extra_info: Optional[Dict] = None) -> List[Document]:
+        """
+        Describe image using local model.
+
+        :param file: file path
+        :param extra_info: additional metadata
+        :return: list of documents
+        """
+        if not self._initialized:
+            self._initialize()
+
+        from llama_index.core.img_utils import img_2_b64
+        from PIL import Image
+
+        # load document image
+        image = Image.open(file)
+        if image.mode != "RGB":
+            image = image.convert("RGB")
+
+        # Encode image into base64 string and keep in document
+        image_str: Optional[str] = None
+        if self._keep_image:
+            image_str = img_2_b64(image)
+
+        # Parse image into text
+        model = self._parser_config["model"]
+        processor = self._parser_config["processor"]
+
+        device = self._parser_config["device"]
+        dtype = self._parser_config["dtype"]
+        model.to(device)
+
+        # unconditional image captioning
+        inputs = processor(image, self._local_prompt, return_tensors="pt").to(device, dtype)
+        out = model.generate(**inputs)
+        text_str = processor.decode(out[0], skip_special_tokens=True)
+
+        return [
+            ImageDocument(
+                text=text_str,
+                image=image_str,
+                image_path=str(file),
+                metadata=extra_info or {},
+            )
+        ]
+
+    def load_api(self, file: Path, extra_info: Optional[Dict] = None) -> List[Document]:
+        """
+        Describe image using OpenAI API.
+
+        :param file: file path
+        :param extra_info: additional metadata
+        :return: list of documents
+        """
+        from llama_index.core.img_utils import img_2_b64
+        from PIL import Image
+
+        # Encode image into base64 string and keep in document
+        image_str: Optional[str] = None
+        if self._keep_image:
+            image = Image.open(file)
+            if image.mode != "RGB":
+                image = image.convert("RGB")
+            image_str = img_2_b64(image)
+
+        client = self._window.core.gpt.get_client()
+        encoded = self._encode_image(str(file))
+        content = [
+            {
+                "type": "text",
+                "text": self._api_prompt
+            },
+            {
+                "type": "image_url",
+                "image_url": {
+                    "url": f"data:image/jpeg;base64,{encoded}",
+                }
+            }
+        ]
+        messages = []
+        messages.append({"role": "user", "content": content})
+        response = client.chat.completions.create(
+            messages=messages,
+            model=self._api_model,
+            max_tokens=self._api_tokens,
+        )
+        text = response.choices[0].message.content.strip()
+        return [
+            ImageDocument(
+                text=text,
+                image=image_str,
+                image_path=str(file),
+                metadata=extra_info or {},
+            )
+        ]
+
+    def _encode_image(self, image_path: str) -> str:
+        """
+        Encode image to base64
+
+        :param image_path: path to image
+        :return: base64 encoded image
+        """
+        with open(image_path, "rb") as image_file:
+            return base64.b64encode(image_file.read()).decode('utf-8')
+
+    def load_data(
+        self, file: Path, extra_info: Optional[Dict] = None
+    ) -> List[Document]:
+        """
+        Parse file.
+
+        :param file: file path
+        :param extra_info: additional metadata
+        :return: list of documents
+        """
+        if self._use_local:
+            is_compiled = self._window.core.config.is_compiled() or self._window.core.platforms.is_snap()
+            if is_compiled:
+                raise ValueError("Local models are not available in compiled version.")
+            return self.load_local(file, extra_info)
+        else:
+            if self._window is None:
+                raise ValueError("Window instance is required to use API model.")
+            return self.load_api(file, extra_info)
+
+    def load_data_custom(
+        self, file: Path, extra_info: Optional[Dict] = None, **kwargs
+    ) -> List[Document]:
+        """
+        Parse file.
+
+        :param file: file path
+        :param extra_info: additional metadata
+        :param kwargs: additional arguments
+        :return: list of documents
+        """
+        extra_args = kwargs.get("extra_args", {})
+        if self._use_local:
+            is_compiled = self._window.core.config.is_compiled() or self._window.core.platforms.is_snap()
+            if is_compiled:
+                raise ValueError("Local models are not available in compiled version.")
+            return self.load_local(file, extra_info)
+        else:
+            if self._window is None:
+                raise ValueError("Window instance is required to use API model.")
+            return self.load_api(file, extra_info)

pygpt_net/provider/loaders/hub/json/base.py

@@ -0,0 +1,114 @@
+"""JSON Reader."""
+
+import json
+import re
+from pathlib import Path
+from typing import Dict, Generator, List, Optional
+
+from llama_index.core.readers.base import BaseReader
+from llama_index.core.schema import Document
+
+
+def _depth_first_yield(
+    json_data: Dict, levels_back: int, path: List[str]
+) -> Generator[str, None, None]:
+    """Do depth first yield of all of the leaf nodes of a JSON.
+
+    Combines keys in the JSON tree using spaces.
+
+    If levels_back is set to 0, prints all levels.
+
+    """
+    if isinstance(json_data, dict):
+        for key, value in json_data.items():
+            new_path = path[:]
+            new_path.append(key)
+            yield from _depth_first_yield(value, levels_back, new_path)
+    elif isinstance(json_data, list):
+        for _, value in enumerate(json_data):
+            yield from _depth_first_yield(value, levels_back, path)
+    else:
+        new_path = path[-levels_back:]
+        new_path.append(str(json_data))
+        yield " ".join(new_path)
+
+
+class JSONReader(BaseReader):
+    """JSON reader.
+
+    Reads JSON documents with options to help suss out relationships between nodes.
+
+    Args:
+        levels_back (int): the number of levels to go back in the JSON tree, 0
+        if you want all levels. If levels_back is None, then we just format the
+        JSON and make each line an embedding
+
+    """
+
+    def __init__(self, levels_back: Optional[int] = None) -> None:
+        """Initialize with arguments."""
+        super().__init__()
+        self.levels_back = levels_back
+
+    def _parse_jsonobj_to_document(
+        self, json_data_object: Dict, extra_info: Optional[Dict] = None
+    ) -> Document:
+        """Parse the json object into a Document.
+
+        Args:
+            json_data_object: The Json Object to be converted.
+            extra_info (Optional[Dict]): Additional information. Default is None.
+
+        Returns:
+            Document: The document.
+        """
+        if self.levels_back is None:
+            json_output = json.dumps(json_data_object, indent=0)
+            lines = json_output.split("\n")
+            useful_lines = [
+                line for line in lines if not re.match(r"^[{}\\[\\],]*$", line)
+            ]
+            return Document(text="\n".join(useful_lines), extra_info=extra_info or {})
+
+        else:
+            lines = [*_depth_first_yield(json_data_object, self.levels_back, [])]
+            return Document(text="\n".join(lines), extra_info=extra_info or {})
+
+    def load_data(
+        self,
+        file: Path,
+        is_jsonl: Optional[bool] = False,
+        extra_info: Optional[Dict] = None,
+    ) -> List[Document]:
+        """Load data from the input file.
+
+        Args:
+            file (Path): Path to the input file.
+            is_jsonl (Optional[bool]): If True, indicates that the file is in JSONL format. Defaults to False.
+            extra_info (Optional[Dict]): Additional information. Default is None.
+
+        Returns:
+            List[Document]: List of documents.
+        """
+        if not isinstance(file, Path):
+            file = Path(file)
+        with open(file, "r") as f:
+            data = []
+            if is_jsonl:
+                for line in f:
+                    data.append(json.loads(line.strip()))
+            else:
+                data = json.load(f)
+            documents = []
+
+            # For a dictionary JSON object, pass the entire data to be parsed as document
+            if isinstance(data, dict):
+                documents.append(self._parse_jsonobj_to_document(data, extra_info))
+            # For a List or Non-Dictionary JSON object loop through and pass each item
+            else:
+                for json_object in data:
+                    documents.append(
+                        self._parse_jsonobj_to_document(json_object, extra_info)
+                    )
+
+            return documents

pygpt_net/provider/loaders/hub/pandas_excel/base.py

@@ -0,0 +1,112 @@
+"""Pandas Excel reader.
+
+Pandas parser for .xlsx files.
+
+"""
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Union
+
+from llama_index.core.readers.base import BaseReader
+from llama_index.core.schema import Document
+
+
+class PandasExcelReader(BaseReader):
+    r"""Pandas-based CSV parser.
+
+    Parses CSVs using the separator detection from Pandas `read_csv`function.
+    If special parameters are required, use the `pandas_config` dict.
+
+    Args:
+
+        pandas_config (dict): Options for the `pandas.read_excel` function call.
+            Refer to https://pandas.pydata.org/docs/reference/api/pandas.read_excel.html
+            for more information. Set to empty dict by default, this means defaults will be used.
+
+    """
+
+    def __init__(
+        self,
+        *args: Any,
+        pandas_config: Optional[dict] = None,
+        concat_rows: bool = True,
+        row_joiner: str = "\n",
+        **kwargs: Any
+    ) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._pandas_config = pandas_config or {}
+        self._concat_rows = concat_rows
+        self._row_joiner = row_joiner if row_joiner else "\n"
+
+    def load_data(
+        self,
+        file: Path,
+        include_sheetname: bool = False,
+        sheet_name: Optional[Union[str, int, list]] = None,
+        extra_info: Optional[Dict] = None,
+        include_different_sheet_docs: bool = False,
+    ) -> List[Document]:
+        """Parse file and extract values from a specific column.
+
+        Args:
+            file (Path): The path to the Excel file to read.
+            include_sheetname (bool): Whether to include the sheet name in the output.
+            sheet_name (Union[str, int, None]): The specific sheet to read from, default is None which reads all sheets.
+            extra_info (Dict): Additional information to be added to the Document object.
+            include_different_sheet_docs: To join the document as a whole or create different docs for different sheets.
+
+        Returns:
+            List[Document]: A list of`Document objects containing the values from the specified column in the Excel file.
+        """
+
+        import pandas as pd
+
+        if sheet_name is not None:
+            sheet_name = (
+                [sheet_name] if not isinstance(sheet_name, list) else sheet_name
+            )
+
+        dfs = pd.read_excel(file, sheet_name=sheet_name, **self._pandas_config)
+
+        if include_different_sheet_docs:
+            documents = []
+            for sheet_name, df in dfs.items():
+                sheet_data = df.values.astype(str).tolist()
+                if self._concat_rows:
+                    text = self._row_joiner.join(
+                        self._row_joiner.join(row) for row in sheet_data
+                    )
+                else:
+                    text = [self._row_joiner.join(row) for row in sheet_data]
+
+                doc_extra_info = {"sheet_name": sheet_name}
+                if extra_info:
+                    doc_extra_info.update(extra_info)
+
+                documents.append(
+                    Document(
+                        text=text,
+                        extra_info=doc_extra_info,
+                    )
+                )
+            return documents
+        else:
+            all_sheets_data = []
+            for sheet_name, df in dfs.items():
+                if include_sheetname:
+                    all_sheets_data.append([sheet_name])
+                all_sheets_data.extend(df.values.astype(str).tolist())
+
+            if self._concat_rows:
+                text = self._row_joiner.join(
+                    self._row_joiner.join(row) for row in all_sheets_data
+                )
+            else:
+                text = [self._row_joiner.join(row) for row in all_sheets_data]
+
+            return [
+                Document(
+                    text=text,
+                    extra_info=extra_info or {},
+                )
+            ]

pygpt_net/provider/loaders/hub/simple_csv/base.py

@@ -0,0 +1,53 @@
+"""Simple CSV reader.
+
+A parser for tabular data files.
+
+"""
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from llama_index.core.readers.base import BaseReader
+from llama_index.core.schema import Document
+
+
+class SimpleCSVReader(BaseReader):
+    """CSV parser.
+
+    Args:
+        encoding (str): Encoding used to open the file.
+            utf-8 by default.
+        concat_rows (bool): whether to concatenate all rows into one document.
+            If set to False, a Document will be created for each row.
+            True by default.
+
+    """
+
+    def __init__(
+        self,
+        *args: Any,
+        concat_rows: bool = True,
+        encoding: str = "utf-8",
+        **kwargs: Any
+    ) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._concat_rows = concat_rows
+        self._encoding = encoding
+
+    def load_data(
+        self, file: Path, extra_info: Optional[Dict] = None
+    ) -> List[Document]:
+        """Parse file."""
+        import csv
+
+        text_list = []
+        with open(file, "r", encoding=self._encoding) as fp:
+            csv_reader = csv.reader(fp)
+            for row in csv_reader:
+                text_list.append(", ".join(row))
+        if self._concat_rows:
+            return [Document(text="\n".join(text_list), extra_info=extra_info or {})]
+        else:
+            return [
+                Document(text=text, extra_info=extra_info or {}) for text in text_list
+            ]

pygpt_net/provider/loaders/hub/video_audio/base.py

@@ -0,0 +1,161 @@
+"""Video audio parser.
+
+Contains parsers for mp3, mp4 files.
+
+Based on: https://github.com/run-llama/llama_index/blob/main/llama-index-integrations/readers/llama-index-readers-file/llama_index/readers/file/video_audio/base.py
+
+"""
+import os.path
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List, Optional, cast
+from fsspec import AbstractFileSystem
+
+from llama_index.core.readers.base import BaseReader
+from llama_index.core.schema import Document
+
+class VideoAudioReader(BaseReader):
+    """Video audio parser.
+
+    Extract text from transcript of video/audio files.
+
+    """
+
+    def __init__(
+            self,
+            *args: Any,
+            model_version: str = "base",
+            use_local: bool = False,
+            window: None,
+            **kwargs: Any
+    ) -> None:
+        """
+        Init parser.
+
+        :param args: arguments
+        :param model_version: model version
+        :param use_local: use local model
+        :param window: Window instance
+        :param kwargs: keyword arguments
+        """
+        super().__init__(*args, **kwargs)
+        self._model_version = model_version
+        self._use_local = use_local
+        self._window = window
+        self._initialized = False
+
+    def _initialize(self) -> None:
+        """Initialize parser."""
+        try:
+            import whisper
+        except ImportError:
+            raise ImportError(
+                "Please install OpenAI whisper model "
+                "'pip install git+https://github.com/openai/whisper.git' or pip install openai-whisper "
+                "to use the model"
+            )
+
+        model = whisper.load_model(self._model_version)
+        self.parser_config = {"model": model}
+        self._initialized = True
+
+    def _is_video(self, file: Path) -> bool:
+        """
+        Check if file is a video.
+
+        :param file: file path
+        :return: True if file is a video
+        """
+        return self._get_ext(file) in ["mp4", "avi", "mov", "mkv", "webm"]
+
+    def _get_ext(self, file: Path) -> str:
+        """
+        Get file extension.
+
+        :param file: file path
+        :return: file extension
+        """
+        return file.suffix[1:]
+
+    def _transcribe_local(self, file: Path) -> str:
+        """
+        Transcribe using local model.
+
+        :param file: file path
+        :return: transcript text
+        """
+        import whisper
+
+        model = cast(whisper.Whisper, self.parser_config["model"])
+        result = model.transcribe(str(file))
+        return result["text"]
+
+    def _transcribe_api(self, file: Path) -> str:
+        """
+        Transcribe using API.
+
+        :param file: file path
+        :return: transcript
+        """
+        return self._window.core.plugins.get("audio_input").get_provider().transcribe(str(file))
+
+    def load_data(
+        self,
+        file: Path,
+        extra_info: Optional[Dict] = None,
+        fs: Optional[AbstractFileSystem] = None,
+    ) -> List[Document]:
+        """
+        Parse video.
+
+        :param file: file path
+        :param extra_info: additional metadata
+        :param fs: file system
+        :return: list of documents
+        """
+        if self._use_local:
+            is_compiled = self._window.core.config.is_compiled() or self._window.core.platforms.is_snap()
+            if is_compiled:
+                raise ValueError("Local models are not available in compiled version.")
+
+            if not self._initialized:
+                self._initialize()
+
+        post_delete = None
+        if self._is_video(file):
+            video_type = self._get_ext(file)
+
+            try:
+                from pydub import AudioSegment
+            except ImportError:
+                raise ImportError("Please install pydub 'pip install pydub' ")
+            if fs:
+                with fs.open(file, "rb") as f:
+                    video = AudioSegment.from_file(f, format=video_type)
+            else:
+                # open file
+                video = AudioSegment.from_file(file, format=video_type)
+
+            # Extract audio from video
+            audio = video.split_to_mono()[0]
+            file_str = str(file)[:-4] + ".mp3"
+            # export file, if file exists add date suffix
+            if os.path.exists(file_str):
+                date_suffix = datetime.now().strftime("%Y%m%d%H%M%S")
+                file_str = str(file)[:-4] + "_" + date_suffix + ".mp3"
+            audio.export(file_str, format="mp3")
+            file = Path(file_str)
+            post_delete = str(file)
+
+        if self._use_local:
+            transcript = self._transcribe_local(file)
+        else:
+            if self._window is None:
+                raise ValueError("Window instance is required to use API model.")
+            transcript = self._transcribe_api(file)
+
+        # remove tmp file
+        if post_delete:
+            Path(post_delete).unlink()
+
+        return [Document(text=transcript, metadata=extra_info or {})]