ai-parrot 0.3.4__cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl

parrot/loaders/excel.py

@@ -0,0 +1,349 @@
+from collections.abc import Callable
+from pathlib import PurePath
+from typing import Union
+from io import StringIO
+import mimetypes
+import magic
+import xlrd
+import numpy as np
+import pandas
+from pandas._libs.parsers import STR_NA_VALUES  # pylint: disable=E0611
+from langchain.text_splitter import (
+    RecursiveCharacterTextSplitter
+)
+from langchain.docstore.document import Document
+from navigator.libs.json import JSONContent
+from .abstract import AbstractLoader
+
+
+excel_based = (
+    "application/vnd.ms-excel.sheet.binary.macroEnabled.12",
+    "application/vnd.ms-excel.sheet.macroEnabled.12",
+    "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    "application/vnd.ms-excel"
+)
+
+supported_extensions = (
+    ".xls",
+    ".xlsx",
+    ".xlsm",
+    ".xlsb"
+)
+
+class ExcelLoader(AbstractLoader):
+    """
+    Loader for Excel files using Pandas (preserving Table structure).
+    """
+
+    _extension = supported_extensions
+
+    def __init__(
+        self,
+        path: Union[list[PurePath], PurePath],
+        tokenizer: Callable = None,
+        text_splitter: Callable = None,
+        source_type: str = 'Excel',
+        **kwargs
+    ):
+        self.path = path
+        # Index Key:
+        self._index_keys: list = kwargs.pop('index', [])
+        self._type: str = kwargs.pop('document_type', 'Excel')
+        self.sheet_name = kwargs.pop('sheet_name', "Sheet 1")
+        self.skiprows = kwargs.pop('skiprows', 0)
+        self._pdargs = kwargs.pop('pd_args', {})
+        self._magic = magic.Magic(mime=True)
+        self.mimetype = kwargs.pop('mimetype', None)
+        self.filter_nan: bool = kwargs.pop('filter_nan', True)
+        self.na_values: list = ["NULL", "TBD"]
+        # Operations over dataframe:
+        self._drop_empty = kwargs.pop('drop_empty', False)
+        self._trim = kwargs.pop('trim', False)
+        self._infer_types = kwargs.pop('infer_types', True)
+        self._fillna: bool = kwargs.pop('fillna', False)
+        self._rename_cols = kwargs.pop('rename_cols', {})
+        self._to_integer: list = kwargs.pop('to_integer', [])
+        super().__init__(
+            tokenizer=tokenizer,
+            text_splitter=text_splitter,
+            source_type=source_type,
+            **kwargs
+        )
+        # JSON encoder:
+        self._encoder = JSONContent()
+
+    def clean_empty(self, df: pandas.DataFrame, columns: list = None) -> pandas.DataFrame:
+        """
+        Clean empty rows and columns from a DataFrame.
+
+        Args:
+            df (pd.DataFrame): The DataFrame to clean.
+
+        Returns:
+            pd.DataFrame: The cleaned DataFrame.
+        """
+        df.dropna(axis=1, how="all", inplace=True)
+        df.dropna(axis=0, how="all", inplace=True)
+        if columns:
+            for column in columns:
+                condition = df[
+                    (df[column].empty)
+                    | (df[column] == "")
+                    | (df[column].isna())
+                ].index
+                df.drop(condition, inplace=True)
+        return df
+
+    def open_excel(self, filename: PurePath) -> pandas.DataFrame:
+        self.logger.debug(
+            f"Opening Excel file {filename}"
+        )
+        ## Define NA Values:
+        default_missing = STR_NA_VALUES.copy()
+        for val in self.na_values:  # pylint: disable=E0203
+            default_missing.add(val)
+            default_missing.add(val)
+        self.na_values = default_missing
+        if filename.suffix not in supported_extensions:
+            raise ValueError(
+                f"Unsupported Excel file format: {filename.suffix}"
+            )
+        if not filename.exists():
+            raise FileNotFoundError(
+                f"Excel file not found: {filename}"
+            )
+        if not self.mimetype:
+            try:
+                self.mimetype = self._magic.from_file(str(filename))
+                self.logger.debug(f":: Detected MIME IS: {self.mimetype}")
+            except Exception as exc:
+                self.logger.error(f":: Error detecting MIME: {exc}")
+                self.mimetype = mimetypes.guess_type(str(filename))[0]
+            if not self.mimetype:
+                # Cannot Detect Mime type:
+                ext = filename.suffix
+                if ext == ".xlsx" or ext == ".xls":
+                    self.mimetype = "application/vnd.ms-excel"
+                elif ext == ".csv" or ext == ".txt":
+                    self.mimetype = "text/csv"
+        if (
+            self.mimetype == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+        ):
+            # xlsx or any openxml based document
+            file_engine = "openpyxl"
+        elif self.mimetype == "application/vnd.ms-excel.sheet.binary.macroEnabled.12":
+            # xlsb
+            file_engine = "pyxlsb"
+        else:
+            # Using extension:
+            ext = filename.suffix
+            if ext == ".xlsx":
+                file_engine = "openpyxl"
+            elif ext == ".xls":
+                file_engine = "xlrd"
+            elif ext == ".xlsb":
+                file_engine = "pyxlsb"
+            else:
+                raise ValueError(
+                    f"Unsupported Excel file format: {filename.suffix}"
+                )
+        # Build Arguments:
+        pd_args = self._pdargs.copy()
+        pd_args['sheet_name'] = self.sheet_name
+        pd_args['skiprows'] = self.skiprows
+        pd_args['engine'] = file_engine
+        pd_args['na_values'] = self.na_values
+        pd_args['na_filter'] = self.filter_nan
+        try:
+            df = pandas.read_excel(
+                filename,
+                keep_default_na=False,
+                **pd_args
+            )
+        except (IndexError, xlrd.biffh.XLRDError) as err:
+            raise ValueError(
+                f"Excel Index error on File {filename}: {err}"
+            ) from err
+        except pandas.errors.EmptyDataError as err:
+            raise ValueError(f"Empty File {filename}: {err}") from err
+        except pandas.errors.ParserError as err:
+            raise ValueError(f"Error Parsing File {filename}: {err}") from err
+        except Exception as err:
+            self.logger.exception(str(err), stack_info=True)
+            raise
+        # Post-Processing the DataFrame:
+        if self._infer_types is True:
+            df.infer_objects()
+        # rename cols:
+        if self._rename_cols:
+            try:
+                # Renaming Pandas Columns:
+                df.rename(columns=self._rename_cols, inplace=True)
+            except Exception as err:
+                self.logger.error(
+                    f"Error Renaming Columns: {err}"
+                )
+        # Clean Empty Rows:
+        df = self.clean_empty(df, self._index_keys)
+        if self._drop_empty:
+            df.dropna(axis=1, how="all", inplace=True)
+            df.dropna(axis=0, how="all", inplace=True)
+            df = df.loc[:, ~df.columns.str.contains("^Unnamed")]
+        if self._trim:
+            cols = df.select_dtypes(include=["object", "string"])
+            for col in cols:
+                df[col] = df[col].astype(str).str.strip()
+            # Trim whitespace from all string columns
+            # df = df.applymap(lambda x: x.strip() if isinstance(x, str) else x)
+        if self._to_integer:
+            for column in self._to_integer:
+                df[column] = df[column].fillna(0)
+                try:
+                    df[column] = pandas.to_numeric(df[column], errors="coerce")
+                    df[column] = df[column].astype('Int64')
+                except Exception:
+                    continue
+        if self._fillna:
+            # Select all integer columns (both nullable and non-nullable)
+            int_columns = df.select_dtypes(
+                include=['int64', 'Int64', 'int32', 'Int32']
+            ).columns  # Add other integer types if necessary
+            # Fill NaN values with zeros in those columns
+            df[int_columns] = df[int_columns].fillna(0)
+            # Select the Strings:
+            str_columns = df.select_dtypes(include=["object", "string"]).columns
+            df[str_columns] = df[str_columns].astype(str).replace(["nan", np.nan], "", regex=True)
+        print(df)
+        print("::: Printing Column Information === ")
+        for column, t in df.dtypes.items():
+            print(column, "->", t, "->", df[column].iloc[0])
+        return df
+
+    def unique_columns(self, df: pandas.DataFrame) -> pandas.DataFrame:
+        """
+        Rename duplicate columns in the DataFrame to ensure they are unique.
+
+        Args:
+            df (pd.DataFrame): The DataFrame with potential duplicate column names.
+
+        Returns:
+            pd.DataFrame: A DataFrame with unique column names.
+        """
+        seen = {}
+        new_columns = []
+        for col in df.columns:
+            new_col = col
+            count = seen.get(col, 0)
+            while new_col in new_columns:
+                count += 1
+                new_col = f"{col}_{count}"
+            new_columns.append(new_col)
+            seen[col] = count
+        df.columns = new_columns
+        return df
+
+    def get_json(self, df: pandas.DataFrame) -> str:
+        """
+        Convert a DataFrame to a JSON string.
+
+        Args:
+            df (pd.DataFrame): The DataFrame to convert.
+
+        Returns:
+            str: The JSON string.
+        """
+        buffer = StringIO()
+        df = self.unique_columns(df)
+        df.to_json(buffer, orient='records')
+        buffer.seek(0)
+        return buffer.getvalue()
+
+    def row_to_json(self, data) -> str:
+        return self._encoder.dumps(data)
+
+    def row_to_string(self, data: dict) -> str:
+        results = []
+        for key, val in data.items():
+            results.append(f'{key}: "{val!s}"')
+        return ', '.join(results)
+
+    def _load_excel(self, path: PurePath) -> list:
+        """
+        Load an Excel file using the Pandas library.
+
+        Args:
+            path (Path): The path to the Excel file.
+
+        Returns:
+            list: A list of Langchain Documents.
+        """
+        if self._check_path(path):
+            self.logger.info(f"Loading Excel file: {path}")
+            df = self.open_excel(path)
+            # Check for row unicity:
+            df = self.unique_columns(df)
+            metadata = {
+                "url": '',
+                "index": '',
+                "source": str(path.name),
+                "filename": path.name,
+                "question": '',
+                "answer": '',
+                "summary": '',
+                "source_type": self._source_type,
+                "type": self._type,
+            }
+            document_meta = {
+                "columns": df.columns.tolist(),
+                "rows": str(len(df)),
+                "sheet": self.sheet_name,
+            }
+            documents = []
+            # remove NaN values:
+            df.fillna('', axis=1, inplace=True)
+            for idx, row in df.iterrows():
+                idk = ''
+                rw = row.to_dict()
+                for col in self._index_keys:
+                    if col in row:
+                        idk = row[col]
+                        break
+                _data = {
+                    "index": idk,
+                    "data": rw,
+                    "document_meta": {
+                        "row_index": idx,
+                        **document_meta,
+                        **rw
+                    }
+                }
+                _meta = {**metadata, **_data}
+                row_data = self.row_to_string(rw)
+                doc = Document(
+                    page_content=row_data,
+                    metadata=_meta
+                )
+                documents.append(doc)
+            return documents
+        return []
+
+    def load(self, max_tokens: int = 768) -> list:
+        documents = []
+        if self.path.is_file():
+            # single File:
+            documents = self._load_excel(self.path)
+        elif self.path.is_dir():
+            documents = []
+            # iterate over the files in the directory
+            for ext in self._extension:
+                for item in self.path.glob(f'*{ext}'):
+                    documents.extend(self._load_excel(item))
+        elif isinstance(self.path, list):
+            pass
+        # Split Table Data:
+        return self.split_documents(documents)
+
+    def parse(self, source):
+        raise NotImplementedError(
+            "Parser method is not implemented for ExcelLoader."
+        )

parrot/loaders/github.py

@@ -0,0 +1,65 @@
+from pathlib import PurePath
+from langchain_community.document_loaders.parsers import LanguageParser
+from langchain_community.document_loaders import GithubFileLoader
+from langchain_text_splitters import Language
+from langchain.text_splitter import (
+    RecursiveCharacterTextSplitter
+)
+from navconfig import config
+from .abstract import AbstractLoader
+
+class GithubLoader(AbstractLoader):
+    """
+    Load Code from a Github Repository.
+    """
+    def __init__(
+        self,
+        repository_url: str,
+        lang: str = 'python',
+        source_type: str = 'code',
+        branch: str = 'main',
+        **kwargs
+    ):
+        super().__init__(source_type=source_type, **kwargs)
+        self._url = repository_url
+        self.branch = branch
+        self.github_token = kwargs.get('github_token', config.get('GITHUB_TOKEN'))
+        self.lang = lang
+        if lang == 'python':
+            self.parser = LanguageParser(language=Language.PYTHON, parser_threshold=500)
+            self.splitter = RecursiveCharacterTextSplitter.from_language(
+                language=Language.PYTHON, chunk_size=1024, chunk_overlap=200
+            )
+            self.suffixes = [".py", ".pyx"]
+        elif lang == 'javascript':
+            self.parser = LanguageParser(language=Language.JS, parser_threshold=500)
+            self.splitter = RecursiveCharacterTextSplitter.from_language(
+                language=Language.JS, chunk_size=1024, chunk_overlap=200
+            )
+            self.suffixes = [".js", ".jsx", ".json", ".ts", ".tsx"]
+        elif lang == 'typescript':
+            self.parser = LanguageParser(language=Language.TS, parser_threshold=500)
+            self.splitter = RecursiveCharacterTextSplitter.from_language(
+                language=Language.TS, chunk_size=1024, chunk_overlap=200
+            )
+            self.suffixes = [".js", ".jsx", ".json", ".ts", ".tsx"]
+        else:
+            raise ValueError(
+                f"Language {lang} not supported for Repository"
+            )
+
+    def load(self) -> list:
+        self.logger.info(f'Loading Github Repository > {self._url}:<{self.branch}>')
+        loader = GithubFileLoader(
+            repo=self._url,
+            github_api_url='https://api.github.com',
+            access_token=self.github_token,
+            branch=self.branch,
+        )
+        docs = loader.load()
+        for doc in docs:
+            doc.metadata['source_type'] = self._source_type
+        return self.split_documents(docs)
+
+    def parse(self, source):
+        raise NotImplementedError("Parser method is not implemented for PDFLoader.")

parrot/loaders/handlers/__init__.py

@@ -0,0 +1,5 @@
+from .data import DataManagement
+
+__all__ = (
+    'DataManagement',
+)

parrot/loaders/handlers/data.py

@@ -0,0 +1,213 @@
+"""
+Data Management for Bots.
+"""
+from asyncdb.exceptions import NoDataFound, DriverError
+from querysource.datasources.drivers.rethink import rethink_default
+from navigator.views import FormModel
+from ..utils.models import BotData
+from ...interfaces import DBInterface
+
+class DataManagement(DBInterface, FormModel):
+    """
+    Managing Data to be inserted into Vector Databases.
+    """
+    model = BotData
+    path: str = '/api/v1/bot_management/data'
+
+    async def get_rethink(self):
+        params = rethink_default.params()
+        return self.get_database('rethink', params=params)
+
+    async def _get_form(self, args, qp, fields):
+        """Get Form information."""
+        bot = args.get('id', None)
+        async with await self.get_rethink() as conn:
+            await conn.use('navigator')
+            filter = {}
+            if bot:
+                filter= {
+                    "chatbot_id": bot
+                }
+                try:
+                    result = await conn.fetch_all(
+                        'chatbots_data',
+                        filter=filter
+                    )
+                except NoDataFound:
+                    return self.json_response(
+                        status=204
+                    )
+            return self.json_response(
+                result, status=200
+            )
+
+    async def put(self):
+        """
+        Creating a new data-store record.
+        """
+        data = await self.validate_payload()
+        args = self.get_arguments()
+        # unique field is ['chatbot_id', 'source_type', 'version']
+        bot = args.get('id', data.chatbot_id)
+        filter = {
+            'chatbot_id': str(bot),
+            'source_type': data.source_type,
+            'version': data.version
+        }
+        tbl = data.Meta.name
+        exists = False
+        async with await self.get_rethink() as conn:
+            await conn.use('navigator')
+            try:
+                result = await conn.fetch_one(
+                    table=tbl,
+                    filter=filter
+                )
+                if result:
+                    exists = True
+            except NoDataFound:
+                exists = False
+            if exists is False:
+                try:
+                    state = await conn.insert(
+                        tbl, data.to_dict(), on_conflict='replace'
+                    )
+                    return self.json_response(
+                        response={
+                            "message": "Bot data inserted",
+                            "state": state,
+                        },
+                        status=203
+                    )
+                except DriverError as exc:
+                    return self.error(
+                        response={
+                            "error": "unable to insert RT data",
+                            "exception": str(exc)
+                        },
+                        status=400
+                    )
+            else:
+                data.version += data.version
+                try:
+                    state = await conn.update(
+                        tbl, data.to_dict(), filter=filter, return_changes=True
+                    )
+                    return self.json_response(
+                        response={
+                            "message": "Bot data updated",
+                            "state": state,
+                        },
+                        status=202
+                    )
+                except DriverError as exc:
+                    return self.error(
+                        response={
+                            "error": "unable to insert RT data",
+                            "exception": str(exc)
+                        },
+                        status=400
+                    )
+
+    async def patch(self):
+        """
+        Patching (updating) existing data.
+        """
+        # data = await self.validate_payload()
+        data = await self.json_data()
+        args = self.get_arguments()
+        # unique field is ['chatbot_id', 'source_type', 'version']
+        bot = args.get('id', data['chatbot_id'])
+        try:
+            filter = {
+                'chatbot_id': str(bot),
+                'source_type': data['source_type'],
+                'version': data['version']
+            }
+        except KeyError:
+            return self.error(
+                response={
+                    "error": "Invalid data for Data Filtering"
+                },
+                status=400
+            )
+        tbl = 'chatbots_data'
+        async with await self.get_rethink() as conn:
+            await conn.use('navigator')
+            try:
+                result = await conn.fetch_one(
+                    table=tbl,
+                    filter=filter
+                )
+                if result:
+                    # update existing:
+                    for k,v in data.items():
+                        result[k] = v
+                    try:
+                        state = await conn.update(
+                            tbl, result, filter=filter, return_changes=True
+                        )
+                        return self.json_response(
+                            response={
+                                "message": "Bot data updated",
+                                "state": state,
+                            },
+                            status=202
+                        )
+                    except DriverError as exc:
+                        return self.error(
+                            response={
+                                "error": "unable to Update RT data",
+                                "exception": str(exc)
+                            },
+                            status=400
+                        )
+            except NoDataFound:
+                return self.error(
+                    response={
+                        "error": f"Data not Found for {filter}",
+                    },
+                    status=400
+                )
+
+    async def delete(self):
+        """
+        Remove a data-store record
+        """
+        data = await self.validate_payload()
+        args = self.get_arguments()
+        # unique field is ['chatbot_id', 'source_type', 'version']
+        bot = args.get('id', data.chatbot_id)
+        filter = {
+            'chatbot_id': str(bot),
+            'source_type': data.source_type,
+            'version': data.version
+        }
+        tbl = data.Meta.name
+        async with await self.get_rethink() as conn:
+            await conn.use('navigator')
+            try:
+                result = await conn.fetch_one(
+                    table=tbl,
+                    filter=filter
+                )
+                if result:
+                    state = await conn.delete(
+                        table=tbl,
+                        filter=filter,
+                        return_changes=True
+                    )
+                    return self.json_response(
+                        response={
+                            "message": "Bot data deleted",
+                            "state": state,
+                        },
+                        status=202
+                    )
+            except NoDataFound:
+                return self.error(
+                    response={
+                        "error": f"Data not Found for {filter}",
+                    },
+                    status=400
+                )