MindsDB 25.1.2.0__py3-none-any.whl → 25.1.5.0__py3-none-any.whl

mindsdb/integrations/utilities/files/file_reader.py

@@ -0,0 +1,258 @@
+import traceback
+import json
+import csv
+from io import BytesIO, StringIO
+from pathlib import Path
+import codecs
+
+import filetype
+import pandas as pd
+from charset_normalizer import from_bytes
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+
+from mindsdb.utilities import log
+
+logger = log.getLogger(__name__)
+
+DEFAULT_CHUNK_SIZE = 500
+DEFAULT_CHUNK_OVERLAP = 250
+
+
+class FileDetectError(Exception):
+    ...
+
+
+def decode(file_obj: BytesIO) -> StringIO:
+    byte_str = file_obj.read()
+    # Move it to StringIO
+    try:
+        # Handle Microsoft's BOM "special" UTF-8 encoding
+        if byte_str.startswith(codecs.BOM_UTF8):
+            data_str = StringIO(byte_str.decode("utf-8-sig"))
+        else:
+            file_encoding_meta = from_bytes(
+                byte_str[: 32 * 1024],
+                steps=32,  # Number of steps/block to extract from my_byte_str
+                chunk_size=1024,  # Set block size of each extraction)
+                explain=False,
+            )
+            best_meta = file_encoding_meta.best()
+            errors = "strict"
+            if best_meta is not None:
+                encoding = file_encoding_meta.best().encoding
+
+                try:
+                    data_str = StringIO(byte_str.decode(encoding, errors))
+                except UnicodeDecodeError:
+                    encoding = "utf-8"
+                    errors = "replace"
+
+                    data_str = StringIO(byte_str.decode(encoding, errors))
+            else:
+                encoding = "utf-8"
+                errors = "replace"
+
+                data_str = StringIO(byte_str.decode(encoding, errors))
+    except Exception as e:
+        logger.error(traceback.format_exc())
+        raise FileDetectError("Could not load into string") from e
+
+    return data_str
+
+
+class FormatDetector:
+
+    def get(self, name, file_obj: BytesIO = None):
+        format = self.get_format_by_name(name)
+        if format is None and file_obj is not None:
+            format = self.get_format_by_content(file_obj)
+
+        if format is not None:
+            return format
+        raise FileDetectError(f'Unable to detect format: {name}')
+
+    def get_format_by_name(self, filename):
+        extension = Path(filename).suffix.strip(".").lower()
+        if extension == "tsv":
+            extension = "csv"
+        return extension or None
+
+    def get_format_by_content(self, file_obj):
+        if self.is_parquet(file_obj):
+            return "parquet"
+
+        file_type = filetype.guess(file_obj)
+        if file_type is None:
+            return
+
+        if file_type.mime in {
+            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+            "application/vnd.ms-excel",
+        }:
+            return 'xlsx'
+
+        if file_type.mime == 'application/pdf':
+            return "pdf"
+
+        file_obj = decode(file_obj)
+
+        if self.is_json(file_obj):
+            return "json"
+
+        if self.is_csv(file_obj):
+            return "csv"
+
+    def is_json(self, data_obj: StringIO) -> bool:
+        # see if its JSON
+        text = data_obj.read(100).strip()
+        data_obj.seek(0)
+        if len(text) > 0:
+            # it looks like a json, then try to parse it
+            if text.startswith("{") or text.startswith("["):
+                try:
+                    json.loads(data_obj.read())
+                    return True
+                except Exception:
+                    return False
+                finally:
+                    data_obj.seek(0)
+        return False
+
+    def is_csv(self, data_obj: StringIO) -> bool:
+        sample = data_obj.readline()  # trying to get dialect from header
+        data_obj.seek(0)
+        try:
+            csv.Sniffer().sniff(sample)
+
+        except Exception:
+            return False
+
+    def is_parquet(self, data: BytesIO) -> bool:
+        # Check first and last 4 bytes equal to PAR1.
+        # Refer: https://parquet.apache.org/docs/file-format/
+        parquet_sig = b"PAR1"
+        data.seek(0, 0)
+        start_meta = data.read(4)
+        data.seek(-4, 2)
+        end_meta = data.read()
+        data.seek(0)
+        if start_meta == parquet_sig and end_meta == parquet_sig:
+            return True
+        return False
+
+
+class FileReader:
+
+    def _get_csv_dialect(self, buffer) -> csv.Dialect:
+        sample = buffer.readline()  # trying to get dialect from header
+        buffer.seek(0)
+        try:
+            if isinstance(sample, bytes):
+                sample = sample.decode()
+            accepted_csv_delimiters = [",", "\t", ";"]
+            try:
+                dialect = csv.Sniffer().sniff(
+                    sample, delimiters=accepted_csv_delimiters
+                )
+                dialect.doublequote = (
+                    True  # assume that all csvs have " as string escape
+                )
+            except Exception:
+                dialect = csv.reader(sample).dialect
+                if dialect.delimiter not in accepted_csv_delimiters:
+                    raise Exception(
+                        f"CSV delimeter '{dialect.delimiter}' is not supported"
+                    )
+
+        except csv.Error:
+            dialect = None
+        return dialect
+
+    def read(self, format, file_obj: BytesIO, **kwargs) -> pd.DataFrame:
+        func = {
+            'parquet': self.read_parquet,
+            'csv': self.read_csv,
+            'xlsx': self.read_excel,
+            'pdf': self.read_pdf,
+            'json': self.read_json,
+            'txt': self.read_txt,
+        }
+
+        if format not in func:
+            raise FileDetectError(f'Unsupported format: {format}')
+        func = func[format]
+
+        return func(file_obj, **kwargs)
+
+    def read_csv(self, file_obj: BytesIO, **kwargs):
+        file_obj = decode(file_obj)
+        dialect = self._get_csv_dialect(file_obj)
+
+        return pd.read_csv(file_obj, sep=dialect.delimiter, index_col=False)
+
+    def read_txt(self, file_obj: BytesIO, **kwargs):
+        file_obj = decode(file_obj)
+
+        try:
+            from langchain_core.documents import Document
+        except ImportError:
+            raise ImportError(
+                "To import TXT document please install 'langchain-community':\n"
+                "    pip install langchain-community"
+            )
+        text = file_obj.read()
+
+        file_name = None
+        if hasattr(file_obj, "name"):
+            file_name = file_obj.name
+        metadata = {"source": file_name}
+        documents = [Document(page_content=text, metadata=metadata)]
+
+        text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=DEFAULT_CHUNK_SIZE, chunk_overlap=DEFAULT_CHUNK_OVERLAP
+        )
+
+        docs = text_splitter.split_documents(documents)
+        return pd.DataFrame(
+            [
+                {"content": doc.page_content, "metadata": doc.metadata}
+                for doc in docs
+            ]
+        )
+
+    def read_pdf(self, file_obj: BytesIO, **kwargs):
+        import fitz  # pymupdf
+
+        with fitz.open(stream=file_obj) as pdf:  # open pdf
+            text = chr(12).join([page.get_text() for page in pdf])
+
+        text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=DEFAULT_CHUNK_SIZE, chunk_overlap=DEFAULT_CHUNK_OVERLAP
+        )
+
+        split_text = text_splitter.split_text(text)
+
+        return pd.DataFrame(
+            {"content": split_text, "metadata": [{}] * len(split_text)}
+        )
+
+    def read_json(self, file_obj: BytesIO, **kwargs):
+        file_obj = decode(file_obj)
+        file_obj.seek(0)
+        json_doc = json.loads(file_obj.read())
+        return pd.json_normalize(json_doc, max_level=0)
+
+    def read_parquet(self, file_obj: BytesIO, **kwargs):
+        return pd.read_parquet(file_obj)
+
+    def read_excel(self, file_obj: BytesIO, sheet_name=None, **kwargs) -> pd.DataFrame:
+
+        file_obj.seek(0)
+        with pd.ExcelFile(file_obj) as xls:
+            if sheet_name is None:
+                # No sheet specified: Return list of sheets
+                sheet_list = xls.sheet_names
+                return pd.DataFrame(sheet_list, columns=["Sheet_Name"])
+            else:
+                # Specific sheet requested: Load that sheet
+                return pd.read_excel(xls, sheet_name=sheet_name)

mindsdb/integrations/utilities/handlers/api_utilities/microsoft/ms_graph_api_utilities.py

@@ -131,7 +131,8 @@ class MSGraphAPIBaseClient:
         response = self._make_request(api_url, params)
 
         # If the response content is a binary file or a TSV file, return the raw content.
-        if response.headers["Content-Type"] in ("application/octet-stream", "text/tab-separated-values"):
+        if response.headers["Content-Type"] in ("application/octet-stream", "text/plain",
+                                                "text/tab-separated-values", "application/pdf"):
             return response.content
         # Otherwise, return the JSON content.
         else:

mindsdb/integrations/utilities/handlers/auth_utilities/microsoft/ms_graph_api_auth_utilities.py

@@ -43,9 +43,14 @@ class MSGraphAPIDelegatedPermissionsManager:
         # Set the redirect URI based on the request origin.
         # If the request origin is 127.0.0.1 (localhost), replace it with localhost.
         # This is done because the only HTTP origin allowed in Microsoft Entra ID app registration is localhost.
-        request_origin = request.headers.get('ORIGIN') or (request.scheme + '://' + request.host)
-        if not request_origin:
-            raise AuthException('Request origin could not be determined!')
+        try:
+            request_origin = request.headers.get('ORIGIN') or (request.scheme + '://' + request.host)
+            if not request_origin:
+                raise AuthException('Request origin could not be determined!')
+        except RuntimeError:
+            # if it is outside of request context (streaming in agent)
+            request_origin = ''
+
         request_origin = request_origin.replace('127.0.0.1', 'localhost') if 'http://127.0.0.1' in request_origin else request_origin
         self.redirect_uri = request_origin + '/verify-auth'
 

mindsdb/integrations/utilities/rag/chains/map_reduce_summarizer_chain.py

@@ -23,7 +23,7 @@ logger = log.getLogger(__name__)
 Summary = namedtuple('Summary', ['source_id', 'content'])
 
 
-def create_map_reduce_documents_chain(summarization_config: SummarizationConfig, input: str) -> MapReduceDocumentsChain:
+def create_map_reduce_documents_chain(summarization_config: SummarizationConfig, input: str) -> ReduceDocumentsChain:
     '''Creats a chain that map reduces documents into a single consolidated summary
 
     Args:
@@ -43,7 +43,7 @@ def create_map_reduce_documents_chain(summarization_config: SummarizationConfig,
     if 'input' in map_prompt.input_variables:
         map_prompt = map_prompt.partial(input=input)
     # Handles summarization of individual chunks.
-    map_chain = LLMChain(llm=summarization_llm, prompt=map_prompt)
+    # map_chain = LLMChain(llm=summarization_llm, prompt=map_prompt)
 
     reduce_prompt_template = summarization_config.reduce_prompt_template
     reduce_prompt = PromptTemplate.from_template(reduce_prompt_template)
@@ -60,18 +60,12 @@ def create_map_reduce_documents_chain(summarization_config: SummarizationConfig,
     )
 
     # Combines & iteratively reduces mapped documents.
-    reduce_documents_chain = ReduceDocumentsChain(
+    return ReduceDocumentsChain(
         combine_documents_chain=combine_documents_chain,
         collapse_documents_chain=combine_documents_chain,
         # Max number of tokens to group documents into.
         token_max=summarization_config.max_summarization_tokens
     )
-    return MapReduceDocumentsChain(
-        llm_chain=map_chain,
-        reduce_documents_chain=reduce_documents_chain,
-        document_variable_name='docs',
-        return_intermediate_steps=False
-    )
 
 
 class MapReduceSummarizerChain(Chain):
@@ -135,6 +129,8 @@ class MapReduceSummarizerChain(Chain):
         document_chunks = []
         for _, row in all_source_chunks.iterrows():
             metadata = row.get(self.metadata_column_name, {})
+            if row.get('chunk_id', None) is not None:
+                metadata['chunk_index'] = row.get('chunk_id', 0)
             document_chunks.append(Document(page_content=row[self.content_column_name], metadata=metadata))
         # Sort by chunk index if present in metadata so the full document is in its original order.
         document_chunks.sort(key=lambda doc: doc.metadata.get('chunk_index', 0) if doc.metadata else 0)

mindsdb/integrations/utilities/rag/loaders/vector_store_loader/pgvector.py

@@ -1,9 +1,9 @@
-from typing import Any, List, Optional, Dict
+from typing import Any, List, Union, Optional, Dict
 
 from langchain_community.vectorstores import PGVector
 from langchain_community.vectorstores.pgvector import Base
 
-from pgvector.sqlalchemy import Vector
+from pgvector.sqlalchemy import SPARSEVEC, Vector
 import sqlalchemy as sa
 from sqlalchemy.dialects.postgresql import JSON
 
@@ -15,9 +15,17 @@ _generated_sa_tables = {}
 
 class PGVectorMDB(PGVector):
     """
-       langchain_community.vectorstores.PGVector adapted for mindsdb vector store table structure
+    langchain_community.vectorstores.PGVector adapted for mindsdb vector store table structure
     """
 
+    def __init__(self, *args, is_sparse: bool = False, vector_size: Optional[int] = None, **kwargs):
+        # todo get is_sparse and vector_size from kb vector table
+        self.is_sparse = is_sparse
+        if is_sparse and vector_size is None:
+            raise ValueError("vector_size is required when is_sparse=True")
+        self.vector_size = vector_size
+        super().__init__(*args, **kwargs)
+
     def __post_init__(
         self,
     ) -> None:
@@ -32,53 +40,94 @@ class PGVectorMDB(PGVector):
                 __tablename__ = collection_name
 
                 id = sa.Column(sa.Integer, primary_key=True)
-                embedding: Vector = sa.Column('embeddings', Vector())
-                document = sa.Column('content', sa.String, nullable=True)
-                cmetadata = sa.Column('metadata', JSON, nullable=True)
+                embedding = sa.Column(
+                    "embeddings",
+                    SPARSEVEC() if self.is_sparse else Vector() if self.vector_size is None else
+                    SPARSEVEC(self.vector_size) if self.is_sparse else Vector(self.vector_size)
+                )
+                document = sa.Column("content", sa.String, nullable=True)
+                cmetadata = sa.Column("metadata", JSON, nullable=True)
 
             _generated_sa_tables[collection_name] = EmbeddingStore
 
         self.EmbeddingStore = _generated_sa_tables[collection_name]
 
     def __query_collection(
-        self,
-        embedding: List[float],
-        k: int = 4,
-        filter: Optional[Dict[str, str]] = None,
+            self,
+            embedding: Union[List[float], Dict[int, float], str],
+            k: int = 4,
+            filter: Optional[Dict[str, str]] = None,
     ) -> List[Any]:
         """Query the collection."""
         with Session(self._bind) as session:
-
-            results: List[Any] = (
-                session.query(
-                    self.EmbeddingStore,
-                    self.distance_strategy(embedding).label("distance"),
-                )
-                .order_by(sa.asc("distance"))
-                .limit(k)
-                .all()
+            if self.is_sparse:
+                # Sparse vectors: expect string in format "{key:value,...}/size" or dictionary
+                if isinstance(embedding, dict):
+                    from pgvector.utils import SparseVector
+                    embedding = SparseVector(embedding, self.vector_size)
+                    embedding_str = embedding.to_text()
+                elif isinstance(embedding, str):
+                    # Use string as is - it should already be in the correct format
+                    embedding_str = embedding
+                # Use inner product for sparse vectors
+                distance_op = "<#>"
+                # For inner product, larger values are better matches
+                order_direction = "ASC"
+            else:
+                # Dense vectors: expect string in JSON array format or list of floats
+                if isinstance(embedding, list):
+                    embedding_str = f"[{','.join(str(x) for x in embedding)}]"
+                elif isinstance(embedding, str):
+                    embedding_str = embedding
+                # Use cosine similarity for dense vectors
+                distance_op = "<=>"
+                # For cosine similarity, smaller values are better matches
+                order_direction = "ASC"
+
+            # Use SQL directly for vector comparison
+            query = sa.text(
+                f"""
+            SELECT t.*, t.embeddings {distance_op} '{embedding_str}' as distance
+            FROM {self.collection_name} t
+            ORDER BY distance {order_direction}
+            LIMIT {k}
+            """
             )
-        for rec, _ in results:
-            if not bool(rec.cmetadata):
-                rec.cmetadata = {0: 0}
+            results = session.execute(query).all()
+
+            # Convert results to the expected format
+            formatted_results = []
+            for rec in results:
+                metadata = rec.metadata if bool(rec.metadata) else {0: 0}
+                embedding_store = self.EmbeddingStore()
+                embedding_store.document = rec.content
+                embedding_store.cmetadata = metadata
+                result = type(
+                    'Result', (), {
+                        'EmbeddingStore': embedding_store,
+                        'distance': rec.distance
+                    }
+                )
+                formatted_results.append(result)
 
-        return results
+            return formatted_results
 
     # aliases for different langchain versions
     def _PGVector__query_collection(self, *args, **kwargs):
+
         return self.__query_collection(*args, **kwargs)
 
     def _query_collection(self, *args, **kwargs):
         return self.__query_collection(*args, **kwargs)
 
     def create_collection(self):
-        raise RuntimeError('Forbidden')
+        raise RuntimeError("Forbidden")
 
     def delete_collection(self):
-        raise RuntimeError('Forbidden')
+        raise RuntimeError("Forbidden")
 
     def delete(self, *args, **kwargs):
-        raise RuntimeError('Forbidden')
+        raise RuntimeError("Forbidden")
 
     def add_embeddings(self, *args, **kwargs):
-        raise RuntimeError('Forbidden')
+        raise RuntimeError("Forbidden")

mindsdb/integrations/utilities/rag/loaders/vector_store_loader/vector_store_loader.py

@@ -7,6 +7,7 @@ from pydantic import BaseModel
 
 from mindsdb.integrations.utilities.rag.settings import VectorStoreType, VectorStoreConfig
 from mindsdb.integrations.utilities.rag.loaders.vector_store_loader.MDBVectorStore import MDBVectorStore
+from mindsdb.integrations.utilities.rag.loaders.vector_store_loader.pgvector import PGVectorMDB
 from mindsdb.utilities import log
 
 
@@ -28,6 +29,20 @@ class VectorStoreLoader(BaseModel):
         Loads the vector store based on the provided config and embeddings model
         :return:
         """
+        if self.config.is_sparse is not None and self.config.vector_size is not None and self.config.kb_table is not None:
+            # Only use PGVector store for sparse vectors.
+            db_handler = self.config.kb_table.get_vector_db()
+            db_args = db_handler.connection_args
+            # Assume we are always using PGVector & psycopg2.
+            connection_str = f"postgresql+psycopg2://{db_args.get('user')}:{db_args.get('password')}@{db_args.get('host')}:{db_args.get('port')}/{db_args.get('dbname', db_args.get('database'))}"
+
+            return PGVectorMDB(
+                connection_string=connection_str,
+                collection_name=self.config.kb_table._kb.vector_database_table,
+                embedding_function=self.embedding_model,
+                is_sparse=self.config.is_sparse,
+                vector_size=self.config.vector_size
+            )
         return MDBVectorStore(kb_table=self.config.kb_table)
 
 
@@ -56,5 +71,7 @@ class VectorStoreFactory:
         return PGVectorMDB(
             connection_string=settings.connection_string,
             collection_name=settings.collection_name,
-            embedding_function=embedding_model
+            embedding_function=embedding_model,
+            is_sparse=settings.is_sparse,
+            vector_size=settings.vector_size
         )