MindsDB 25.3.4.2__py3-none-any.whl → 25.4.2.0__py3-none-any.whl

mindsdb/interfaces/knowledge_base/controller.py

@@ -4,6 +4,7 @@ from typing import Dict, List, Optional
 
 import pandas as pd
 import hashlib
+import numpy as np
 
 from mindsdb_sql_parser.ast import (
     BinaryOperation,
@@ -34,12 +35,20 @@ from mindsdb.interfaces.knowledge_base.preprocessing.models import Preprocessing
 from mindsdb.interfaces.knowledge_base.preprocessing.document_preprocessor import PreprocessorFactory
 from mindsdb.interfaces.model.functions import PredictorRecordNotFound
 from mindsdb.utilities.exception import EntityExistsError, EntityNotExistsError
+from mindsdb.integrations.utilities.sql_utils import FilterCondition, FilterOperator
 
 from mindsdb.api.executor.command_executor import ExecuteCommands
 from mindsdb.utilities import log
+from mindsdb.integrations.utilities.rag.rerankers.reranker_compressor import LLMReranker
 
 logger = log.getLogger(__name__)
 
+KB_TO_VECTORDB_COLUMNS = {
+    'id': 'original_row_id',
+    'chunk_id': 'id',
+    'chunk_content': 'content'
+}
+
 
 class KnowledgeBaseTable:
     """
@@ -77,46 +86,150 @@ class KnowledgeBaseTable:
         """
         logger.debug(f"Processing select query: {query}")
 
-        # replace content with embeddings
-        query_traversal(query.where, self._replace_query_content)
-        logger.debug("Replaced content with embeddings in where clause")
+        # Extract the content query text for potential reranking
+
+        db_handler = self.get_vector_db()
 
+        logger.debug("Replaced content with embeddings in where clause")
         # set table name
         query.from_table = Identifier(parts=[self._kb.vector_database_table])
         logger.debug(f"Set table name to: {self._kb.vector_database_table}")
 
-        # remove embeddings from result
-        targets = []
+        requested_kb_columns = []
         for target in query.targets:
             if isinstance(target, Star):
-                targets.extend([
-                    Identifier(TableField.ID.value),
-                    Identifier(TableField.CONTENT.value),
-                    Identifier(TableField.METADATA.value),
-                ])
-            elif isinstance(target, Identifier) and target.parts[-1].lower() != TableField.EMBEDDINGS.value:
-                targets.append(target)
-        query.targets = targets
-        logger.debug(f"Modified query targets: {targets}")
+                requested_kb_columns = None
+                break
+            else:
+                requested_kb_columns.append(target.parts[-1].lower())
+
+        query.targets = [
+            Identifier(TableField.ID.value),
+            Identifier(TableField.CONTENT.value),
+            Identifier(TableField.METADATA.value),
+            Identifier(TableField.DISTANCE.value),
+        ]
 
         # Get response from vector db
-        db_handler = self.get_vector_db()
         logger.debug(f"Using vector db handler: {type(db_handler)}")
 
-        df = db_handler.dispatch_select(query)
+        # extract values from conditions and prepare for vectordb
+        conditions = []
+        query_text = None
+        reranking_threshold = None
+        query_conditions = db_handler.extract_conditions(query.where)
+        if query_conditions is not None:
+            for item in query_conditions:
+                if item.column == "reranking_threshold" and item.op.value == "=":
+                    try:
+                        reranking_threshold = float(item.value)
+                        # Validate range: must be between 0 and 1
+                        if not (0 <= reranking_threshold <= 1):
+                            raise ValueError(f"reranking_threshold must be between 0 and 1, got: {reranking_threshold}")
+                        logger.debug(f"Found reranking_threshold in query: {reranking_threshold}")
+                    except (ValueError, TypeError) as e:
+                        error_msg = f"Invalid reranking_threshold value: {item.value}. {str(e)}"
+                        logger.error(error_msg)
+                        raise ValueError(error_msg)
+                elif item.column == TableField.CONTENT.value:
+                    query_text = item.value
+
+                    # replace content with embeddings
+                    conditions.append(FilterCondition(
+                        column=TableField.EMBEDDINGS.value,
+                        value=self._content_to_embeddings(item.value),
+                        op=FilterOperator.EQUAL,
+                    ))
+                else:
+                    conditions.append(item)
 
-        if df is not None:
+        logger.debug(f"Extracted query text: {query_text}")
 
-            logger.debug(f"Query returned {len(df)} rows")
-            logger.debug(f"Columns in response: {df.columns.tolist()}")
-            # Log a sample of IDs to help diagnose issues
-            if not df.empty:
-                logger.debug(f"Sample of IDs in response: {df['id'].head().tolist()}")
-        else:
-            logger.warning("Query returned no data")
+        self.addapt_conditions_columns(conditions)
+        df = db_handler.dispatch_select(query, conditions)
+        df = self.addapt_result_columns(df)
+
+        logger.debug(f"Query returned {len(df)} rows")
+        logger.debug(f"Columns in response: {df.columns.tolist()}")
+        # Check if we have a rerank_model configured in KB params
+
+        df = self.add_relevance(df, query_text, reranking_threshold)
+
+        # filter by targets
+        if requested_kb_columns is not None:
+            df = df[requested_kb_columns]
+        return df
+
+    def add_relevance(self, df, query_text, reranking_threshold=None):
+        relevance_column = TableField.RELEVANCE.value
+
+        rerank_model = self._kb.params.get("rerank_model")
+        if rerank_model and query_text and len(df) > 0:
+            # Use reranker for relevance score
+            try:
+                logger.info(f"Using reranker model {rerank_model} for relevance calculation")
+                reranker_params = {"model": rerank_model}
+                # Apply custom filtering threshold if provided
+                if reranking_threshold is not None:
+                    reranker_params["filtering_threshold"] = reranking_threshold
+                    logger.info(f"Using custom filtering threshold: {reranking_threshold}")
+
+                reranker = LLMReranker(**reranker_params)
+                # Get documents to rerank
+                documents = df['chunk_content'].tolist()
+                # Use the get_scores method with disable_events=True
+                scores = reranker.get_scores(query_text, documents)
+                # Add scores as the relevance column
+                df[relevance_column] = scores
+
+                # Filter by threshold
+                scores_array = np.array(scores)
+                df = df[scores_array > reranker.filtering_threshold]
+                logger.debug(f"Applied reranking with model {rerank_model}, threshold: {reranker.filtering_threshold}")
+            except Exception as e:
+                logger.error(f"Error during reranking: {str(e)}")
+                # Fallback to distance-based relevance
+                if 'distance' in df.columns:
+                    df[relevance_column] = 1 / (1 + df['distance'])
+                else:
+                    logger.info("No distance or reranker available")
+
+        elif 'distance' in df.columns:
+            # Calculate relevance from distance
+            logger.info("Calculating relevance from vector distance")
+            df[relevance_column] = 1 / (1 + df['distance'])
 
+        else:
+            df[relevance_column] = None
+            df['distance'] = None
+        # Sort by relevance
+        df = df.sort_values(by=relevance_column, ascending=False)
         return df
 
+    def addapt_conditions_columns(self, conditions):
+        if conditions is None:
+            return
+        for condition in conditions:
+            if condition.column in KB_TO_VECTORDB_COLUMNS:
+                condition.column = KB_TO_VECTORDB_COLUMNS[condition.column]
+
+    def addapt_result_columns(self, df):
+        col_update = {}
+        for kb_col, vec_col in KB_TO_VECTORDB_COLUMNS.items():
+            if vec_col in df.columns:
+                col_update[vec_col] = kb_col
+
+        df = df.rename(columns=col_update)
+
+        columns = list(df.columns)
+        # update id, get from metadata
+        df[TableField.ID.value] = df[TableField.METADATA.value].apply(
+            lambda m: None if m is None else m.get('original_row_id')
+        )
+
+        # id on first place
+        return df[[TableField.ID.value] + columns]
+
     def insert_files(self, file_names: List[str]):
         """Process and insert files"""
         if not self.document_loader:
@@ -202,7 +315,9 @@ class KnowledgeBaseTable:
 
         # send to vectordb
         db_handler = self.get_vector_db()
-        db_handler.query(query)
+        conditions = db_handler.extract_conditions(query.where)
+        self.addapt_conditions_columns(conditions)
+        db_handler.dispatch_update(query, conditions)
 
     def delete_query(self, query: Delete):
         """
@@ -217,7 +332,9 @@ class KnowledgeBaseTable:
 
         # send to vectordb
         db_handler = self.get_vector_db()
-        db_handler.dispatch_delete(query)
+        conditions = db_handler.extract_conditions(query.where)
+        self.addapt_conditions_columns(conditions)
+        db_handler.dispatch_delete(query, conditions)
 
     def hybrid_search(
         self,

mindsdb/interfaces/knowledge_base/preprocessing/document_preprocessor.py

@@ -92,9 +92,7 @@ class DocumentPreprocessor:
 
     def _generate_chunk_id(
         self,
-        content: str,
         chunk_index: Optional[int] = None,
-        content_column: str = None,
         provided_id: str = None,
     ) -> str:
         """Generate deterministic ID for a chunk"""
@@ -262,15 +260,8 @@ Please give a short succinct context to situate this chunk within the overall do
             if doc.metadata:
                 metadata.update(doc.metadata)
 
-            # Pass through doc.id and content_column
-            content_column = (
-                doc.metadata.get("content_column") if doc.metadata else None
-            )
             chunk_id = self._generate_chunk_id(
-                processed_content,
-                chunk_index,
-                content_column=content_column,
-                provided_id=doc.id,
+                chunk_index=chunk_index, provided_id=doc.id
             )
             processed_chunks.append(
                 ProcessedChunk(
@@ -335,7 +326,7 @@ class TextChunkingPreprocessor(DocumentPreprocessor):
 
                 # Pass through doc.id and content_column
                 id = self._generate_chunk_id(
-                    chunk_doc.content, content_column=content_column, provided_id=doc.id
+                    chunk_index=0, provided_id=doc.id
                 )
                 processed_chunks.append(
                     ProcessedChunk(
@@ -358,9 +349,7 @@ class TextChunkingPreprocessor(DocumentPreprocessor):
 
                     # Pass through doc.id and content_column
                     chunk_id = self._generate_chunk_id(
-                        chunk_doc.content,
-                        i,
-                        content_column=content_column,
+                        chunk_index=i,
                         provided_id=doc.id,
                     )
                     processed_chunks.append(

mindsdb/interfaces/query_context/context_controller.py

@@ -156,10 +156,12 @@ class QueryContextController:
         last_values = {}
         for query, info in l_query.get_init_queries():
 
-            data, columns_info = dn.query(
+            response = dn.query(
                 query=query,
                 session=session
             )
+            data = response.data_frame
+            columns_info = response.columns
 
             if len(data) == 0:
                 value = None

mindsdb/utilities/config.py

@@ -201,6 +201,14 @@ class Config:
                     "host": api_host,
                     "port": "55432",
                     "database": "mindsdb"
+                },
+                "mcp": {
+                    "host": api_host,
+                    "port": "47337",
+                    "enabled": True,
+                    "restart_on_failure": True,
+                    "max_restart_count": 1,
+                    "max_restart_interval_seconds": 60
                 }
             },
             "cache": {

mindsdb/utilities/starters.py

@@ -31,3 +31,10 @@ def start_ml_task_queue(*args, **kwargs):
 def start_scheduler(*args, **kwargs):
     from mindsdb.interfaces.jobs.scheduler import start
     start(*args, **kwargs)
+
+
+def start_mcp(*args, **kwargs):
+    """Start the MCP server"""
+    from mindsdb.api.mcp.start import start
+
+    start(*args, **kwargs)