MindsDB 25.3.4.2__py3-none-any.whl → 25.4.1.0__py3-none-any.whl

mindsdb/__about__.py

@@ -1,6 +1,6 @@
 __title__ = 'MindsDB'
 __package_name__ = 'mindsdb'
-__version__ = '25.3.4.2'
+__version__ = '25.4.1.0'
 __description__ = "MindsDB's AI SQL Server enables developers to build AI tools that need access to real-time data to perform their tasks"
 __email__ = "jorge@mindsdb.com"
 __author__ = 'MindsDB Inc'

mindsdb/__main__.py

@@ -24,7 +24,8 @@ from mindsdb.__about__ import __version__ as mindsdb_version
 from mindsdb.utilities.config import config
 from mindsdb.utilities.exception import EntityNotExistsError
 from mindsdb.utilities.starters import (
-    start_http, start_mysql, start_mongo, start_postgres, start_ml_task_queue, start_scheduler, start_tasks
+    start_http, start_mysql, start_mongo, start_postgres, start_ml_task_queue, start_scheduler, start_tasks,
+    start_mcp
 )
 from mindsdb.utilities.ps import is_pid_listen_port, get_child_pids
 from mindsdb.utilities.functions import get_versions_where_predictors_become_obsolete
@@ -57,6 +58,7 @@ class TrunkProcessEnum(Enum):
     JOBS = 'jobs'
     TASKS = 'tasks'
     ML_TASK_QUEUE = 'ml_task_queue'
+    MCP = 'mcp'
 
     @classmethod
     def _missing_(cls, value):
@@ -221,9 +223,9 @@ if __name__ == '__main__':
     ctx.set_default()
 
     # ---- CHECK SYSTEM ----
-    if not (sys.version_info[0] >= 3 and sys.version_info[1] >= 9):
+    if not (sys.version_info[0] >= 3 and sys.version_info[1] >= 10):
         print("""
-     MindsDB requires Python >= 3.9 to run
+     MindsDB requires Python >= 3.10 to run
 
      Once you have supported Python version installed you can start mindsdb as follows:
 
@@ -385,6 +387,7 @@ if __name__ == '__main__':
 
     http_api_config = config['api']['http']
     mysql_api_config = config['api']['mysql']
+    mcp_api_config = config['api']['mcp']
     trunc_processes_struct = {
         TrunkProcessEnum.HTTP: TrunkProcessData(
             name=TrunkProcessEnum.HTTP.value,
@@ -434,11 +437,25 @@ if __name__ == '__main__':
             name=TrunkProcessEnum.ML_TASK_QUEUE.value,
             entrypoint=start_ml_task_queue,
             args=(config.cmd_args.verbose,)
+        ),
+        TrunkProcessEnum.MCP: TrunkProcessData(
+            name=TrunkProcessEnum.MCP.value,
+            entrypoint=start_mcp,
+            port=mcp_api_config.get('port', 47337),
+            args=(config.cmd_args.verbose,),
+            restart_on_failure=mcp_api_config.get('restart_on_failure', False),
+            max_restart_count=mcp_api_config.get('max_restart_count', TrunkProcessData.max_restart_count),
+            max_restart_interval_seconds=mcp_api_config.get(
+                'max_restart_interval_seconds', TrunkProcessData.max_restart_interval_seconds
+            )
         )
     }
 
     for api_enum in api_arr:
-        trunc_processes_struct[api_enum].need_to_run = True
+        if api_enum in trunc_processes_struct:
+            trunc_processes_struct[api_enum].need_to_run = True
+        else:
+            logger.error(f"ERROR: {api_enum} API is not a valid api in config")
 
     if config['jobs']['disable'] is False:
         trunc_processes_struct[TrunkProcessEnum.JOBS].need_to_run = True

mindsdb/api/mcp/start.py

@@ -0,0 +1,152 @@
+from contextlib import asynccontextmanager
+from collections.abc import AsyncIterator
+from typing import Optional, Dict, Any
+from dataclasses import dataclass
+
+from mcp.server.fastmcp import FastMCP
+from mindsdb.api.mysql.mysql_proxy.classes.fake_mysql_proxy import FakeMysqlProxy
+from mindsdb.api.executor.data_types.response_type import RESPONSE_TYPE as SQL_RESPONSE_TYPE
+from mindsdb.utilities import log
+from mindsdb.utilities.config import Config
+from mindsdb.interfaces.storage import db
+
+logger = log.getLogger(__name__)
+
+
+@dataclass
+class AppContext:
+    db: Any
+
+
+@asynccontextmanager
+async def app_lifespan(server: FastMCP) -> AsyncIterator[AppContext]:
+    """Manage application lifecycle with type-safe context"""
+    # Initialize on startup
+    db.init()
+    try:
+        yield AppContext(db=db)
+    finally:
+        # TODO: We need better way to handle this in storage/db.py
+        pass
+
+
+# Configure server with lifespan
+mcp = FastMCP(
+    "MindsDB",
+    lifespan=app_lifespan,
+    dependencies=["mindsdb"]  # Add any additional dependencies
+)
+# MCP Queries
+LISTING_QUERY = "SHOW DATABASES"
+
+
+@mcp.tool()
+def query(query: str, context: Optional[Dict] = None) -> Dict[str, Any]:
+    """
+    Execute a SQL query against MindsDB
+
+    Args:
+        query: The SQL query to execute
+        context: Optional context parameters for the query
+
+    Returns:
+        Dict containing the query results or error information
+    """
+
+    if context is None:
+        context = {}
+
+    logger.debug(f'Incoming MCP query: {query}')
+
+    mysql_proxy = FakeMysqlProxy()
+    mysql_proxy.set_context(context)
+
+    try:
+        result = mysql_proxy.process_query(query)
+
+        if result.type == SQL_RESPONSE_TYPE.OK:
+            return {"type": SQL_RESPONSE_TYPE.OK}
+
+        if result.type == SQL_RESPONSE_TYPE.TABLE:
+            return {
+                "type": SQL_RESPONSE_TYPE.TABLE,
+                "data": result.data.to_lists(json_types=True),
+                "column_names": [
+                    x["alias"] or x["name"] if "alias" in x else x["name"]
+                    for x in result.columns
+                ],
+            }
+        else:
+            return {
+                "type": SQL_RESPONSE_TYPE.ERROR,
+                "error_code": 0,
+                "error_message": "Unknown response type"
+            }
+
+    except Exception as e:
+        logger.error(f"Error processing query: {str(e)}")
+        return {
+            "type": SQL_RESPONSE_TYPE.ERROR,
+            "error_code": 0,
+            "error_message": str(e)
+        }
+
+
+@mcp.tool()
+def list_databases() -> Dict[str, Any]:
+    """
+    List all databases in MindsDB along with their tables
+
+    Returns:
+        Dict containing the list of databases and their tables
+    """
+
+    mysql_proxy = FakeMysqlProxy()
+
+    try:
+        result = mysql_proxy.process_query(LISTING_QUERY)
+        if result.type == SQL_RESPONSE_TYPE.ERROR:
+            return {
+                "type": "error",
+                "error_code": result.error_code,
+                "error_message": result.error_message,
+            }
+
+        elif result.type == SQL_RESPONSE_TYPE.OK:
+            return {"type": "ok"}
+
+        elif result.type == SQL_RESPONSE_TYPE.TABLE:
+            data = result.data.to_lists(json_types=True)
+            return data
+
+    except Exception as e:
+        return {
+            "type": "error",
+            "error_code": 0,
+            "error_message": str(e),
+        }
+
+
+def start(*args, **kwargs):
+    """Start the MCP server
+    Args:
+        host (str): Host to bind to
+        port (int): Port to listen on
+    """
+    config = Config()
+    port = int(config['api'].get('mcp', {}).get('port', 47337))
+    host = config['api'].get('mcp', {}).get('host', '127.0.0.1')
+
+    logger.info(f"Starting MCP server on {host}:{port}")
+    mcp.settings.host = host
+    mcp.settings.port = port
+
+    try:
+        mcp.run(transport="sse")  # Use SSE transport instead of stdio
+    except Exception as e:
+        logger.error(f"Error starting MCP server: {str(e)}")
+        raise
+
+
+if __name__ == "__main__":
+    start()

mindsdb/integrations/libs/vectordatabase_handler.py

@@ -89,7 +89,7 @@ class VectorStoreHandler(BaseHandler):
         else:
             return value
 
-    def _extract_conditions(self, where_statement) -> Optional[List[FilterCondition]]:
+    def extract_conditions(self, where_statement) -> Optional[List[FilterCondition]]:
         conditions = []
         # parse conditions
         if where_statement is not None:
@@ -110,13 +110,7 @@ class VectorStoreHandler(BaseHandler):
                             right_hand = node.args[1].value
                     elif isinstance(node.args[1], Tuple):
                         # Constant could be actually a list i.e. [1.2, 3.2]
-                        right_hand = [
-                            ast.literal_eval(item.value)
-                            if isinstance(item, Constant)
-                            and not isinstance(item.value, list)
-                            else item.value
-                            for item in node.args[1].items
-                        ]
+                        right_hand = [item.value for item in node.args[1].items]
                     else:
                         raise Exception(f"Unsupported right hand side: {node.args[1]}")
                     conditions.append(
@@ -125,18 +119,21 @@ class VectorStoreHandler(BaseHandler):
 
             query_traversal(where_statement, _extract_comparison_conditions)
 
-            # try to treat conditions that are not in TableField as metadata conditions
-            for condition in conditions:
-                if not self._is_condition_allowed(condition):
-                    condition.column = (
-                        TableField.METADATA.value + "." + condition.column
-                    )
-
         else:
             conditions = None
 
         return conditions
 
+    def _convert_metadata_filters(self, conditions):
+        if conditions is None:
+            return
+        # try to treat conditions that are not in TableField as metadata conditions
+        for condition in conditions:
+            if not self._is_condition_allowed(condition):
+                condition.column = (
+                    TableField.METADATA.value + "." + condition.column
+                )
+
     def _is_columns_allowed(self, columns: List[str]) -> bool:
         """
         Check if columns are allowed.
@@ -325,14 +322,16 @@ class VectorStoreHandler(BaseHandler):
         if not df_insert.empty:
             self.insert(table_name, df_insert)
 
-    def dispatch_delete(self, query: Delete):
+    def dispatch_delete(self, query: Delete, conditions: List[FilterCondition] = None):
         """
         Dispatch delete query to the appropriate method.
         """
         # parse key arguments
         table_name = query.table.parts[-1]
-        where_statement = query.where
-        conditions = self._extract_conditions(where_statement)
+        if conditions is None:
+            where_statement = query.where
+            conditions = self.extract_conditions(where_statement)
+        self._convert_metadata_filters(conditions)
 
         # dispatch delete
         return self.delete(table_name, conditions=conditions)
@@ -356,9 +355,10 @@ class VectorStoreHandler(BaseHandler):
             )
 
         # check if columns are allowed
-        where_statement = query.where
         if conditions is None:
-            conditions = self._extract_conditions(where_statement)
+            where_statement = query.where
+            conditions = self.extract_conditions(where_statement)
+        self._convert_metadata_filters(conditions)
 
         # get offset and limit
         offset = query.offset.value if query.offset is not None else None

mindsdb/integrations/utilities/rag/rerankers/reranker_compressor.py

@@ -127,17 +127,21 @@ class LLMReranker(BaseDocumentCompressor):
                     ranked_results.append((batch[idx][1], score))
 
                     # Check if we should stop early
-                    high_scoring_docs = [r for r in ranked_results if r[1] >= self.filtering_threshold]
-                    can_stop_early = (
-                        self.early_stop  # Early stopping is enabled
-                        and self.num_docs_to_keep  # We have a target number of docs
-                        and len(high_scoring_docs) >= self.num_docs_to_keep  # Found enough good docs
-                        and score >= self.early_stop_threshold  # Current doc is good enough
-                    )
-
-                    if can_stop_early:
-                        log.info(f"Early stopping after finding {self.num_docs_to_keep} documents with high confidence")
-                        return ranked_results
+                    try:
+                        high_scoring_docs = [r for r in ranked_results if r[1] >= self.filtering_threshold]
+                        can_stop_early = (
+                            self.early_stop  # Early stopping is enabled
+                            and self.num_docs_to_keep  # We have a target number of docs
+                            and len(high_scoring_docs) >= self.num_docs_to_keep  # Found enough good docs
+                            and score >= self.early_stop_threshold  # Current doc is good enough
+                        )
+
+                        if can_stop_early:
+                            log.info(f"Early stopping after finding {self.num_docs_to_keep} documents with high confidence")
+                            return ranked_results
+                    except Exception as e:
+                        # Don't let early stopping errors stop the whole process
+                        log.warning(f"Error in early stopping check: {str(e)}")
 
             except Exception as e:
                 log.error(f"Batch processing error: {str(e)}")
@@ -222,3 +226,109 @@ class LLMReranker(BaseDocumentCompressor):
             "temperature": self.temperature,
             "remove_irrelevant": self.remove_irrelevant,
         }
+
+    def get_scores(self, query: str, documents: list[str], disable_events: bool = True):
+        """
+        Get relevance scores for documents given a query.
+        Args:
+            query: The query text
+            documents: List of document texts to score
+            disable_events: Whether to disable event dispatching (default True)
+        Returns:
+            List of relevance scores
+        """
+        query_document_pairs = [(query, doc) for doc in documents]
+        # Create event loop and run async code
+        import asyncio
+        try:
+            loop = asyncio.get_running_loop()
+        except RuntimeError:
+            # If no running loop exists, create a new one
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+        # If disable_events is True, we need to modify the _rank function to not use dispatch_custom_event
+        if disable_events:
+            # Create a wrapper function that doesn't dispatch events
+            async def _rank_without_events(query_document_pairs):
+                ranked_results = []
+                # Process in larger batches for better throughput
+                batch_size = min(self.max_concurrent_requests * 2, len(query_document_pairs))
+                for i in range(0, len(query_document_pairs), batch_size):
+                    batch = query_document_pairs[i:i + batch_size]
+                    try:
+                        # Define a no-events version of search_relevancy inside this closure
+                        async def search_relevancy_no_events(query, document):
+                            await self._init_client()
+                            async with self._semaphore:
+                                for attempt in range(self.max_retries):
+                                    try:
+                                        response = await self.client.chat.completions.create(
+                                            model=self.model,
+                                            messages=[
+                                                {"role": "system", "content": "Rate the relevance of the document to the query. Respond with 'yes' or 'no'."},
+                                                {"role": "user", "content": f"Query: {query}\nDocument: {document}\nIs this document relevant?"}
+                                            ],
+                                            temperature=self.temperature,
+                                            n=1,
+                                            logprobs=True,
+                                            max_tokens=1
+                                        )
+                                        # Extract response and confidence score
+                                        answer = response.choices[0].message.content
+                                        logprob = response.choices[0].logprobs.content[0].logprob
+                                        # No event dispatch here
+                                        return {"document": document, "answer": answer, "logprob": logprob}
+                                    except Exception as e:
+                                        if attempt == self.max_retries - 1:
+                                            log.error(f"Failed after {self.max_retries} attempts: {str(e)}")
+                                            raise
+                                        # Exponential backoff with jitter
+                                        retry_delay = self.retry_delay * (2 ** attempt) + random.uniform(0, 0.1)
+                                        await asyncio.sleep(retry_delay)
+                        # Use our no-events version for this batch
+                        results = await asyncio.gather(
+                            *[search_relevancy_no_events(query=query, document=document) for (query, document) in batch],
+                            return_exceptions=True
+                        )
+                        for idx, result in enumerate(results):
+                            if isinstance(result, Exception):
+                                log.error(f"Error processing document {i+idx}: {str(result)}")
+                                ranked_results.append((batch[idx][1], 0.0))
+                                continue
+                            answer = result["answer"]
+                            logprob = result["logprob"]
+                            prob = math.exp(logprob)
+                            # Convert answer to score using the model's confidence
+                            if answer.lower().strip() == "yes":
+                                score = prob  # If yes, use the model's confidence
+                            elif answer.lower().strip() == "no":
+                                score = 1 - prob  # If no, invert the confidence
+                            else:
+                                score = 0.5 * prob  # For unclear answers, reduce confidence
+                            ranked_results.append((batch[idx][1], score))
+                            # Check if we should stop early
+                            try:
+                                high_scoring_docs = [r for r in ranked_results if r[1] >= self.filtering_threshold]
+                                can_stop_early = (
+                                    self.early_stop  # Early stopping is enabled
+                                    and self.num_docs_to_keep  # We have a target number of docs
+                                    and len(high_scoring_docs) >= self.num_docs_to_keep  # Found enough good docs
+                                    and score >= self.early_stop_threshold  # Current doc is good enough
+                                )
+                                if can_stop_early:
+                                    log.info(f"Early stopping after finding {self.num_docs_to_keep} documents with high confidence")
+                                    return ranked_results
+                            except Exception as e:
+                                # Don't let early stopping errors stop the whole process
+                                log.warning(f"Error in early stopping check: {str(e)}")
+                    except Exception as e:
+                        log.error(f"Batch processing error: {str(e)}")
+                        continue
+                return ranked_results
+            # Use our no-events version
+            documents_and_scores = loop.run_until_complete(_rank_without_events(query_document_pairs))
+        else:
+            # Use the original _rank method
+            documents_and_scores = loop.run_until_complete(self._rank(query_document_pairs))
+        scores = [score for _, score in documents_and_scores]
+        return scores

mindsdb/interfaces/database/projects.py

@@ -296,6 +296,19 @@ class Project:
         ]
         return data
 
+    def get_knowledge_bases(self):
+        from mindsdb.api.executor.controllers.session_controller import SessionController
+        session = SessionController()
+
+        return {
+            kb['name']: {
+                'type': 'knowledge_base',
+                'id': kb['id'],
+                'deletable': True
+            }
+            for kb in session.kb_controller.list(self.name)
+        }
+
     def get_views(self):
         records = (
             db.session.query(db.View).filter_by(
@@ -353,6 +366,8 @@ class Project:
         for agent in agents:
             data[agent['name']] = agent['metadata']
 
+        data.update(self.get_knowledge_bases())
+
         return data
 
     def get_columns(self, table_name: str):

mindsdb/interfaces/knowledge_base/controller.py

@@ -4,6 +4,7 @@ from typing import Dict, List, Optional
 
 import pandas as pd
 import hashlib
+import numpy as np
 
 from mindsdb_sql_parser.ast import (
     BinaryOperation,
@@ -37,9 +38,16 @@ from mindsdb.utilities.exception import EntityExistsError, EntityNotExistsError
 
 from mindsdb.api.executor.command_executor import ExecuteCommands
 from mindsdb.utilities import log
+from mindsdb.integrations.utilities.rag.rerankers.reranker_compressor import LLMReranker
 
 logger = log.getLogger(__name__)
 
+KB_TO_VECTORDB_COLUMNS = {
+    'id': 'original_row_id',
+    'chunk_id': 'id',
+    'chunk_content': 'content'
+}
+
 
 class KnowledgeBaseTable:
     """
@@ -103,7 +111,9 @@ class KnowledgeBaseTable:
         db_handler = self.get_vector_db()
         logger.debug(f"Using vector db handler: {type(db_handler)}")
 
-        df = db_handler.dispatch_select(query)
+        conditions = db_handler.extract_conditions(query.where)
+        self.addapt_conditions_columns(conditions)
+        df = db_handler.dispatch_select(query, conditions)
 
         if df is not None:
 
@@ -115,8 +125,72 @@ class KnowledgeBaseTable:
         else:
             logger.warning("Query returned no data")
 
+        rerank_model = self._kb.params.get("rerank_model")
+        if rerank_model and df is not None and not df.empty:
+            try:
+                logger.info(f"Using reranker model: {rerank_model}")
+                reranker = LLMReranker(model=rerank_model)
+                # convert response from a dataframe to a list of strings
+                content_column = df[TableField.CONTENT.value]
+                # convert to list
+                documents = content_column.tolist()
+                # Extract query text from WHERE clause if it exists
+                query_text = ""
+                if query.where:
+                    def extract_content(node, **kwargs):
+                        nonlocal query_text
+                        is_binary_op = isinstance(node, BinaryOperation)
+                        is_identifier = isinstance(node.args[0], Identifier)
+                        is_content = node.args[0].parts[-1].lower() == 'content'
+                        is_constant = isinstance(node.args[1], Constant)
+                        if is_binary_op and is_identifier and is_content and is_constant:
+                            query_text = node.args[1].value
+                    query_traversal(query.where, extract_content)
+                    logger.debug(f"Extracted query text: {query_text}")
+                # Get scores from reranker
+                scores = reranker.get_scores(query_text, documents)
+                # Add scores as a new column for filtering
+                scores_array = np.array(scores)
+                # Add temporary column for sorting
+                df['_relevance_score'] = scores
+                # Filter by score threshold using numpy array for element-wise comparison
+                df = df[scores_array > reranker.filtering_threshold]
+                # Sort by relevance (higher score = more relevant)
+                df = df.sort_values(by='_relevance_score', ascending=False)
+                # Remove temporary column
+                # df = df.drop(columns=['_relevance_score'])
+                # Apply original limit if it exists
+                if query.limit and len(df) > query.limit.value:
+                    df = df.iloc[:query.limit.value]
+                logger.debug(f"Applied reranking with model {rerank_model}")
+            except Exception as e:
+                logger.error(f"Error during reranking: {str(e)}")
+
+        df = self.addapt_result_columns(df)
         return df
 
+    def addapt_conditions_columns(self, conditions):
+        if conditions is None:
+            return
+        for condition in conditions:
+            if condition.column in KB_TO_VECTORDB_COLUMNS:
+                condition.column = KB_TO_VECTORDB_COLUMNS[condition.column]
+
+    def addapt_result_columns(self, df):
+        col_update = {}
+        for kb_col, vec_col in KB_TO_VECTORDB_COLUMNS.items():
+            if vec_col in df.columns:
+                col_update[vec_col] = kb_col
+
+        df = df.rename(columns=col_update)
+
+        columns = list(df.columns)
+        # update id, get from metadata
+        df[TableField.ID.value] = df[TableField.METADATA.value].apply(lambda m: m.get('original_row_id'))
+
+        # id on first place
+        return df[[TableField.ID.value] + columns]
+
     def insert_files(self, file_names: List[str]):
         """Process and insert files"""
         if not self.document_loader:
@@ -217,7 +291,9 @@ class KnowledgeBaseTable:
 
         # send to vectordb
         db_handler = self.get_vector_db()
-        db_handler.dispatch_delete(query)
+        conditions = db_handler.extract_conditions(query.where)
+        self.addapt_conditions_columns(conditions)
+        db_handler.dispatch_delete(query, conditions)
 
     def hybrid_search(
         self,

mindsdb/utilities/config.py

@@ -201,6 +201,14 @@ class Config:
                     "host": api_host,
                     "port": "55432",
                     "database": "mindsdb"
+                },
+                "mcp": {
+                    "host": api_host,
+                    "port": "47337",
+                    "enabled": True,
+                    "restart_on_failure": True,
+                    "max_restart_count": 1,
+                    "max_restart_interval_seconds": 60
                 }
             },
             "cache": {

mindsdb/utilities/starters.py

@@ -31,3 +31,10 @@ def start_ml_task_queue(*args, **kwargs):
 def start_scheduler(*args, **kwargs):
     from mindsdb.interfaces.jobs.scheduler import start
     start(*args, **kwargs)
+
+
+def start_mcp(*args, **kwargs):
+    """Start the MCP server"""
+    from mindsdb.api.mcp.start import start
+
+    start(*args, **kwargs)