MindsDB 25.3.4.1__py3-none-any.whl → 25.4.1.0__py3-none-any.whl

mindsdb/integrations/utilities/rag/rerankers/reranker_compressor.py

@@ -127,17 +127,21 @@ class LLMReranker(BaseDocumentCompressor):
                     ranked_results.append((batch[idx][1], score))
 
                     # Check if we should stop early
-                    high_scoring_docs = [r for r in ranked_results if r[1] >= self.filtering_threshold]
-                    can_stop_early = (
-                        self.early_stop  # Early stopping is enabled
-                        and self.num_docs_to_keep  # We have a target number of docs
-                        and len(high_scoring_docs) >= self.num_docs_to_keep  # Found enough good docs
-                        and score >= self.early_stop_threshold  # Current doc is good enough
-                    )
-
-                    if can_stop_early:
-                        log.info(f"Early stopping after finding {self.num_docs_to_keep} documents with high confidence")
-                        return ranked_results
+                    try:
+                        high_scoring_docs = [r for r in ranked_results if r[1] >= self.filtering_threshold]
+                        can_stop_early = (
+                            self.early_stop  # Early stopping is enabled
+                            and self.num_docs_to_keep  # We have a target number of docs
+                            and len(high_scoring_docs) >= self.num_docs_to_keep  # Found enough good docs
+                            and score >= self.early_stop_threshold  # Current doc is good enough
+                        )
+
+                        if can_stop_early:
+                            log.info(f"Early stopping after finding {self.num_docs_to_keep} documents with high confidence")
+                            return ranked_results
+                    except Exception as e:
+                        # Don't let early stopping errors stop the whole process
+                        log.warning(f"Error in early stopping check: {str(e)}")
 
             except Exception as e:
                 log.error(f"Batch processing error: {str(e)}")
@@ -222,3 +226,109 @@ class LLMReranker(BaseDocumentCompressor):
             "temperature": self.temperature,
             "remove_irrelevant": self.remove_irrelevant,
         }
+
+    def get_scores(self, query: str, documents: list[str], disable_events: bool = True):
+        """
+        Get relevance scores for documents given a query.
+        Args:
+            query: The query text
+            documents: List of document texts to score
+            disable_events: Whether to disable event dispatching (default True)
+        Returns:
+            List of relevance scores
+        """
+        query_document_pairs = [(query, doc) for doc in documents]
+        # Create event loop and run async code
+        import asyncio
+        try:
+            loop = asyncio.get_running_loop()
+        except RuntimeError:
+            # If no running loop exists, create a new one
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+        # If disable_events is True, we need to modify the _rank function to not use dispatch_custom_event
+        if disable_events:
+            # Create a wrapper function that doesn't dispatch events
+            async def _rank_without_events(query_document_pairs):
+                ranked_results = []
+                # Process in larger batches for better throughput
+                batch_size = min(self.max_concurrent_requests * 2, len(query_document_pairs))
+                for i in range(0, len(query_document_pairs), batch_size):
+                    batch = query_document_pairs[i:i + batch_size]
+                    try:
+                        # Define a no-events version of search_relevancy inside this closure
+                        async def search_relevancy_no_events(query, document):
+                            await self._init_client()
+                            async with self._semaphore:
+                                for attempt in range(self.max_retries):
+                                    try:
+                                        response = await self.client.chat.completions.create(
+                                            model=self.model,
+                                            messages=[
+                                                {"role": "system", "content": "Rate the relevance of the document to the query. Respond with 'yes' or 'no'."},
+                                                {"role": "user", "content": f"Query: {query}\nDocument: {document}\nIs this document relevant?"}
+                                            ],
+                                            temperature=self.temperature,
+                                            n=1,
+                                            logprobs=True,
+                                            max_tokens=1
+                                        )
+                                        # Extract response and confidence score
+                                        answer = response.choices[0].message.content
+                                        logprob = response.choices[0].logprobs.content[0].logprob
+                                        # No event dispatch here
+                                        return {"document": document, "answer": answer, "logprob": logprob}
+                                    except Exception as e:
+                                        if attempt == self.max_retries - 1:
+                                            log.error(f"Failed after {self.max_retries} attempts: {str(e)}")
+                                            raise
+                                        # Exponential backoff with jitter
+                                        retry_delay = self.retry_delay * (2 ** attempt) + random.uniform(0, 0.1)
+                                        await asyncio.sleep(retry_delay)
+                        # Use our no-events version for this batch
+                        results = await asyncio.gather(
+                            *[search_relevancy_no_events(query=query, document=document) for (query, document) in batch],
+                            return_exceptions=True
+                        )
+                        for idx, result in enumerate(results):
+                            if isinstance(result, Exception):
+                                log.error(f"Error processing document {i+idx}: {str(result)}")
+                                ranked_results.append((batch[idx][1], 0.0))
+                                continue
+                            answer = result["answer"]
+                            logprob = result["logprob"]
+                            prob = math.exp(logprob)
+                            # Convert answer to score using the model's confidence
+                            if answer.lower().strip() == "yes":
+                                score = prob  # If yes, use the model's confidence
+                            elif answer.lower().strip() == "no":
+                                score = 1 - prob  # If no, invert the confidence
+                            else:
+                                score = 0.5 * prob  # For unclear answers, reduce confidence
+                            ranked_results.append((batch[idx][1], score))
+                            # Check if we should stop early
+                            try:
+                                high_scoring_docs = [r for r in ranked_results if r[1] >= self.filtering_threshold]
+                                can_stop_early = (
+                                    self.early_stop  # Early stopping is enabled
+                                    and self.num_docs_to_keep  # We have a target number of docs
+                                    and len(high_scoring_docs) >= self.num_docs_to_keep  # Found enough good docs
+                                    and score >= self.early_stop_threshold  # Current doc is good enough
+                                )
+                                if can_stop_early:
+                                    log.info(f"Early stopping after finding {self.num_docs_to_keep} documents with high confidence")
+                                    return ranked_results
+                            except Exception as e:
+                                # Don't let early stopping errors stop the whole process
+                                log.warning(f"Error in early stopping check: {str(e)}")
+                    except Exception as e:
+                        log.error(f"Batch processing error: {str(e)}")
+                        continue
+                return ranked_results
+            # Use our no-events version
+            documents_and_scores = loop.run_until_complete(_rank_without_events(query_document_pairs))
+        else:
+            # Use the original _rank method
+            documents_and_scores = loop.run_until_complete(self._rank(query_document_pairs))
+        scores = [score for _, score in documents_and_scores]
+        return scores

mindsdb/interfaces/database/projects.py

@@ -296,6 +296,19 @@ class Project:
         ]
         return data
 
+    def get_knowledge_bases(self):
+        from mindsdb.api.executor.controllers.session_controller import SessionController
+        session = SessionController()
+
+        return {
+            kb['name']: {
+                'type': 'knowledge_base',
+                'id': kb['id'],
+                'deletable': True
+            }
+            for kb in session.kb_controller.list(self.name)
+        }
+
     def get_views(self):
         records = (
             db.session.query(db.View).filter_by(
@@ -353,6 +366,8 @@ class Project:
         for agent in agents:
             data[agent['name']] = agent['metadata']
 
+        data.update(self.get_knowledge_bases())
+
         return data
 
     def get_columns(self, table_name: str):

mindsdb/interfaces/knowledge_base/controller.py

@@ -4,6 +4,7 @@ from typing import Dict, List, Optional
 
 import pandas as pd
 import hashlib
+import numpy as np
 
 from mindsdb_sql_parser.ast import (
     BinaryOperation,
@@ -37,9 +38,16 @@ from mindsdb.utilities.exception import EntityExistsError, EntityNotExistsError
 
 from mindsdb.api.executor.command_executor import ExecuteCommands
 from mindsdb.utilities import log
+from mindsdb.integrations.utilities.rag.rerankers.reranker_compressor import LLMReranker
 
 logger = log.getLogger(__name__)
 
+KB_TO_VECTORDB_COLUMNS = {
+    'id': 'original_row_id',
+    'chunk_id': 'id',
+    'chunk_content': 'content'
+}
+
 
 class KnowledgeBaseTable:
     """
@@ -103,7 +111,9 @@ class KnowledgeBaseTable:
         db_handler = self.get_vector_db()
         logger.debug(f"Using vector db handler: {type(db_handler)}")
 
-        df = db_handler.dispatch_select(query)
+        conditions = db_handler.extract_conditions(query.where)
+        self.addapt_conditions_columns(conditions)
+        df = db_handler.dispatch_select(query, conditions)
 
         if df is not None:
 
@@ -115,8 +125,72 @@ class KnowledgeBaseTable:
         else:
             logger.warning("Query returned no data")
 
+        rerank_model = self._kb.params.get("rerank_model")
+        if rerank_model and df is not None and not df.empty:
+            try:
+                logger.info(f"Using reranker model: {rerank_model}")
+                reranker = LLMReranker(model=rerank_model)
+                # convert response from a dataframe to a list of strings
+                content_column = df[TableField.CONTENT.value]
+                # convert to list
+                documents = content_column.tolist()
+                # Extract query text from WHERE clause if it exists
+                query_text = ""
+                if query.where:
+                    def extract_content(node, **kwargs):
+                        nonlocal query_text
+                        is_binary_op = isinstance(node, BinaryOperation)
+                        is_identifier = isinstance(node.args[0], Identifier)
+                        is_content = node.args[0].parts[-1].lower() == 'content'
+                        is_constant = isinstance(node.args[1], Constant)
+                        if is_binary_op and is_identifier and is_content and is_constant:
+                            query_text = node.args[1].value
+                    query_traversal(query.where, extract_content)
+                    logger.debug(f"Extracted query text: {query_text}")
+                # Get scores from reranker
+                scores = reranker.get_scores(query_text, documents)
+                # Add scores as a new column for filtering
+                scores_array = np.array(scores)
+                # Add temporary column for sorting
+                df['_relevance_score'] = scores
+                # Filter by score threshold using numpy array for element-wise comparison
+                df = df[scores_array > reranker.filtering_threshold]
+                # Sort by relevance (higher score = more relevant)
+                df = df.sort_values(by='_relevance_score', ascending=False)
+                # Remove temporary column
+                # df = df.drop(columns=['_relevance_score'])
+                # Apply original limit if it exists
+                if query.limit and len(df) > query.limit.value:
+                    df = df.iloc[:query.limit.value]
+                logger.debug(f"Applied reranking with model {rerank_model}")
+            except Exception as e:
+                logger.error(f"Error during reranking: {str(e)}")
+
+        df = self.addapt_result_columns(df)
         return df
 
+    def addapt_conditions_columns(self, conditions):
+        if conditions is None:
+            return
+        for condition in conditions:
+            if condition.column in KB_TO_VECTORDB_COLUMNS:
+                condition.column = KB_TO_VECTORDB_COLUMNS[condition.column]
+
+    def addapt_result_columns(self, df):
+        col_update = {}
+        for kb_col, vec_col in KB_TO_VECTORDB_COLUMNS.items():
+            if vec_col in df.columns:
+                col_update[vec_col] = kb_col
+
+        df = df.rename(columns=col_update)
+
+        columns = list(df.columns)
+        # update id, get from metadata
+        df[TableField.ID.value] = df[TableField.METADATA.value].apply(lambda m: m.get('original_row_id'))
+
+        # id on first place
+        return df[[TableField.ID.value] + columns]
+
     def insert_files(self, file_names: List[str]):
         """Process and insert files"""
         if not self.document_loader:
@@ -217,7 +291,9 @@ class KnowledgeBaseTable:
 
         # send to vectordb
         db_handler = self.get_vector_db()
-        db_handler.dispatch_delete(query)
+        conditions = db_handler.extract_conditions(query.where)
+        self.addapt_conditions_columns(conditions)
+        db_handler.dispatch_delete(query, conditions)
 
     def hybrid_search(
         self,

mindsdb/utilities/config.py

@@ -201,6 +201,14 @@ class Config:
                     "host": api_host,
                     "port": "55432",
                     "database": "mindsdb"
+                },
+                "mcp": {
+                    "host": api_host,
+                    "port": "47337",
+                    "enabled": True,
+                    "restart_on_failure": True,
+                    "max_restart_count": 1,
+                    "max_restart_interval_seconds": 60
                 }
             },
             "cache": {

mindsdb/utilities/render/sqlalchemy_render.py

@@ -27,6 +27,7 @@ types_map = {}
 for type_name in sa_type_names:
     types_map[type_name.upper()] = getattr(sa.types, type_name)
 types_map['BOOL'] = types_map['BOOLEAN']
+types_map['DEC'] = types_map['DECIMAL']
 
 
 class RenderError(Exception):
@@ -43,6 +44,11 @@ class INTERVAL(ColumnElement):
 @compiles(INTERVAL)
 def _compile_interval(element, compiler, **kw):
     items = element.info.split(' ', maxsplit=1)
+    if compiler.dialect.name == 'oracle' and len(items) == 2:
+        # replace to singular names (remove leading S if exists)
+        if items[1].upper().endswith('S'):
+            items[1] = items[1][:-1]
+
     if compiler.dialect.driver in ['snowflake']:
         # quote all
         args = " ".join(map(str, items))
@@ -118,6 +124,8 @@ class SqlalchemyRender:
         self.dialect = dialect(paramstyle="named")
         self.dialect.div_is_floordiv = False
 
+        self.selects_stack = []
+
         if dialect_name == 'mssql':
             # update version to MS_2008_VERSION for supports_multivalues_insert
             self.dialect.server_version_info = (10,)
@@ -143,8 +151,10 @@ class SqlalchemyRender:
                 part = self.dialect.identifier_preparer.quote(i)
 
             parts2.append(part)
-
-        return sa.column('.'.join(parts2), is_literal=True)
+        text = '.'.join(parts2)
+        if identifier.is_outer and self.dialect.name == 'oracle':
+            text += '(+)'
+        return sa.column(text, is_literal=True)
 
     def get_alias(self, alias):
         if alias is None or len(alias.parts) == 0:
@@ -152,6 +162,9 @@ class SqlalchemyRender:
         if len(alias.parts) > 1:
             raise NotImplementedError(f'Multiple alias {alias.parts}')
 
+        if self.selects_stack:
+            self.selects_stack[-1]['aliases'].append(alias)
+
         is_quoted = get_is_quoted(alias)[0]
         return AttributedStr(alias.parts[0], is_quoted)
 
@@ -205,12 +218,18 @@ class SqlalchemyRender:
                 alias = self.get_alias(t.alias)
                 col = col.label(alias)
         elif isinstance(t, ast.Function):
-            fnc = self.to_function(t)
+            col = self.to_function(t)
             if t.alias:
                 alias = self.get_alias(t.alias)
+                col = col.label(alias)
             else:
                 alias = str(t.op)
-            col = fnc.label(alias)
+                if self.selects_stack:
+                    aliases = self.selects_stack[-1]['aliases']
+                    if alias not in aliases:
+                        aliases.append(alias)
+                        col = col.label(alias)
+
         elif isinstance(t, ast.BinaryOperation):
             ops = {
                 "+": operators.add,
@@ -432,9 +451,9 @@ class SqlalchemyRender:
             return typename
 
         typename = typename.upper()
-        if re.match(r'^INT[\d]*$', typename):
+        if re.match(r'^INT[\d]+$', typename):
             typename = 'BIGINT'
-        if re.match(r'^FLOAT[\d]*$', typename):
+        if re.match(r'^FLOAT[\d]+$', typename):
             typename = 'FLOAT'
 
         return types_map[typename]
@@ -513,6 +532,9 @@ class SqlalchemyRender:
             return self.prepare_union(node)
 
         cols = []
+
+        self.selects_stack.append({'aliases': []})
+
         for t in node.targets:
             col = self.to_expression(t)
             cols.append(col)
@@ -647,6 +669,8 @@ class SqlalchemyRender:
             else:
                 raise NotImplementedError(f'Select mode: {node.mode}')
 
+        self.selects_stack.pop()
+
         return query
 
     def prepare_union(self, from_table):

mindsdb/utilities/starters.py

@@ -31,3 +31,10 @@ def start_ml_task_queue(*args, **kwargs):
 def start_scheduler(*args, **kwargs):
     from mindsdb.interfaces.jobs.scheduler import start
     start(*args, **kwargs)
+
+
+def start_mcp(*args, **kwargs):
+    """Start the MCP server"""
+    from mindsdb.api.mcp.start import start
+
+    start(*args, **kwargs)