MindsDB 25.4.2.1__py3-none-any.whl → 25.4.3.0__py3-none-any.whl

mindsdb/__about__.py

@@ -1,6 +1,6 @@
 __title__ = 'MindsDB'
 __package_name__ = 'mindsdb'
-__version__ = '25.4.2.1'
+__version__ = '25.4.3.0'
 __description__ = "MindsDB's AI SQL Server enables developers to build AI tools that need access to real-time data to perform their tasks"
 __email__ = "jorge@mindsdb.com"
 __author__ = 'MindsDB Inc'

mindsdb/__main__.py

@@ -299,15 +299,38 @@ if __name__ == '__main__':
         logger.debug(f"Checking if default project {config.get('default_project')} exists")
         project_controller = ProjectController()
 
-        current_default_project = project_controller.get(is_default=True)
-        if current_default_project.record.name != config.get('default_project'):
+        try:
+            current_default_project = project_controller.get(is_default=True)
+        except EntityNotExistsError:
+            # In previous versions, the default project could be deleted. This is no longer possible.
+            current_default_project = None
+
+        if current_default_project:
+            if current_default_project.record.name != config.get('default_project'):
+                try:
+                    project_controller.get(name=config.get('default_project'))
+                    log.critical(f"A project with the name '{config.get('default_project')}' already exists")
+                    sys.exit(1)
+                except EntityNotExistsError:
+                    pass
+                project_controller.update(current_default_project.record.id, new_name=config.get('default_project'))
+
+        # Legacy: If the default project does not exist, mark the new one as default.
+        else:
             try:
-                new_default_project = project_controller.get(name=config.get('default_project'))
-                log.critical(f"A project with the name '{config.get('default_project')}' already exists")
-                sys.exit(1)
+                project_controller.get(name=config.get('default_project'))
             except EntityNotExistsError:
-                pass
-            project_controller.update(current_default_project.record.id, new_name=config.get('default_project'))
+                log.critical(
+                    f"A project with the name '{config.get('default_project')}' does not exist"
+                )
+                raise
+
+            project_controller.update(
+                name=config.get('default_project'),
+                new_metadata={
+                    "is_default": True
+                }
+            )
 
     apis = os.getenv('MINDSDB_APIS') or config.cmd_args.api
 

mindsdb/api/executor/sql_query/steps/fetch_dataframe_partition.py

@@ -1,6 +1,4 @@
 import pandas as pd
-import threading
-import queue
 from typing import List
 
 from mindsdb_sql_parser import ASTNode
@@ -11,9 +9,10 @@ from mindsdb.interfaces.query_context.context_controller import RunningQuery
 from mindsdb.api.executor.sql_query.result_set import ResultSet
 from mindsdb.utilities import log
 from mindsdb.utilities.config import Config
-from mindsdb.utilities.context import Context, context as ctx
 from mindsdb.utilities.partitioning import get_max_thread_count, split_data_frame
 from mindsdb.api.executor.sql_query.steps.fetch_dataframe import get_table_alias, get_fill_param_fnc
+from mindsdb.utilities.context_executor import ContextThreadPoolExecutor
+
 
 from .base import BaseStepCall
 
@@ -178,9 +177,6 @@ class FetchDataframePartitionCall(BaseStepCall):
         """
 
         # create communication queues
-        queue_in = queue.Queue()
-        queue_out = queue.Queue()
-        self.stop_event = threading.Event()
 
         if thread_count is None:
             thread_count = get_max_thread_count()
@@ -191,16 +187,9 @@ class FetchDataframePartitionCall(BaseStepCall):
         if partition_size < 10:
             partition_size = 10
 
-        # create N workers pool
-        workers = []
         results = []
 
-        try:
-            for i in range(thread_count):
-                worker = threading.Thread(target=self._worker, daemon=True, args=(ctx.dump(), queue_in,
-                                                                                  queue_out, self.stop_event))
-                worker.start()
-                workers.append(worker)
+        with ContextThreadPoolExecutor(max_workers=thread_count) as executor:
 
             while True:
                 # fetch batch
@@ -220,69 +209,23 @@ class FetchDataframePartitionCall(BaseStepCall):
                 max_track_value = run_query.get_max_track_value(df)
 
                 # split into chunks and send to workers
-                sent_chunks = 0
+                futures = []
                 for df2 in split_data_frame(df, partition_size):
-                    queue_in.put([sent_chunks, df2])
-                    sent_chunks += 1
+                    futures.append(executor.submit(self.exec_sub_steps, df2))
 
-                batch_results = []
-                for i in range(sent_chunks):
-                    res = queue_out.get()
-                    if 'error' in res:
+                for future in futures:
+                    try:
+                        results.append(future.result())
+                    except Exception as e:
                         if on_error == 'skip':
-                            logger.error(res['error'])
+                            logger.error(e)
                         else:
-                            raise RuntimeError(res['error'])
-
-                    if res['data']:
-                        batch_results.append(res)
-
-                # sort results
-                batch_results.sort(key=lambda x: x['num'])
-
-                results.append(self.concat_results(
-                    [item['data'] for item in batch_results]
-                ))
+                            executor.shutdown()
+                            raise e
 
                 # TODO
                 #  1. get next batch without updating track_value:
                 #    it allows to keep queue_in filled with data between fetching batches
                 run_query.set_progress(df, max_track_value)
-        finally:
-            self.close_workers(workers)
 
         return self.concat_results(results)
-
-    def close_workers(self, workers: List[threading.Thread]):
-        """
-        Sent signal to workers to stop
-        """
-
-        self.stop_event.set()
-        for worker in workers:
-            if worker.is_alive():
-                worker.join()
-
-    def _worker(self, context: Context, queue_in: queue.Queue, queue_out: queue.Queue, stop_event: threading.Event):
-        """
-        Worker function. Execute incoming tasks unless stop_event is set
-        """
-        ctx.load(context)
-        while True:
-            if stop_event.is_set():
-                break
-
-            try:
-                chunk_num, df = queue_in.get(timeout=1)
-                if df is None:
-                    continue
-
-                sub_data = self.exec_sub_steps(df)
-
-                queue_out.put({'data': sub_data, 'num': chunk_num})
-            except queue.Empty:
-                continue
-
-            except Exception as e:
-                queue_out.put({'error': str(e)})
-                stop_event.set()

mindsdb/integrations/handlers/chromadb_handler/chromadb_handler.py

@@ -244,6 +244,7 @@ class ChromaDBHandler(VectorStoreHandler):
         offset: int = None,
         limit: int = None,
     ) -> pd.DataFrame:
+
         collection = self._client.get_collection(table_name)
         filters = self._translate_metadata_condition(conditions)
 
@@ -313,7 +314,7 @@ class ChromaDBHandler(VectorStoreHandler):
             TableField.ID.value: ids,
             TableField.CONTENT.value: documents,
             TableField.METADATA.value: metadatas,
-            TableField.EMBEDDINGS.value: embeddings,
+            TableField.EMBEDDINGS.value: list(embeddings),
         }
 
         if columns is not None:

mindsdb/integrations/libs/vectordatabase_handler.py

@@ -278,8 +278,16 @@ class VectorStoreHandler(BaseHandler):
         return self.do_upsert(table_name, df)
 
     def do_upsert(self, table_name, df):
-        # if handler supports it, call upsert method
+        """Upsert data into table, handling document updates and deletions.
 
+        Args:
+            table_name (str): Name of the table
+            df (pd.DataFrame): DataFrame containing the data to upsert
+
+        The function handles three cases:
+        1. New documents: Insert them
+        2. Updated documents: Delete old chunks and insert new ones
+        """
         id_col = TableField.ID.value
         content_col = TableField.CONTENT.value
 

mindsdb/integrations/utilities/rag/rerankers/reranker_compressor.py

@@ -18,7 +18,7 @@ log = logging.getLogger(__name__)
 
 
 class LLMReranker(BaseDocumentCompressor):
-    filtering_threshold: float = 0.5  # Default threshold for filtering
+    filtering_threshold: float = 0.0  # Default threshold for filtering
     model: str = DEFAULT_RERANKING_MODEL  # Model to use for reranking
     temperature: float = 0.0  # Temperature for the model
     openai_api_key: Optional[str] = None

mindsdb/interfaces/database/projects.py

@@ -69,6 +69,12 @@ class Project:
         self.id = record.id
 
     def delete(self):
+        if self.record.metadata_ and self.record.metadata_.get('is_default', False):
+            raise Exception(
+                f"Project '{self.name}' can not be deleted, because it is default project."
+                "The default project can be changed in the config file or by setting the environment variable MINDSDB_DEFAULT_PROJECT."
+            )
+
         tables = self.get_tables()
         tables = [key for key, val in tables.items() if val['type'] != 'table']
         if len(tables) > 0:
@@ -466,7 +472,7 @@ class ProjectController:
 
         if new_metadata is not None:
             project.metadata = new_metadata
-            project.record.metadata = new_metadata
+            project.record.metadata_ = new_metadata
             flag_modified(project.record, 'metadata_')
 
         db.session.commit()

mindsdb/interfaces/knowledge_base/controller.py

@@ -3,7 +3,6 @@ import copy
 from typing import Dict, List, Optional
 
 import pandas as pd
-import hashlib
 import numpy as np
 
 from mindsdb_sql_parser.ast import (
@@ -155,19 +154,19 @@ class KnowledgeBaseTable:
         # extract values from conditions and prepare for vectordb
         conditions = []
         query_text = None
-        reranking_threshold = None
+        relevance_threshold = None
         query_conditions = db_handler.extract_conditions(query.where)
         if query_conditions is not None:
             for item in query_conditions:
-                if item.column == "reranking_threshold" and item.op.value == "=":
+                if item.column == "relevance_threshold" and item.op.value == "=":
                     try:
-                        reranking_threshold = float(item.value)
+                        relevance_threshold = float(item.value)
                         # Validate range: must be between 0 and 1
-                        if not (0 <= reranking_threshold <= 1):
-                            raise ValueError(f"reranking_threshold must be between 0 and 1, got: {reranking_threshold}")
-                        logger.debug(f"Found reranking_threshold in query: {reranking_threshold}")
+                        if not (0 <= relevance_threshold <= 1):
+                            raise ValueError(f"relevance_threshold must be between 0 and 1, got: {relevance_threshold}")
+                        logger.debug(f"Found relevance_threshold in query: {relevance_threshold}")
                     except (ValueError, TypeError) as e:
-                        error_msg = f"Invalid reranking_threshold value: {item.value}. {str(e)}"
+                        error_msg = f"Invalid relevance_threshold value: {item.value}. {str(e)}"
                         logger.error(error_msg)
                         raise ValueError(error_msg)
                 elif item.column == TableField.CONTENT.value:
@@ -185,6 +184,16 @@ class KnowledgeBaseTable:
         logger.debug(f"Extracted query text: {query_text}")
 
         self.addapt_conditions_columns(conditions)
+
+        # Set default limit if query is present
+        if query_text is not None:
+            limit = query.limit.value if query.limit is not None else None
+            if limit is None:
+                limit = 10
+            elif limit > 100:
+                limit = 100
+            query.limit = Constant(limit)
+
         df = db_handler.dispatch_select(query, conditions)
         df = self.addapt_result_columns(df)
 
@@ -192,14 +201,14 @@ class KnowledgeBaseTable:
         logger.debug(f"Columns in response: {df.columns.tolist()}")
         # Check if we have a rerank_model configured in KB params
 
-        df = self.add_relevance(df, query_text, reranking_threshold)
+        df = self.add_relevance(df, query_text, relevance_threshold)
 
         # filter by targets
         if requested_kb_columns is not None:
             df = df[requested_kb_columns]
         return df
 
-    def add_relevance(self, df, query_text, reranking_threshold=None):
+    def add_relevance(self, df, query_text, relevance_threshold=None):
         relevance_column = TableField.RELEVANCE.value
 
         reranking_model_params = self._kb.params.get("reranking_model")
@@ -208,9 +217,9 @@ class KnowledgeBaseTable:
             try:
                 logger.info(f"Using knowledge reranking model from params: {reranking_model_params}")
                 # Apply custom filtering threshold if provided
-                if reranking_threshold is not None:
-                    reranking_model_params["filtering_threshold"] = reranking_threshold
-                    logger.info(f"Using custom filtering threshold: {reranking_threshold}")
+                if relevance_threshold is not None:
+                    reranking_model_params["filtering_threshold"] = relevance_threshold
+                    logger.info(f"Using custom filtering threshold: {relevance_threshold}")
 
                 reranker = get_reranking_model_from_params(reranking_model_params)
                 # Get documents to rerank
@@ -236,8 +245,8 @@ class KnowledgeBaseTable:
             # Calculate relevance from distance
             logger.info("Calculating relevance from vector distance")
             df[relevance_column] = 1 / (1 + df['distance'])
-            if reranking_threshold is not None:
-                df = df[df[relevance_column] > reranking_threshold]
+            if relevance_threshold is not None:
+                df = df[df[relevance_column] > relevance_threshold]
 
         else:
             df[relevance_column] = None
@@ -333,12 +342,21 @@ class KnowledgeBaseTable:
 
         emb_col = TableField.EMBEDDINGS.value
         cont_col = TableField.CONTENT.value
+
+        db_handler = self.get_vector_db()
+        conditions = db_handler.extract_conditions(query.where)
+        doc_id = None
+        for condition in conditions:
+            if condition.column == 'chunk_id' and condition.op == FilterOperator.EQUAL:
+                doc_id = condition.value
+
         if cont_col in query.update_columns:
             content = query.update_columns[cont_col]
 
             # Apply preprocessing to content if configured
             if self.document_preprocessor:
                 doc = Document(
+                    id=doc_id,
                     content=content.value,
                     metadata={}  # Empty metadata for content-only updates
                 )
@@ -354,8 +372,6 @@ class KnowledgeBaseTable:
         query.table = Identifier(parts=[self._kb.vector_database_table])
 
         # send to vectordb
-        db_handler = self.get_vector_db()
-        conditions = db_handler.extract_conditions(query.where)
         self.addapt_conditions_columns(conditions)
         db_handler.dispatch_update(query, conditions)
 
@@ -409,7 +425,11 @@ class KnowledgeBaseTable:
         db_handler.delete(self._kb.vector_database_table)
 
     def insert(self, df: pd.DataFrame):
-        """Insert dataframe to KB table."""
+        """Insert dataframe to KB table.
+
+        Args:
+            df: DataFrame to insert
+        """
         if df.empty:
             return
 
@@ -754,22 +774,9 @@ class KnowledgeBaseTable:
         return {}
 
     def _generate_document_id(self, content: str, content_column: str, provided_id: str = None) -> str:
-        """
-        Generate a deterministic document ID from content and column name.
-        If provided_id exists, combines it with content_column.
-
-        Args:
-            content: The content string
-            content_column: Name of the content column
-            provided_id: Optional user-provided ID
-        Returns:
-            Deterministic document ID
-        """
-        if provided_id is not None:
-            return f"{provided_id}_{content_column}"
-
-        id_string = f"content={content}_column={content_column}"
-        return hashlib.sha256(id_string.encode()).hexdigest()
+        """Generate a deterministic document ID using the utility function."""
+        from mindsdb.interfaces.knowledge_base.utils import generate_document_id
+        return generate_document_id(content, content_column, provided_id)
 
     def _convert_metadata_value(self, value):
         """

mindsdb/interfaces/knowledge_base/preprocessing/document_preprocessor.py

@@ -1,7 +1,6 @@
 from typing import List, Dict, Optional, Any
 import pandas as pd
 from langchain_text_splitters import RecursiveCharacterTextSplitter
-import hashlib
 import asyncio
 
 
@@ -43,7 +42,11 @@ class DocumentPreprocessor:
         self.splitter = None  # Will be set by child classes
 
     def process_documents(self, documents: List[Document]) -> List[ProcessedChunk]:
-        """Base implementation - should be overridden by child classes"""
+        """Base implementation - should be overridden by child classes
+
+        Args:
+            documents: List of documents to process
+        """
         raise NotImplementedError("Subclasses must implement process_documents")
 
     def _split_document(self, doc: Document) -> List[Document]:
@@ -80,27 +83,22 @@ class DocumentPreprocessor:
             metadata=data.get("metadata", {}),
         )
 
-    def _generate_deterministic_id(
-        self, content: str, content_column: str = None, provided_id: str = None
-    ) -> str:
-        """Generate a deterministic ID based on content and column"""
-        if provided_id is not None:
-            return f"{provided_id}_{content_column}"
-
-        id_string = f"content={content}_column={content_column}"
-        return hashlib.sha256(id_string.encode()).hexdigest()
-
     def _generate_chunk_id(
         self,
         chunk_index: Optional[int] = None,
+        total_chunks: Optional[int] = None,
+        start_char: Optional[int] = None,
+        end_char: Optional[int] = None,
         provided_id: str = None,
     ) -> str:
-        """Generate deterministic ID for a chunk"""
-        base_id = provided_id
-        chunk_id = (
-            f"{base_id}_chunk_{chunk_index}" if chunk_index is not None else base_id
-        )
-        logger.debug(f"Generated chunk ID: {chunk_id} for content hash: {base_id}")
+        """Generate human-readable deterministic ID for a chunk
+        Format: <doc_id>:<chunk_number>of<total_chunks>:<start_char>to<end_char>
+        """
+        if provided_id is None:
+            raise ValueError("Document ID must be provided for chunk ID generation")
+
+        chunk_id = f"{provided_id}:{chunk_index + 1}of{total_chunks}:{start_char}to{end_char}"
+        logger.debug(f"Generated chunk ID: {chunk_id}")
         return chunk_id
 
     def _prepare_chunk_metadata(
@@ -207,14 +205,10 @@ Please give a short succinct context to situate this chunk within the overall do
         processed_chunks = []
 
         for doc_index, doc in enumerate(documents):
-            # Get content_column from metadata if available
-            content_column = (
-                doc.metadata.get("content_column") if doc.metadata else None
-            )
 
-            # Ensure document has an ID
+            # Document ID must be provided by this point
             if doc.id is None:
-                doc.id = self._generate_deterministic_id(doc.content, content_column)
+                raise ValueError("Document ID must be provided before preprocessing")
 
             # Skip empty or whitespace-only content
             if not doc.content or not doc.content.strip():
@@ -298,68 +292,55 @@ class TextChunkingPreprocessor(DocumentPreprocessor):
         processed_chunks = []
 
         for doc in documents:
-            # Get content_column from metadata if available
-            content_column = (
-                doc.metadata.get("content_column") if doc.metadata else None
-            )
 
-            # Ensure document has an ID
+            # Document ID must be provided by this point
             if doc.id is None:
-                doc.id = self._generate_deterministic_id(doc.content, content_column)
+                raise ValueError("Document ID must be provided before preprocessing")
 
             # Skip empty or whitespace-only content
             if not doc.content or not doc.content.strip():
                 continue
 
             chunk_docs = self._split_document(doc)
+            total_chunks = len(chunk_docs)
 
-            # Single chunk case
-            if len(chunk_docs) == 1:
-                chunk_doc = chunk_docs[0]
+            # Track character positions
+            current_pos = 0
+            for i, chunk_doc in enumerate(chunk_docs):
                 if not chunk_doc.content or not chunk_doc.content.strip():
                     continue
 
+                # Calculate chunk positions
+                start_char = current_pos
+                end_char = start_char + len(chunk_doc.content)
+                current_pos = end_char + 1  # +1 for separator
+
                 # Initialize metadata
                 metadata = {}
                 if doc.metadata:
                     metadata.update(doc.metadata)
 
-                # Pass through doc.id and content_column
-                id = self._generate_chunk_id(
-                    chunk_index=0, provided_id=doc.id
+                # Add position metadata
+                metadata["start_char"] = start_char
+                metadata["end_char"] = end_char
+
+                # Generate chunk ID with total chunks
+                chunk_id = self._generate_chunk_id(
+                    chunk_index=i,
+                    total_chunks=total_chunks,
+                    start_char=start_char,
+                    end_char=end_char,
+                    provided_id=doc.id
                 )
+
                 processed_chunks.append(
                     ProcessedChunk(
-                        id=id,
+                        id=chunk_id,
                         content=chunk_doc.content,
                         embeddings=doc.embeddings,
-                        metadata=self._prepare_chunk_metadata(doc.id, None, metadata),
+                        metadata=self._prepare_chunk_metadata(doc.id, i, metadata),
                     )
                 )
-            else:
-                # Multiple chunks case
-                for i, chunk_doc in enumerate(chunk_docs):
-                    if not chunk_doc.content or not chunk_doc.content.strip():
-                        continue
-
-                    # Initialize metadata
-                    metadata = {}
-                    if doc.metadata:
-                        metadata.update(doc.metadata)
-
-                    # Pass through doc.id and content_column
-                    chunk_id = self._generate_chunk_id(
-                        chunk_index=i,
-                        provided_id=doc.id,
-                    )
-                    processed_chunks.append(
-                        ProcessedChunk(
-                            id=chunk_id,
-                            content=chunk_doc.content,
-                            embeddings=doc.embeddings,
-                            metadata=self._prepare_chunk_metadata(doc.id, i, metadata),
-                        )
-                    )
 
         return processed_chunks
 

mindsdb/interfaces/knowledge_base/utils.py

@@ -0,0 +1,28 @@
+"""Utilities for knowledge base operations."""
+import hashlib
+
+
+def generate_document_id(content: str, content_column: str, provided_id: str = None) -> str:
+    """
+    Generate a deterministic document ID from content and column name.
+    If provided_id exists, combines it with content_column.
+    For generated IDs, uses a short hash of just the content to ensure
+    same content gets same base ID across different columns.
+
+    Args:
+        content: The content string
+        content_column: Name of the content column
+        provided_id: Optional user-provided ID
+    Returns:
+        Deterministic document ID in format: <base_id>_<column>
+        where base_id is either the provided_id or a 16-char hash of content
+    """
+    if provided_id is not None:
+        base_id = provided_id
+    else:
+        # Generate a shorter 16-character hash based only on content
+        hash_obj = hashlib.md5(content.encode())
+        base_id = hash_obj.hexdigest()[:16]
+
+    # Append column name to maintain uniqueness across columns
+    return f"{base_id}_{content_column}"

mindsdb/utilities/auth.py

@@ -15,9 +15,11 @@ def get_aws_meta_data() -> dict:
         'ami-id': None,
         'instance-id': None
     }
+    aws_token = requests.put("http://169.254.169.254/latest/api/token", headers={'X-aws-ec2-metadata-token-ttl-seconds': '30'}).text
     for key in aws_meta_data.keys():
         resp = requests.get(
             f'http://169.254.169.254/latest/meta-data/{key}',
+            headers={'X-aws-ec2-metadata-token': aws_token},
             timeout=1
         )
         if resp.status_code != 200:
@@ -35,7 +37,9 @@ def register_oauth_client():
     aws_meta_data = get_aws_meta_data()
 
     current_aws_meta_data = config.get('aws_meta_data', {})
-    oauth_meta = config.get('auth', {}).get('oauth', {})
+    oauth_meta = config.get('auth', {}).get('oauth')
+    if oauth_meta is None:
+        return
 
     public_hostname = aws_meta_data['public-hostname']
     if (

mindsdb/utilities/cache.py

@@ -56,6 +56,7 @@ import os
 import time
 from abc import ABC
 from pathlib import Path
+import re
 import hashlib
 import typing as t
 
@@ -154,7 +155,9 @@ class FileCache(BaseCache):
                     pass
 
     def file_path(self, name):
-        return self.path / name
+        # Sanitize the key to avoid table (file) names with backticks and slashes.
+        sanitized_name = re.sub(r'[^\w\-.]', '_', name)
+        return self.path / sanitized_name
 
     def set_df(self, name, df):
         path = self.file_path(name)