MindsDB 25.8.3.0__py3-none-any.whl → 25.9.1.1__py3-none-any.whl

mindsdb/integrations/handlers/mongodb_handler/utils/mongodb_render.py

@@ -5,7 +5,7 @@ from bson.objectid import ObjectId
 from mindsdb_sql_parser.ast import Select, Update, Identifier, Star, Constant, Tuple, BinaryOperation, Latest, TypeCast
 from mindsdb_sql_parser.ast.base import ASTNode
 
-from mindsdb.api.mongo.utilities.mongodb_query import MongoQuery
+from mindsdb.integrations.handlers.mongodb_handler.utils.mongodb_query import MongoQuery
 
 
 class MongodbRender:
@@ -27,7 +27,7 @@ class MongodbRender:
             return self.select(node)
         elif isinstance(node, Update):
             return self.update(node)
-        raise NotImplementedError(f'Unknown statement: {node.__class__.__name__}')
+        raise NotImplementedError(f"Unknown statement: {node.__class__.__name__}")
 
     def update(self, node: Update) -> MongoQuery:
         """
@@ -43,17 +43,8 @@ class MongodbRender:
         mquery = MongoQuery(collection)
 
         filters = self.handle_where(node.where)
-        row = {
-            k: v.value
-            for k, v in node.update_columns.items()
-        }
-        mquery.add_step({
-            'method': 'update_many',
-            'args': [
-                filters,
-                {"$set": row}
-            ]
-        })
+        row = {k: v.value for k, v in node.update_columns.items()}
+        mquery.add_step({"method": "update_many", "args": [filters, {"$set": row}]})
         return mquery
 
     def select(self, node: Select):
@@ -67,7 +58,7 @@ class MongodbRender:
             MongoQuery: The converted MongoQuery instance.
         """
         if not isinstance(node.from_table, Identifier):
-            raise NotImplementedError(f'Not supported from {node.from_table}')
+            raise NotImplementedError(f"Not supported from {node.from_table}")
 
         collection = node.from_table.parts[-1]
 
@@ -77,10 +68,10 @@ class MongodbRender:
             filters = self.handle_where(node.where)
 
         group = {}
-        project = {'_id': 0}  # Hide _id field when it has not been explicitly requested.
+        project = {"_id": 0}  # Hide _id field when it has not been explicitly requested.
         if node.distinct:
             # Group by distinct fields.
-            group = {'_id': {}}
+            group = {"_id": {}}
 
         if node.targets is not None:
             for col in node.targets:
@@ -95,12 +86,12 @@ class MongodbRender:
                     else:
                         alias = col.alias.parts[-1]
 
-                    project[alias] = f'${name}'  # Project field.
+                    project[alias] = f"${name}"  # Project field.
 
                     # Group by distinct fields.
                     if node.distinct:
-                        group['_id'][name] = f'${name}'  # Group field.
-                        group[name] = {'$first': f'${name}'}  # Show field.
+                        group["_id"][name] = f"${name}"  # Group field.
+                        group[name] = {"$first": f"${name}"}  # Show field.
 
                 elif isinstance(col, Constant):
                     val = str(col.value)  # Convert to string becuase it is interpreted as an index.
@@ -112,19 +103,19 @@ class MongodbRender:
 
         if node.group_by is not None:
             # TODO
-            raise NotImplementedError(f'Group {node.group_by}')
+            raise NotImplementedError(f"Group {node.group_by}")
 
         sort = {}
         if node.order_by is not None:
             for col in node.order_by:
                 name = col.field.parts[-1]
-                direction = 1 if col.direction.upper() == 'ASC' else -1
+                direction = 1 if col.direction.upper() == "ASC" else -1
                 sort[name] = direction
 
         # Compose the MongoDB query.
         mquery = MongoQuery(collection)
 
-        method = 'aggregate'
+        method = "aggregate"
         arg = []
 
         # MongoDB related pipeline steps for the aggregate method.
@@ -150,10 +141,7 @@ class MongodbRender:
         if node.limit is not None:
             arg.append({"$limit": int(node.limit.value)})
 
-        mquery.add_step({
-            'method': method,
-            'args': [arg]
-        })
+        mquery.add_step({"method": method, "args": [arg]})
 
         return mquery
 
@@ -168,34 +156,34 @@ class MongodbRender:
             dict: The converted MongoDB query filters.
         """
         # TODO: UnaryOperation, function.
-        if not type(node) in [BinaryOperation]:
-            raise NotImplementedError(f'Not supported type {type(node)}')
+        if type(node) not in [BinaryOperation]:
+            raise NotImplementedError(f"Not supported type {type(node)}")
 
         op = node.op.lower()
         arg1, arg2 = node.args
 
-        if op in ('and', 'or'):
+        if op in ("and", "or"):
             query1 = self.handle_where(arg1)
             query2 = self.handle_where(arg2)
 
             ops = {
-                'and': '$and',
-                'or': '$or',
+                "and": "$and",
+                "or": "$or",
             }
             query = {ops[op]: [query1, query2]}
             return query
 
         ops_map = {
-            '>=': '$gte',
-            '>': '$gt',
-            '<': '$lt',
-            '<=': '$lte',
-            '<>': '$ne',
-            '!=': '$ne',
-            '=': '$eq',
-            '==': '$eq',
-            'is': '$eq',
-            'is not': '$ne',
+            ">=": "$gte",
+            ">": "$gt",
+            "<": "$lt",
+            "<=": "$lte",
+            "<>": "$ne",
+            "!=": "$ne",
+            "=": "$eq",
+            "==": "$eq",
+            "is": "$eq",
+            "is not": "$ne",
         }
 
         if isinstance(arg1, Identifier):
@@ -203,35 +191,29 @@ class MongodbRender:
             # Simple operation.
             if isinstance(arg2, Constant):
                 # Identifier and Constant.
-                val = ObjectId(arg2.value) if var_name == '_id' else arg2.value
-                if op in ('=', '=='):
+                val = ObjectId(arg2.value) if var_name == "_id" else arg2.value
+                if op in ("=", "=="):
                     pass
                 elif op in ops_map:
                     op2 = ops_map[op]
                     val = {op2: val}
                 else:
-                    raise NotImplementedError(f'Not supported operator {op}')
+                    raise NotImplementedError(f"Not supported operator {op}")
 
                 return {var_name: val}
 
             # IN condition.
             elif isinstance(arg2, Tuple):
                 # Should be IN, NOT IN.
-                ops = {
-                    'in': '$in',
-                    'not in': '$nin'
-                }
+                ops = {"in": "$in", "not in": "$nin"}
                 # Must be list of Constants.
-                values = [
-                    i.value
-                    for i in arg2.items
-                ]
+                values = [i.value for i in arg2.items]
 
                 if op in ops:
                     op2 = ops[op]
                     cond = {op2: values}
                 else:
-                    raise NotImplementedError(f'Not supported operator {op}')
+                    raise NotImplementedError(f"Not supported operator {op}")
 
                 return {var_name: cond}
 
@@ -242,13 +224,9 @@ class MongodbRender:
         if op in ops_map:
             op2 = ops_map[op]
         else:
-            raise NotImplementedError(f'Not supported operator {op}')
+            raise NotImplementedError(f"Not supported operator {op}")
 
-        return {
-            '$expr': {
-                op2: [val1, val2]
-            }
-        }
+        return {"$expr": {op2: [val1, val2]}}
 
     def where_element_convert(self, node: Union[Identifier, Latest, Constant, TypeCast]) -> Any:
         """
@@ -265,22 +243,18 @@ class MongodbRender:
             RuntimeError: If the date format is not supported.
         """
         if isinstance(node, Identifier):
-            return f'${node.parts[-1]}'
+            return f"${node.parts[-1]}"
         elif isinstance(node, Latest):
-            return 'LATEST'
+            return "LATEST"
         elif isinstance(node, Constant):
             return node.value
-        elif isinstance(node, TypeCast)\
-                and node.type_name.upper() in ('DATE', 'DATETIME'):
-            formats = [
-                "%Y-%m-%d",
-                "%Y-%m-%dT%H:%M:%S.%f"
-            ]
+        elif isinstance(node, TypeCast) and node.type_name.upper() in ("DATE", "DATETIME"):
+            formats = ["%Y-%m-%d", "%Y-%m-%dT%H:%M:%S.%f"]
             for format in formats:
                 try:
                     return dt.datetime.strptime(node.arg.value, format)
                 except ValueError:
                     pass
-            raise RuntimeError(f'Not supported date format. Supported: {formats}')
+            raise RuntimeError(f"Not supported date format. Supported: {formats}")
         else:
-            raise NotImplementedError(f'Unknown where element {node}')
+            raise NotImplementedError(f"Unknown where element {node}")

mindsdb/integrations/handlers/tpot_handler/requirements.txt

@@ -1,2 +1,2 @@
 tpot<=0.11.7
-type_infer==0.0.23
+type_infer==0.0.25

mindsdb/integrations/libs/base.py

@@ -58,7 +58,7 @@ class BaseHandler:
 
         Args:
             query (Any): query in native format (str for sql databases,
-                dict for mongo, etc)
+                etc)
 
         Returns:
             HandlerResponse

mindsdb/integrations/libs/llm/config.py

@@ -113,3 +113,18 @@ class WriterConfig(BaseLLMConfig):
     writer_api_key: Optional[str] = Field(default=None)
     writer_org_id: Optional[str] = Field(default=None)
     base_url: Optional[str] = Field(default=None)
+
+
+# https://api.python.langchain.com/en/latest/llms/langchain_aws.llms.bedrock.BedrockLLM.html#langchain_aws.llms.bedrock.BedrockLLM
+class BedrockConfig(BaseLLMConfig):
+    model_id: str
+    aws_access_key_id: Optional[str] = Field(default=None)
+    aws_secret_access_key: Optional[str] = Field(default=None)
+    aws_session_token: Optional[str] = Field(default=None)
+    region_name: Optional[str] = Field(default=None)
+    credentials_profile_name: Optional[str] = Field(default=None)
+    endpoint_url: Optional[str] = Field(default=None)
+    stop: Optional[List[str]] = Field(default=None)
+    temperature: Optional[float] = Field(default=0.7)
+    max_tokens: Optional[int] = Field(default=None)
+    model_kwargs: Optional[Dict[str, Any]] = Field(default=None)

mindsdb/integrations/libs/llm/utils.py

@@ -16,6 +16,7 @@ from mindsdb.integrations.libs.llm.config import (
     NvidiaNIMConfig,
     MindsdbConfig,
     WriterConfig,
+    BedrockConfig,
 )
 from mindsdb.utilities.config import config
 from langchain_text_splitters import Language, RecursiveCharacterTextSplitter
@@ -222,6 +223,20 @@ def get_llm_config(provider: str, args: Dict) -> BaseLLMConfig:
             writer_org_id=args.get("writer_org_id", None),
             base_url=args.get("base_url", None),
         )
+    if provider == "bedrock":
+        return BedrockConfig(
+            model_id=args.get("model_name"),
+            temperature=temperature,
+            max_tokens=args.get("max_tokens", None),
+            stop=args.get("stop", None),
+            base_url=args.get("endpoint_url", None),
+            aws_access_key_id=args.get("aws_access_key_id", None),
+            aws_secret_access_key=args.get("aws_secret_access_key", None),
+            aws_session_token=args.get("aws_session_token", None),
+            region_name=args.get("aws_region_name", None),
+            credentials_profile_name=args.get("credentials_profile_name", None),
+            model_kwargs=args.get("model_kwargs", None),
+        )
 
     raise ValueError(f"Provider {provider} is not supported.")
 

mindsdb/interfaces/agents/constants.py

@@ -208,6 +208,7 @@ DEFAULT_TEMPERATURE = 0.0
 USER_COLUMN = "question"
 DEFAULT_EMBEDDINGS_MODEL_PROVIDER = "openai"
 DEFAULT_EMBEDDINGS_MODEL_CLASS = OpenAIEmbeddings
+MAX_INSERT_BATCH_SIZE = 50_000
 DEFAULT_TIKTOKEN_MODEL_NAME = os.getenv("DEFAULT_TIKTOKEN_MODEL_NAME", "gpt-4")
 AGENT_CHUNK_POLLING_INTERVAL_SECONDS = os.getenv("AGENT_CHUNK_POLLING_INTERVAL_SECONDS", 1.0)
 DEFAULT_TEXT2SQL_DATABASE = "mindsdb"

mindsdb/interfaces/agents/langchain_agent.py

@@ -40,6 +40,7 @@ from .mindsdb_chat_model import ChatMindsdb
 from .callback_handlers import LogCallbackHandler, ContextCaptureCallback
 from .langfuse_callback_handler import LangfuseCallbackHandler, get_skills
 from .safe_output_parser import SafeOutputParser
+from .providers import get_bedrock_chat_model
 
 from mindsdb.interfaces.agents.constants import (
     OPEN_AI_CHAT_MODELS,
@@ -176,6 +177,9 @@ def create_chat_model(args: Dict):
         return ChatGoogleGenerativeAI(**model_kwargs)
     if args["provider"] == "writer":
         return ChatWriter(**model_kwargs)
+    if args["provider"] == "bedrock":
+        ChatBedrock = get_bedrock_chat_model()
+        return ChatBedrock(**model_kwargs)
     if args["provider"] == "mindsdb":
         return ChatMindsdb(**model_kwargs)
     raise ValueError(f"Unknown provider: {args['provider']}")

mindsdb/interfaces/agents/providers.py

@@ -0,0 +1,20 @@
+import importlib.util
+
+
+def get_bedrock_chat_model():
+    try:
+        from langchain_aws.chat_models import ChatBedrock
+    except ModuleNotFoundError:
+        raise RuntimeError("bedrock connector is not installed. Please install it with `pip install langchain-aws`")
+
+    if not importlib.util.find_spec("transformers"):
+        raise RuntimeError(
+            "`transformers` module is required for bedrock to count tokens. Please install it with `pip install transformers`"
+        )
+
+    class ChatBedrockPatched(ChatBedrock):
+        def _prepare_input_and_invoke(self, *args, **kwargs):
+            kwargs.pop("stop_sequences", None)
+            return super()._prepare_input_and_invoke(*args, **kwargs)
+
+    return ChatBedrockPatched

mindsdb/interfaces/knowledge_base/controller.py

@@ -29,7 +29,7 @@ from mindsdb.integrations.handlers.langchain_embedding_handler.langchain_embeddi
     construct_model_from_args,
 )
 
-from mindsdb.interfaces.agents.constants import DEFAULT_EMBEDDINGS_MODEL_CLASS
+from mindsdb.interfaces.agents.constants import DEFAULT_EMBEDDINGS_MODEL_CLASS, MAX_INSERT_BATCH_SIZE
 from mindsdb.interfaces.agents.langchain_agent import create_chat_model, get_llm_provider
 from mindsdb.interfaces.database.projects import ProjectController
 from mindsdb.interfaces.variables.variables_controller import variables_controller
@@ -245,22 +245,34 @@ class KnowledgeBaseTable:
         keyword_search_cols_and_values = []
         query_text = None
         relevance_threshold = None
+        relevance_threshold_allowed_operators = [
+            FilterOperator.GREATER_THAN_OR_EQUAL.value,
+            FilterOperator.GREATER_THAN.value,
+        ]
+        gt_filtering = False
         hybrid_search_enabled_flag = False
         query_conditions = db_handler.extract_conditions(query.where)
         hybrid_search_alpha = None  # Default to None, meaning no alpha weighted blending
         if query_conditions is not None:
             for item in query_conditions:
-                if item.column == "relevance" and item.op.value == FilterOperator.GREATER_THAN_OR_EQUAL.value:
+                if (item.column == "relevance") and (item.op.value in relevance_threshold_allowed_operators):
                     try:
                         relevance_threshold = float(item.value)
                         # Validate range: must be between 0 and 1
                         if not (0 <= relevance_threshold <= 1):
                             raise ValueError(f"relevance_threshold must be between 0 and 1, got: {relevance_threshold}")
+                        if item.op.value == FilterOperator.GREATER_THAN.value:
+                            gt_filtering = True
                         logger.debug(f"Found relevance_threshold in query: {relevance_threshold}")
                     except (ValueError, TypeError) as e:
                         error_msg = f"Invalid relevance_threshold value: {item.value}. {str(e)}"
                         logger.error(error_msg)
                         raise ValueError(error_msg)
+                elif (item.column == "relevance") and (item.op.value not in relevance_threshold_allowed_operators):
+                    raise ValueError(
+                        f"Invalid operator for relevance: {item.op.value}. Only the following operators are allowed: "
+                        f"{','.join(relevance_threshold_allowed_operators)}."
+                    )
                 elif item.column == "reranking":
                     if item.value is False or (isinstance(item.value, str) and item.value.lower() == "false"):
                         disable_reranking = True
@@ -279,10 +291,6 @@ class KnowledgeBaseTable:
                     if not (0 <= item.value <= 1):
                         raise ValueError(f"Invalid hybrid_search_alpha value: {item.value}. Must be between 0 and 1.")
                     hybrid_search_alpha = item.value
-                elif item.column == "relevance" and item.op.value != FilterOperator.GREATER_THAN_OR_EQUAL.value:
-                    raise ValueError(
-                        f"Invalid operator for relevance: {item.op.value}. Only GREATER_THAN_OR_EQUAL is allowed."
-                    )
                 elif item.column == TableField.CONTENT.value:
                     query_text = item.value
 
@@ -368,6 +376,11 @@ class KnowledgeBaseTable:
         # Check if we have a rerank_model configured in KB params
         df = self.add_relevance(df, query_text, relevance_threshold, disable_reranking)
 
+        # if relevance filtering method is strictly GREATER THAN we filter the df
+        if gt_filtering:
+            relevance_scores = TableField.RELEVANCE.value
+            df = df[relevance_scores > relevance_threshold]
+
         return df
 
     def _get_allowed_metadata_columns(self) -> List[str] | None:
@@ -410,7 +423,7 @@ class KnowledgeBaseTable:
 
             # Filter by threshold
             scores_array = np.array(scores)
-            df = df[scores_array > reranker.filtering_threshold]
+            df = df[scores_array >= reranker.filtering_threshold]
             logger.debug(f"Applied reranking with params: {reranking_model_params}")
 
         elif "distance" in df.columns:
@@ -493,6 +506,8 @@ class KnowledgeBaseTable:
         """Process and insert raw data rows"""
         if not rows:
             return
+        if len(rows) > MAX_INSERT_BATCH_SIZE:
+            raise ValueError("Input data is too large, please load data in batches")
 
         df = pd.DataFrame(rows)
 
@@ -1078,6 +1093,7 @@ class KnowledgeBaseController:
             raise EntityExistsError("Knowledge base already exists", name)
 
         embedding_params = get_model_params(params.get("embedding_model", {}), "default_embedding_model")
+        params["embedding_model"] = embedding_params
 
         # if model_name is None:  # Legacy
         model_name = self._create_embedding_model(
@@ -1104,6 +1120,7 @@ class KnowledgeBaseController:
             params["reranking_model"] = {}
 
         reranking_model_params = get_model_params(reranking_model_params, "default_reranking_model")
+        params["reranking_model"] = reranking_model_params
         if reranking_model_params:
             # Get reranking model from params.
             # This is called here to check validaity of the parameters.
@@ -1228,6 +1245,7 @@ class KnowledgeBaseController:
                 raise RuntimeError(f"Problem with embedding model config: {e}")
             return
 
+        params = copy.deepcopy(params)
         if "provider" in params:
             engine = params.pop("provider").lower()