MindsDB 25.8.3.0__py3-none-any.whl → 25.9.1.0__py3-none-any.whl

mindsdb/{api/mongo/utilities → integrations/handlers/mongodb_handler/utils}/mongodb_ast.py

@@ -6,9 +6,8 @@ from mindsdb_sql_parser.ast import OrderBy, Identifier, Star, Select, Constant,
 
 
 class MongoToAst:
-
     """
-      Converts query mongo to AST format
+    Converts query mongo to AST format
     """
 
     def from_mongoqeury(self, query):
@@ -19,24 +18,22 @@ class MongoToAst:
         filter, projection = None, None
         sort, limit, skip = None, None, None
         for step in query.pipeline:
-            if step['method'] == 'find':
-                filter = step['args'][0]
+            if step["method"] == "find":
+                filter = step["args"][0]
                 if len(step) > 1:
-                    projection = step['args'][1]
-            elif step['method'] == 'sort':
-                sort = step['args'][0]
-            elif step['method'] == 'limit':
-                limit = step['args'][0]
-            elif step['method'] == 'skip':
-                skip = step['args'][0]
-
-        return self.find(collection, filter=filter,
-                         sort=sort, projection=projection,
-                         limit=limit, skip=skip)
-
-    def find(self, collection: t.Union[list, str],
-             filter=None, sort=None, projection=None,
-             limit=None, skip=None, **kwargs):
+                    projection = step["args"][1]
+            elif step["method"] == "sort":
+                sort = step["args"][0]
+            elif step["method"] == "limit":
+                limit = step["args"][0]
+            elif step["method"] == "skip":
+                skip = step["args"][0]
+
+        return self.find(collection, filter=filter, sort=sort, projection=projection, limit=limit, skip=skip)
+
+    def find(
+        self, collection: t.Union[list, str], filter=None, sort=None, projection=None, limit=None, skip=None, **kwargs
+    ):
         # https://www.mongodb.com/docs/v4.2/reference/method/db.collection.find/
 
         order_by = None
@@ -44,12 +41,7 @@ class MongoToAst:
             # sort is dict
             order_by = []
             for col, direction in sort.items():
-                order_by.append(
-                    OrderBy(
-                        field=Identifier(parts=[col]),
-                        direction='DESC' if direction == -1 else 'ASC'
-                    )
-                )
+                order_by.append(OrderBy(field=Identifier(parts=[col]), direction="DESC" if direction == -1 else "ASC"))
 
         if projection is not None:
             targets = []
@@ -59,9 +51,7 @@ class MongoToAst:
                     alias = Identifier(parts=[alias])
                 else:
                     alias = None
-                targets.append(
-                    Identifier(path_str=col, alias=alias)
-                )
+                targets.append(Identifier(path_str=col, alias=alias))
         else:
             targets = [Star()]
 
@@ -92,13 +82,13 @@ class MongoToAst:
 
     def convert_filter(self, filter):
         cond_ops = {
-            '$and': 'and',
-            '$or': 'or',
+            "$and": "and",
+            "$or": "or",
         }
 
         ast_filter = None
         for k, v in filter.items():
-            if k in ('$or', '$and'):
+            if k in ("$or", "$and"):
                 # suppose it is one key in dict
 
                 op = cond_ops[k]
@@ -116,7 +106,7 @@ class MongoToAst:
                     arg1 = BinaryOperation(op=op, args=[arg1, node])
 
                 return arg1
-            if k in ('$where', '$expr'):
+            if k in ("$where", "$expr"):
                 # try to parse simple expression like 'this.saledate > this.latest'
                 return MongoWhereParser(v).to_ast()
 
@@ -129,25 +119,12 @@ class MongoToAst:
             if ast_filter is None:
                 ast_filter = ast_com
             else:
-                ast_filter = BinaryOperation(op='and', args=[
-                    ast_filter,
-                    ast_com
-                ])
+                ast_filter = BinaryOperation(op="and", args=[ast_filter, ast_com])
         return ast_filter
 
     def handle_filter(self, value):
-        ops = {
-            '$ge': '>=',
-            '$gt': '>',
-            '$lt': '<',
-            '$le': '<=',
-            '$ne': '!=',
-            '$eq': '='
-        }
-        in_ops = {
-            '$in': 'in',
-            '$nin': 'not in'
-        }
+        ops = {"$ge": ">=", "$gt": ">", "$lt": "<", "$le": "<=", "$ne": "!=", "$eq": "="}
+        in_ops = {"$in": "in", "$nin": "not in"}
 
         if isinstance(value, dict):
             key, value = list(value.items())[0]
@@ -158,18 +135,18 @@ class MongoToAst:
             if key in in_ops:
                 op = in_ops[key]
                 if not isinstance(value, list):
-                    raise NotImplementedError(f'Unknown type {key}, {value}')
+                    raise NotImplementedError(f"Unknown type {key}, {value}")
                 value = Tuple(value)
 
                 return op, value
 
-            raise NotImplementedError(f'Unknown type {key}')
+            raise NotImplementedError(f"Unknown type {key}")
 
         elif isinstance(value, list):
-            raise NotImplementedError(f'Unknown filter {value}')
+            raise NotImplementedError(f"Unknown filter {value}")
         else:
             # is simple type
-            op = '='
+            op = "="
             value = value
             return op, value
 
@@ -181,13 +158,12 @@ class MongoWhereParser:
     def to_ast(self):
         # parse as python string
         # replace '=' with '=='
-        query = re.sub(r'([^=><])=([^=])', r'\1==\2', self.query)
+        query = re.sub(r"([^=><])=([^=])", r"\1==\2", self.query)
 
-        tree = py_ast.parse(query, mode='eval')
+        tree = py_ast.parse(query, mode="eval")
         return self.process(tree.body)
 
     def process(self, node):
-
         if isinstance(node, py_ast.BoolOp):
             # is AND or OR
             op = node.op.__class__.__name__
@@ -202,7 +178,7 @@ class MongoWhereParser:
         if isinstance(node, py_ast.Compare):
             # it is
             if len(node.ops) != 1:
-                raise NotImplementedError(f'Multiple ops {node.ops}')
+                raise NotImplementedError(f"Multiple ops {node.ops}")
             op = self.compare_op(node.ops[0])
             arg1 = self.process(node.left)
             arg2 = self.process(node.comparators[0])
@@ -210,7 +186,7 @@ class MongoWhereParser:
 
         if isinstance(node, py_ast.Name):
             # is special operator: latest, ...
-            if node.id == 'latest':
+            if node.id == "latest":
                 return Latest()
 
         if isinstance(node, py_ast.Constant):
@@ -228,28 +204,27 @@ class MongoWhereParser:
 
         if isinstance(node, py_ast.Attribute):
             # is 'this.field' - is attribute
-            if node.value.id != 'this':
-                raise NotImplementedError(f'Unknown variable {node.value.id}')
+            if node.value.id != "this":
+                raise NotImplementedError(f"Unknown variable {node.value.id}")
             return Identifier(parts=[node.attr])
 
-        raise NotImplementedError(f'Unknown node {node}')
+        raise NotImplementedError(f"Unknown node {node}")
 
     def compare_op(self, op):
-
         opname = op.__class__.__name__
 
         # TODO: in, not
 
         ops = {
-            'Eq': '=',
-            'NotEq': '!=',
-            'Gt': '>',
-            'Lt': '<',
-            'GtE': '>=',
-            'LtE': '<=',
+            "Eq": "=",
+            "NotEq": "!=",
+            "Gt": ">",
+            "Lt": "<",
+            "GtE": ">=",
+            "LtE": "<=",
         }
         if opname not in ops:
-            raise NotImplementedError(f'Unknown $where op: {opname}')
+            raise NotImplementedError(f"Unknown $where op: {opname}")
         return ops[opname]
 
     @staticmethod

mindsdb/{api/mongo/utilities → integrations/handlers/mongodb_handler/utils}/mongodb_parser.py

@@ -1,4 +1,3 @@
-
 import ast as py_ast
 
 import dateutil.parser
@@ -8,30 +7,27 @@ from .mongodb_query import MongoQuery
 
 
 class MongodbParser:
-    '''
-        Converts string into MongoQuery
-    '''
+    """
+    Converts string into MongoQuery
+    """
 
     def from_string(self, call_str):
-        tree = py_ast.parse(call_str.strip(), mode='eval')
+        tree = py_ast.parse(call_str.strip(), mode="eval")
         calls = self.process(tree.body)
         # first call contents collection
-        method1 = calls[0]['method']
+        method1 = calls[0]["method"]
         if len(method1) < 2:
-            raise IndexError('Collection not found')
+            raise IndexError("Collection not found")
         collection = method1[-2]
 
         mquery = MongoQuery(collection)
 
         # keep only last name
-        calls[0]['method'] = [method1[-1]]
+        calls[0]["method"] = [method1[-1]]
 
         # convert method names: get first item of list
         for c in calls:
-            mquery.add_step({
-                'method': c['method'][0],
-                'args': c['args']
-            })
+            mquery.add_step({"method": c["method"][0], "args": c["args"]})
 
         return mquery
 
@@ -49,20 +45,17 @@ class MongodbParser:
                 func = node.func.id
 
                 # special functions:
-                if func == 'ISODate':
+                if func == "ISODate":
                     return dateutil.parser.isoparse(args[0])
-                if func == 'ObjectId':
+                if func == "ObjectId":
                     return ObjectId(args[0])
             elif isinstance(node.func, py_ast.Attribute):
                 # it can be an attribute or pipeline
                 previous_call, func = self.process_func_name(node.func)
             else:
-                raise NotImplementedError(f'Unknown function type: {node.func}')
+                raise NotImplementedError(f"Unknown function type: {node.func}")
 
-            call = [{
-                'method': func,
-                'args': args
-            }]
+            call = [{"method": func, "args": args}]
             if previous_call is not None:
                 call = previous_call + call
 
@@ -75,7 +68,6 @@ class MongodbParser:
             return elements
 
         if isinstance(node, py_ast.Dict):
-
             keys = []
             for node2 in node.keys:
                 if isinstance(node2, py_ast.Constant):
@@ -85,7 +77,7 @@ class MongodbParser:
                 elif isinstance(node2, py_ast.Name):
                     value = node2.id
                 else:
-                    raise NotImplementedError(f'Unknown dict key {node2}')
+                    raise NotImplementedError(f"Unknown dict key {node2}")
 
                 keys.append(value)
 
@@ -98,11 +90,11 @@ class MongodbParser:
         if isinstance(node, py_ast.Name):
             # special attributes
             name = node.id
-            if name == 'true':
+            if name == "true":
                 return True
-            elif name == 'false':
+            elif name == "false":
                 return False
-            elif name == 'null':
+            elif name == "null":
                 return None
 
         if isinstance(node, py_ast.Constant):
@@ -122,7 +114,7 @@ class MongodbParser:
                 value = self.process(node.operand)
                 return -value
 
-        raise NotImplementedError(f'Unknown node {node}')
+        raise NotImplementedError(f"Unknown node {node}")
 
     def process_func_name(self, node):
         previous_call = None

mindsdb/{api/mongo/utilities → integrations/handlers/mongodb_handler/utils}/mongodb_query.py

@@ -4,17 +4,15 @@ from bson import ObjectId
 
 
 class MongoJSONEncoder(json.JSONEncoder):
-
     def default(self, obj):
         if isinstance(obj, dt.datetime):
-            return f'ISODate({obj.isoformat()})'
+            return f"ISODate({obj.isoformat()})"
         if isinstance(obj, ObjectId):
-            return f'ObjectId({str(obj)})'
+            return f"ObjectId({str(obj)})"
         return super(MongoJSONEncoder, self).default(obj)
 
 
 class MongoQuery:
-
     def __init__(self, collection, pipline=None):
         self.collection = collection
         self.pipeline = []
@@ -30,10 +28,8 @@ class MongoQuery:
         #     'args': [{c: 3}]
         # }
 
-        if 'method' not in step \
-                or 'args' not in step \
-                or not isinstance(step['args'], list):
-            raise AttributeError(f'Wrong step {step}')
+        if "method" not in step or "args" not in step or not isinstance(step["args"], list):
+            raise AttributeError(f"Wrong step {step}")
 
         self.pipeline.append(step)
 
@@ -43,10 +39,8 @@ class MongoQuery:
     def __getattr__(self, item):
         # return callback to save step of pipeline
         def fnc(*args):
-            self.pipeline.append({
-                'method': item,
-                'args': args
-            })
+            self.pipeline.append({"method": item, "args": args})
+
         return fnc
 
     def __str__(self):
@@ -72,13 +66,13 @@ class MongoQuery:
          "db_test.fish.find({a:1}, {b:2}).sort({c:3})"
         """
 
-        call_str = f'db.{self.collection}'
+        call_str = f"db.{self.collection}"
         for step in self.pipeline:
             args_str = []
-            for arg in step['args']:
+            for arg in step["args"]:
                 args_str.append(MongoJSONEncoder().encode(arg))
-            call_str += f'.{step["method"]}({",".join(args_str)})'
+            call_str += f".{step['method']}({','.join(args_str)})"
         return call_str
 
     def __repr__(self):
-        return f'MongoQuery({self.collection}, {str(self.pipeline)})'
+        return f"MongoQuery({self.collection}, {str(self.pipeline)})"

mindsdb/integrations/handlers/mongodb_handler/utils/mongodb_render.py

@@ -5,7 +5,7 @@ from bson.objectid import ObjectId
 from mindsdb_sql_parser.ast import Select, Update, Identifier, Star, Constant, Tuple, BinaryOperation, Latest, TypeCast
 from mindsdb_sql_parser.ast.base import ASTNode
 
-from mindsdb.api.mongo.utilities.mongodb_query import MongoQuery
+from mindsdb.integrations.handlers.mongodb_handler.utils.mongodb_query import MongoQuery
 
 
 class MongodbRender:
@@ -27,7 +27,7 @@ class MongodbRender:
             return self.select(node)
         elif isinstance(node, Update):
             return self.update(node)
-        raise NotImplementedError(f'Unknown statement: {node.__class__.__name__}')
+        raise NotImplementedError(f"Unknown statement: {node.__class__.__name__}")
 
     def update(self, node: Update) -> MongoQuery:
         """
@@ -43,17 +43,8 @@ class MongodbRender:
         mquery = MongoQuery(collection)
 
         filters = self.handle_where(node.where)
-        row = {
-            k: v.value
-            for k, v in node.update_columns.items()
-        }
-        mquery.add_step({
-            'method': 'update_many',
-            'args': [
-                filters,
-                {"$set": row}
-            ]
-        })
+        row = {k: v.value for k, v in node.update_columns.items()}
+        mquery.add_step({"method": "update_many", "args": [filters, {"$set": row}]})
         return mquery
 
     def select(self, node: Select):
@@ -67,7 +58,7 @@ class MongodbRender:
             MongoQuery: The converted MongoQuery instance.
         """
         if not isinstance(node.from_table, Identifier):
-            raise NotImplementedError(f'Not supported from {node.from_table}')
+            raise NotImplementedError(f"Not supported from {node.from_table}")
 
         collection = node.from_table.parts[-1]
 
@@ -77,10 +68,10 @@ class MongodbRender:
             filters = self.handle_where(node.where)
 
         group = {}
-        project = {'_id': 0}  # Hide _id field when it has not been explicitly requested.
+        project = {"_id": 0}  # Hide _id field when it has not been explicitly requested.
         if node.distinct:
             # Group by distinct fields.
-            group = {'_id': {}}
+            group = {"_id": {}}
 
         if node.targets is not None:
             for col in node.targets:
@@ -95,12 +86,12 @@ class MongodbRender:
                     else:
                         alias = col.alias.parts[-1]
 
-                    project[alias] = f'${name}'  # Project field.
+                    project[alias] = f"${name}"  # Project field.
 
                     # Group by distinct fields.
                     if node.distinct:
-                        group['_id'][name] = f'${name}'  # Group field.
-                        group[name] = {'$first': f'${name}'}  # Show field.
+                        group["_id"][name] = f"${name}"  # Group field.
+                        group[name] = {"$first": f"${name}"}  # Show field.
 
                 elif isinstance(col, Constant):
                     val = str(col.value)  # Convert to string becuase it is interpreted as an index.
@@ -112,19 +103,19 @@ class MongodbRender:
 
         if node.group_by is not None:
             # TODO
-            raise NotImplementedError(f'Group {node.group_by}')
+            raise NotImplementedError(f"Group {node.group_by}")
 
         sort = {}
         if node.order_by is not None:
             for col in node.order_by:
                 name = col.field.parts[-1]
-                direction = 1 if col.direction.upper() == 'ASC' else -1
+                direction = 1 if col.direction.upper() == "ASC" else -1
                 sort[name] = direction
 
         # Compose the MongoDB query.
         mquery = MongoQuery(collection)
 
-        method = 'aggregate'
+        method = "aggregate"
         arg = []
 
         # MongoDB related pipeline steps for the aggregate method.
@@ -150,10 +141,7 @@ class MongodbRender:
         if node.limit is not None:
             arg.append({"$limit": int(node.limit.value)})
 
-        mquery.add_step({
-            'method': method,
-            'args': [arg]
-        })
+        mquery.add_step({"method": method, "args": [arg]})
 
         return mquery
 
@@ -168,34 +156,34 @@ class MongodbRender:
             dict: The converted MongoDB query filters.
         """
         # TODO: UnaryOperation, function.
-        if not type(node) in [BinaryOperation]:
-            raise NotImplementedError(f'Not supported type {type(node)}')
+        if type(node) not in [BinaryOperation]:
+            raise NotImplementedError(f"Not supported type {type(node)}")
 
         op = node.op.lower()
         arg1, arg2 = node.args
 
-        if op in ('and', 'or'):
+        if op in ("and", "or"):
             query1 = self.handle_where(arg1)
             query2 = self.handle_where(arg2)
 
             ops = {
-                'and': '$and',
-                'or': '$or',
+                "and": "$and",
+                "or": "$or",
             }
             query = {ops[op]: [query1, query2]}
             return query
 
         ops_map = {
-            '>=': '$gte',
-            '>': '$gt',
-            '<': '$lt',
-            '<=': '$lte',
-            '<>': '$ne',
-            '!=': '$ne',
-            '=': '$eq',
-            '==': '$eq',
-            'is': '$eq',
-            'is not': '$ne',
+            ">=": "$gte",
+            ">": "$gt",
+            "<": "$lt",
+            "<=": "$lte",
+            "<>": "$ne",
+            "!=": "$ne",
+            "=": "$eq",
+            "==": "$eq",
+            "is": "$eq",
+            "is not": "$ne",
         }
 
         if isinstance(arg1, Identifier):
@@ -203,35 +191,29 @@ class MongodbRender:
             # Simple operation.
             if isinstance(arg2, Constant):
                 # Identifier and Constant.
-                val = ObjectId(arg2.value) if var_name == '_id' else arg2.value
-                if op in ('=', '=='):
+                val = ObjectId(arg2.value) if var_name == "_id" else arg2.value
+                if op in ("=", "=="):
                     pass
                 elif op in ops_map:
                     op2 = ops_map[op]
                     val = {op2: val}
                 else:
-                    raise NotImplementedError(f'Not supported operator {op}')
+                    raise NotImplementedError(f"Not supported operator {op}")
 
                 return {var_name: val}
 
             # IN condition.
             elif isinstance(arg2, Tuple):
                 # Should be IN, NOT IN.
-                ops = {
-                    'in': '$in',
-                    'not in': '$nin'
-                }
+                ops = {"in": "$in", "not in": "$nin"}
                 # Must be list of Constants.
-                values = [
-                    i.value
-                    for i in arg2.items
-                ]
+                values = [i.value for i in arg2.items]
 
                 if op in ops:
                     op2 = ops[op]
                     cond = {op2: values}
                 else:
-                    raise NotImplementedError(f'Not supported operator {op}')
+                    raise NotImplementedError(f"Not supported operator {op}")
 
                 return {var_name: cond}
 
@@ -242,13 +224,9 @@ class MongodbRender:
         if op in ops_map:
             op2 = ops_map[op]
         else:
-            raise NotImplementedError(f'Not supported operator {op}')
+            raise NotImplementedError(f"Not supported operator {op}")
 
-        return {
-            '$expr': {
-                op2: [val1, val2]
-            }
-        }
+        return {"$expr": {op2: [val1, val2]}}
 
     def where_element_convert(self, node: Union[Identifier, Latest, Constant, TypeCast]) -> Any:
         """
@@ -265,22 +243,18 @@ class MongodbRender:
             RuntimeError: If the date format is not supported.
         """
         if isinstance(node, Identifier):
-            return f'${node.parts[-1]}'
+            return f"${node.parts[-1]}"
         elif isinstance(node, Latest):
-            return 'LATEST'
+            return "LATEST"
         elif isinstance(node, Constant):
             return node.value
-        elif isinstance(node, TypeCast)\
-                and node.type_name.upper() in ('DATE', 'DATETIME'):
-            formats = [
-                "%Y-%m-%d",
-                "%Y-%m-%dT%H:%M:%S.%f"
-            ]
+        elif isinstance(node, TypeCast) and node.type_name.upper() in ("DATE", "DATETIME"):
+            formats = ["%Y-%m-%d", "%Y-%m-%dT%H:%M:%S.%f"]
             for format in formats:
                 try:
                     return dt.datetime.strptime(node.arg.value, format)
                 except ValueError:
                     pass
-            raise RuntimeError(f'Not supported date format. Supported: {formats}')
+            raise RuntimeError(f"Not supported date format. Supported: {formats}")
         else:
-            raise NotImplementedError(f'Unknown where element {node}')
+            raise NotImplementedError(f"Unknown where element {node}")

mindsdb/integrations/libs/base.py

@@ -58,7 +58,7 @@ class BaseHandler:
 
         Args:
             query (Any): query in native format (str for sql databases,
-                dict for mongo, etc)
+                etc)
 
         Returns:
             HandlerResponse

mindsdb/interfaces/agents/constants.py

@@ -208,6 +208,7 @@ DEFAULT_TEMPERATURE = 0.0
 USER_COLUMN = "question"
 DEFAULT_EMBEDDINGS_MODEL_PROVIDER = "openai"
 DEFAULT_EMBEDDINGS_MODEL_CLASS = OpenAIEmbeddings
+MAX_INSERT_BATCH_SIZE = 50_000
 DEFAULT_TIKTOKEN_MODEL_NAME = os.getenv("DEFAULT_TIKTOKEN_MODEL_NAME", "gpt-4")
 AGENT_CHUNK_POLLING_INTERVAL_SECONDS = os.getenv("AGENT_CHUNK_POLLING_INTERVAL_SECONDS", 1.0)
 DEFAULT_TEXT2SQL_DATABASE = "mindsdb"