MindsDB 25.8.3.0__py3-none-any.whl → 25.9.1.1__py3-none-any.whl

mindsdb/integrations/handlers/mongodb_handler/mongodb_handler.py

@@ -10,13 +10,13 @@ from pymongo import MongoClient
 from pymongo.errors import ServerSelectionTimeoutError, OperationFailure, ConfigurationError, InvalidURI
 from typing import Text, List, Dict, Any, Union
 
-from mindsdb.api.mongo.utilities.mongodb_query import MongoQuery
-from mindsdb.api.mongo.utilities.mongodb_parser import MongodbParser
+from mindsdb.integrations.handlers.mongodb_handler.utils.mongodb_query import MongoQuery
+from mindsdb.integrations.handlers.mongodb_handler.utils.mongodb_parser import MongodbParser
 from mindsdb.integrations.libs.base import DatabaseHandler
 from mindsdb.integrations.libs.response import (
     HandlerStatusResponse as StatusResponse,
     HandlerResponse as Response,
-    RESPONSE_TYPE
+    RESPONSE_TYPE,
 )
 from mindsdb.utilities import log
 from .utils.mongodb_render import MongodbRender
@@ -31,7 +31,7 @@ class MongoDBHandler(DatabaseHandler):
     """
 
     _SUBSCRIBE_SLEEP_INTERVAL = 0.5
-    name = 'mongodb'
+    name = "mongodb"
 
     def __init__(self, name: Text, **kwargs: Any) -> None:
         """
@@ -42,13 +42,13 @@ class MongoDBHandler(DatabaseHandler):
             kwargs: Arbitrary keyword arguments including the connection data.
         """
         super().__init__(name)
-        connection_data = kwargs['connection_data']
+        connection_data = kwargs["connection_data"]
         self.host = connection_data.get("host")
         self.port = int(connection_data.get("port") or 27017)
         self.user = connection_data.get("username")
         self.password = connection_data.get("password")
-        self.database = connection_data.get('database')
-        self.flatten_level = connection_data.get('flatten_level', 0)
+        self.database = connection_data.get("database")
+        self.flatten_level = connection_data.get("flatten_level", 0)
 
         self.connection = None
         self.is_connected = False
@@ -72,31 +72,27 @@ class MongoDBHandler(DatabaseHandler):
         """
         kwargs = {}
         if isinstance(self.user, str) and len(self.user) > 0:
-            kwargs['username'] = self.user
+            kwargs["username"] = self.user
 
         if isinstance(self.password, str) and len(self.password) > 0:
-            kwargs['password'] = self.password
+            kwargs["password"] = self.password
 
-        if re.match(r'/?.*tls=true', self.host.lower()):
-            kwargs['tls'] = True
+        if re.match(r"/?.*tls=true", self.host.lower()):
+            kwargs["tls"] = True
 
-        if re.match(r'/?.*tls=false', self.host.lower()):
-            kwargs['tls'] = False
+        if re.match(r"/?.*tls=false", self.host.lower()):
+            kwargs["tls"] = False
 
         try:
-            connection = MongoClient(
-                self.host,
-                port=self.port,
-                **kwargs
-            )
+            connection = MongoClient(self.host, port=self.port, **kwargs)
         except InvalidURI as invalid_uri_error:
-            logger.error(f'Invalid URI provided for MongoDB connection: {invalid_uri_error}!')
+            logger.error(f"Invalid URI provided for MongoDB connection: {invalid_uri_error}!")
             raise
         except ConfigurationError as config_error:
-            logger.error(f'Configuration error connecting to MongoDB: {config_error}!')
+            logger.error(f"Configuration error connecting to MongoDB: {config_error}!")
             raise
         except Exception as unknown_error:
-            logger.error(f'Unknown error connecting to MongoDB: {unknown_error}!')
+            logger.error(f"Unknown error connecting to MongoDB: {unknown_error}!")
             raise
 
         # Get the database name from the connection if it's not provided.
@@ -107,7 +103,9 @@ class MongoDBHandler(DatabaseHandler):
         self.connection = connection
         return self.connection
 
-    def subscribe(self, stop_event: threading.Event, callback: callable, table_name: Text, columns: List = None, **kwargs: Any) -> None:
+    def subscribe(
+        self, stop_event: threading.Event, callback: callable, table_name: Text, columns: List = None, **kwargs: Any
+    ) -> None:
         """
         Subscribes to changes in a MongoDB collection and calls the provided callback function when changes occur.
 
@@ -131,26 +129,26 @@ class MongoDBHandler(DatabaseHandler):
                 time.sleep(self._SUBSCRIBE_SLEEP_INTERVAL)
                 continue
 
-            _id = res['documentKey']['_id']
-            if res['operationType'] == 'insert':
+            _id = res["documentKey"]["_id"]
+            if res["operationType"] == "insert":
                 if columns is not None:
-                    updated_columns = set(res['fullDocument'].keys())
+                    updated_columns = set(res["fullDocument"].keys())
                     if not set(columns) & set(updated_columns):
                         # Do nothing.
                         continue
 
-                callback(row=res['fullDocument'], key={'_id': _id})
+                callback(row=res["fullDocument"], key={"_id": _id})
 
-            if res['operationType'] == 'update':
+            if res["operationType"] == "update":
                 if columns is not None:
-                    updated_columns = set(res['updateDescription']['updatedFields'].keys())
+                    updated_columns = set(res["updateDescription"]["updatedFields"].keys())
                     if not set(columns) & set(updated_columns):
                         # Do nothing.
                         continue
 
                 # Get the full document.
-                full_doc = con[self.database][table_name].find_one(res['documentKey'])
-                callback(row=full_doc, key={'_id': _id})
+                full_doc = con[self.database][table_name].find_one(res["documentKey"])
+                callback(row=full_doc, key={"_id": _id})
 
     def disconnect(self) -> None:
         """
@@ -178,14 +176,20 @@ class MongoDBHandler(DatabaseHandler):
 
             # Check if the database exists.
             if self.database not in con.list_database_names():
-                raise ValueError(f'Database {self.database} not found!')
+                raise ValueError(f"Database {self.database} not found!")
 
             response.success = True
-        except (InvalidURI, ServerSelectionTimeoutError, OperationFailure, ConfigurationError, ValueError) as known_error:
-            logger.error(f'Error connecting to MongoDB {self.database}, {known_error}!')
+        except (
+            InvalidURI,
+            ServerSelectionTimeoutError,
+            OperationFailure,
+            ConfigurationError,
+            ValueError,
+        ) as known_error:
+            logger.error(f"Error connecting to MongoDB {self.database}, {known_error}!")
             response.error_message = str(known_error)
         except Exception as unknown_error:
-            logger.error(f'Unknown error connecting to MongoDB {self.database}, {unknown_error}!')
+            logger.error(f"Unknown error connecting to MongoDB {self.database}, {unknown_error}!")
             response.error_message = str(unknown_error)
 
         if response.success and need_to_close:
@@ -211,13 +215,10 @@ class MongoDBHandler(DatabaseHandler):
 
         if isinstance(query, dict):
             # Fallback for the previous API.
-            mquery = MongoQuery(query['collection'])
+            mquery = MongoQuery(query["collection"])
 
-            for c in query['call']:
-                mquery.add_step({
-                    'method': c['method'],
-                    'args': c['args']
-                })
+            for c in query["call"]:
+                mquery.add_step({"method": c["method"], "args": c["args"]})
 
             query = mquery
 
@@ -229,16 +230,15 @@ class MongoDBHandler(DatabaseHandler):
         # Check if the collection exists.
         if collection not in con[database].list_collection_names():
             return Response(
-                RESPONSE_TYPE.ERROR,
-                error_message=f'Collection {collection} not found in database {database}!'
+                RESPONSE_TYPE.ERROR, error_message=f"Collection {collection} not found in database {database}!"
             )
 
         try:
             cursor = con[database][collection]
 
             for step in query.pipeline:
-                fnc = getattr(cursor, step['method'])
-                cursor = fnc(*step['args'])
+                fnc = getattr(cursor, step["method"])
+                cursor = fnc(*step["args"])
 
             result = []
             if not isinstance(cursor, pymongo.results.UpdateResult):
@@ -254,16 +254,10 @@ class MongoDBHandler(DatabaseHandler):
                 columns = list(self.get_columns(collection).data_frame.Field)
                 df = pd.DataFrame([], columns=columns)
 
-            response = Response(
-                RESPONSE_TYPE.TABLE,
-                df
-            )
+            response = Response(RESPONSE_TYPE.TABLE, df)
         except Exception as e:
-            logger.error(f'Error running query: {query} on {self.database}.{collection}!')
-            response = Response(
-                RESPONSE_TYPE.ERROR,
-                error_message=str(e)
-            )
+            logger.error(f"Error running query: {query} on {self.database}.{collection}!")
+            response = Response(RESPONSE_TYPE.ERROR, error_message=str(e))
 
         return response
 
@@ -289,7 +283,7 @@ class MongoDBHandler(DatabaseHandler):
             if level > 0:
                 if isinstance(v, dict):
                     for k2, v2 in self.flatten(v, level=level - 1).items():
-                        add[f'{k}.{k2}'] = v2
+                        add[f"{k}.{k2}"] = v2
                     del_keys.append(k)
 
         if add:
@@ -324,15 +318,10 @@ class MongoDBHandler(DatabaseHandler):
         """
         con = self.connect()
         collections = con[self.database].list_collection_names()
-        collections_ar = [
-            [i] for i in collections
-        ]
-        df = pd.DataFrame(collections_ar, columns=['table_name'])
+        collections_ar = [[i] for i in collections]
+        df = pd.DataFrame(collections_ar, columns=["table_name"])
 
-        response = Response(
-            RESPONSE_TYPE.TABLE,
-            df
-        )
+        response = Response(RESPONSE_TYPE.TABLE, df)
 
         return response
 
@@ -363,10 +352,7 @@ class MongoDBHandler(DatabaseHandler):
             for k, v in record.items():
                 data.append([k, type(v).__name__])
 
-        df = pd.DataFrame(data, columns=['Field', 'Type'])
+        df = pd.DataFrame(data, columns=["Field", "Type"])
 
-        response = Response(
-            RESPONSE_TYPE.TABLE,
-            df
-        )
+        response = Response(RESPONSE_TYPE.TABLE, df)
         return response

mindsdb/integrations/handlers/mongodb_handler/requirements.txt

@@ -0,0 +1 @@
+pymongo == 4.8.0

mindsdb/{api/mongo/utilities → integrations/handlers/mongodb_handler/utils}/mongodb_ast.py

@@ -6,9 +6,8 @@ from mindsdb_sql_parser.ast import OrderBy, Identifier, Star, Select, Constant,
 
 
 class MongoToAst:
-
     """
-      Converts query mongo to AST format
+    Converts query mongo to AST format
     """
 
     def from_mongoqeury(self, query):
@@ -19,24 +18,22 @@ class MongoToAst:
         filter, projection = None, None
         sort, limit, skip = None, None, None
         for step in query.pipeline:
-            if step['method'] == 'find':
-                filter = step['args'][0]
+            if step["method"] == "find":
+                filter = step["args"][0]
                 if len(step) > 1:
-                    projection = step['args'][1]
-            elif step['method'] == 'sort':
-                sort = step['args'][0]
-            elif step['method'] == 'limit':
-                limit = step['args'][0]
-            elif step['method'] == 'skip':
-                skip = step['args'][0]
-
-        return self.find(collection, filter=filter,
-                         sort=sort, projection=projection,
-                         limit=limit, skip=skip)
-
-    def find(self, collection: t.Union[list, str],
-             filter=None, sort=None, projection=None,
-             limit=None, skip=None, **kwargs):
+                    projection = step["args"][1]
+            elif step["method"] == "sort":
+                sort = step["args"][0]
+            elif step["method"] == "limit":
+                limit = step["args"][0]
+            elif step["method"] == "skip":
+                skip = step["args"][0]
+
+        return self.find(collection, filter=filter, sort=sort, projection=projection, limit=limit, skip=skip)
+
+    def find(
+        self, collection: t.Union[list, str], filter=None, sort=None, projection=None, limit=None, skip=None, **kwargs
+    ):
         # https://www.mongodb.com/docs/v4.2/reference/method/db.collection.find/
 
         order_by = None
@@ -44,12 +41,7 @@ class MongoToAst:
             # sort is dict
             order_by = []
             for col, direction in sort.items():
-                order_by.append(
-                    OrderBy(
-                        field=Identifier(parts=[col]),
-                        direction='DESC' if direction == -1 else 'ASC'
-                    )
-                )
+                order_by.append(OrderBy(field=Identifier(parts=[col]), direction="DESC" if direction == -1 else "ASC"))
 
         if projection is not None:
             targets = []
@@ -59,9 +51,7 @@ class MongoToAst:
                     alias = Identifier(parts=[alias])
                 else:
                     alias = None
-                targets.append(
-                    Identifier(path_str=col, alias=alias)
-                )
+                targets.append(Identifier(path_str=col, alias=alias))
         else:
             targets = [Star()]
 
@@ -92,13 +82,13 @@ class MongoToAst:
 
     def convert_filter(self, filter):
         cond_ops = {
-            '$and': 'and',
-            '$or': 'or',
+            "$and": "and",
+            "$or": "or",
         }
 
         ast_filter = None
         for k, v in filter.items():
-            if k in ('$or', '$and'):
+            if k in ("$or", "$and"):
                 # suppose it is one key in dict
 
                 op = cond_ops[k]
@@ -116,7 +106,7 @@ class MongoToAst:
                     arg1 = BinaryOperation(op=op, args=[arg1, node])
 
                 return arg1
-            if k in ('$where', '$expr'):
+            if k in ("$where", "$expr"):
                 # try to parse simple expression like 'this.saledate > this.latest'
                 return MongoWhereParser(v).to_ast()
 
@@ -129,25 +119,12 @@ class MongoToAst:
             if ast_filter is None:
                 ast_filter = ast_com
             else:
-                ast_filter = BinaryOperation(op='and', args=[
-                    ast_filter,
-                    ast_com
-                ])
+                ast_filter = BinaryOperation(op="and", args=[ast_filter, ast_com])
         return ast_filter
 
     def handle_filter(self, value):
-        ops = {
-            '$ge': '>=',
-            '$gt': '>',
-            '$lt': '<',
-            '$le': '<=',
-            '$ne': '!=',
-            '$eq': '='
-        }
-        in_ops = {
-            '$in': 'in',
-            '$nin': 'not in'
-        }
+        ops = {"$ge": ">=", "$gt": ">", "$lt": "<", "$le": "<=", "$ne": "!=", "$eq": "="}
+        in_ops = {"$in": "in", "$nin": "not in"}
 
         if isinstance(value, dict):
             key, value = list(value.items())[0]
@@ -158,18 +135,18 @@ class MongoToAst:
             if key in in_ops:
                 op = in_ops[key]
                 if not isinstance(value, list):
-                    raise NotImplementedError(f'Unknown type {key}, {value}')
+                    raise NotImplementedError(f"Unknown type {key}, {value}")
                 value = Tuple(value)
 
                 return op, value
 
-            raise NotImplementedError(f'Unknown type {key}')
+            raise NotImplementedError(f"Unknown type {key}")
 
         elif isinstance(value, list):
-            raise NotImplementedError(f'Unknown filter {value}')
+            raise NotImplementedError(f"Unknown filter {value}")
         else:
             # is simple type
-            op = '='
+            op = "="
             value = value
             return op, value
 
@@ -181,13 +158,12 @@ class MongoWhereParser:
     def to_ast(self):
         # parse as python string
         # replace '=' with '=='
-        query = re.sub(r'([^=><])=([^=])', r'\1==\2', self.query)
+        query = re.sub(r"([^=><])=([^=])", r"\1==\2", self.query)
 
-        tree = py_ast.parse(query, mode='eval')
+        tree = py_ast.parse(query, mode="eval")
         return self.process(tree.body)
 
     def process(self, node):
-
         if isinstance(node, py_ast.BoolOp):
             # is AND or OR
             op = node.op.__class__.__name__
@@ -202,7 +178,7 @@ class MongoWhereParser:
         if isinstance(node, py_ast.Compare):
             # it is
             if len(node.ops) != 1:
-                raise NotImplementedError(f'Multiple ops {node.ops}')
+                raise NotImplementedError(f"Multiple ops {node.ops}")
             op = self.compare_op(node.ops[0])
             arg1 = self.process(node.left)
             arg2 = self.process(node.comparators[0])
@@ -210,7 +186,7 @@ class MongoWhereParser:
 
         if isinstance(node, py_ast.Name):
             # is special operator: latest, ...
-            if node.id == 'latest':
+            if node.id == "latest":
                 return Latest()
 
         if isinstance(node, py_ast.Constant):
@@ -228,28 +204,27 @@ class MongoWhereParser:
 
         if isinstance(node, py_ast.Attribute):
             # is 'this.field' - is attribute
-            if node.value.id != 'this':
-                raise NotImplementedError(f'Unknown variable {node.value.id}')
+            if node.value.id != "this":
+                raise NotImplementedError(f"Unknown variable {node.value.id}")
             return Identifier(parts=[node.attr])
 
-        raise NotImplementedError(f'Unknown node {node}')
+        raise NotImplementedError(f"Unknown node {node}")
 
     def compare_op(self, op):
-
         opname = op.__class__.__name__
 
         # TODO: in, not
 
         ops = {
-            'Eq': '=',
-            'NotEq': '!=',
-            'Gt': '>',
-            'Lt': '<',
-            'GtE': '>=',
-            'LtE': '<=',
+            "Eq": "=",
+            "NotEq": "!=",
+            "Gt": ">",
+            "Lt": "<",
+            "GtE": ">=",
+            "LtE": "<=",
         }
         if opname not in ops:
-            raise NotImplementedError(f'Unknown $where op: {opname}')
+            raise NotImplementedError(f"Unknown $where op: {opname}")
         return ops[opname]
 
     @staticmethod

mindsdb/{api/mongo/utilities → integrations/handlers/mongodb_handler/utils}/mongodb_parser.py

@@ -1,4 +1,3 @@
-
 import ast as py_ast
 
 import dateutil.parser
@@ -8,30 +7,27 @@ from .mongodb_query import MongoQuery
 
 
 class MongodbParser:
-    '''
-        Converts string into MongoQuery
-    '''
+    """
+    Converts string into MongoQuery
+    """
 
     def from_string(self, call_str):
-        tree = py_ast.parse(call_str.strip(), mode='eval')
+        tree = py_ast.parse(call_str.strip(), mode="eval")
         calls = self.process(tree.body)
         # first call contents collection
-        method1 = calls[0]['method']
+        method1 = calls[0]["method"]
         if len(method1) < 2:
-            raise IndexError('Collection not found')
+            raise IndexError("Collection not found")
         collection = method1[-2]
 
         mquery = MongoQuery(collection)
 
         # keep only last name
-        calls[0]['method'] = [method1[-1]]
+        calls[0]["method"] = [method1[-1]]
 
         # convert method names: get first item of list
         for c in calls:
-            mquery.add_step({
-                'method': c['method'][0],
-                'args': c['args']
-            })
+            mquery.add_step({"method": c["method"][0], "args": c["args"]})
 
         return mquery
 
@@ -49,20 +45,17 @@ class MongodbParser:
                 func = node.func.id
 
                 # special functions:
-                if func == 'ISODate':
+                if func == "ISODate":
                     return dateutil.parser.isoparse(args[0])
-                if func == 'ObjectId':
+                if func == "ObjectId":
                     return ObjectId(args[0])
             elif isinstance(node.func, py_ast.Attribute):
                 # it can be an attribute or pipeline
                 previous_call, func = self.process_func_name(node.func)
             else:
-                raise NotImplementedError(f'Unknown function type: {node.func}')
+                raise NotImplementedError(f"Unknown function type: {node.func}")
 
-            call = [{
-                'method': func,
-                'args': args
-            }]
+            call = [{"method": func, "args": args}]
             if previous_call is not None:
                 call = previous_call + call
 
@@ -75,7 +68,6 @@ class MongodbParser:
             return elements
 
         if isinstance(node, py_ast.Dict):
-
             keys = []
             for node2 in node.keys:
                 if isinstance(node2, py_ast.Constant):
@@ -85,7 +77,7 @@ class MongodbParser:
                 elif isinstance(node2, py_ast.Name):
                     value = node2.id
                 else:
-                    raise NotImplementedError(f'Unknown dict key {node2}')
+                    raise NotImplementedError(f"Unknown dict key {node2}")
 
                 keys.append(value)
 
@@ -98,11 +90,11 @@ class MongodbParser:
         if isinstance(node, py_ast.Name):
             # special attributes
             name = node.id
-            if name == 'true':
+            if name == "true":
                 return True
-            elif name == 'false':
+            elif name == "false":
                 return False
-            elif name == 'null':
+            elif name == "null":
                 return None
 
         if isinstance(node, py_ast.Constant):
@@ -122,7 +114,7 @@ class MongodbParser:
                 value = self.process(node.operand)
                 return -value
 
-        raise NotImplementedError(f'Unknown node {node}')
+        raise NotImplementedError(f"Unknown node {node}")
 
     def process_func_name(self, node):
         previous_call = None

mindsdb/{api/mongo/utilities → integrations/handlers/mongodb_handler/utils}/mongodb_query.py

@@ -4,17 +4,15 @@ from bson import ObjectId
 
 
 class MongoJSONEncoder(json.JSONEncoder):
-
     def default(self, obj):
         if isinstance(obj, dt.datetime):
-            return f'ISODate({obj.isoformat()})'
+            return f"ISODate({obj.isoformat()})"
         if isinstance(obj, ObjectId):
-            return f'ObjectId({str(obj)})'
+            return f"ObjectId({str(obj)})"
         return super(MongoJSONEncoder, self).default(obj)
 
 
 class MongoQuery:
-
     def __init__(self, collection, pipline=None):
         self.collection = collection
         self.pipeline = []
@@ -30,10 +28,8 @@ class MongoQuery:
         #     'args': [{c: 3}]
         # }
 
-        if 'method' not in step \
-                or 'args' not in step \
-                or not isinstance(step['args'], list):
-            raise AttributeError(f'Wrong step {step}')
+        if "method" not in step or "args" not in step or not isinstance(step["args"], list):
+            raise AttributeError(f"Wrong step {step}")
 
         self.pipeline.append(step)
 
@@ -43,10 +39,8 @@ class MongoQuery:
     def __getattr__(self, item):
         # return callback to save step of pipeline
         def fnc(*args):
-            self.pipeline.append({
-                'method': item,
-                'args': args
-            })
+            self.pipeline.append({"method": item, "args": args})
+
         return fnc
 
     def __str__(self):
@@ -72,13 +66,13 @@ class MongoQuery:
          "db_test.fish.find({a:1}, {b:2}).sort({c:3})"
         """
 
-        call_str = f'db.{self.collection}'
+        call_str = f"db.{self.collection}"
         for step in self.pipeline:
             args_str = []
-            for arg in step['args']:
+            for arg in step["args"]:
                 args_str.append(MongoJSONEncoder().encode(arg))
-            call_str += f'.{step["method"]}({",".join(args_str)})'
+            call_str += f".{step['method']}({','.join(args_str)})"
         return call_str
 
     def __repr__(self):
-        return f'MongoQuery({self.collection}, {str(self.pipeline)})'
+        return f"MongoQuery({self.collection}, {str(self.pipeline)})"