MindsDB 1.2.9__py3-none-any.whl → 24.12.4.0__py3-none-any.whl

mindsdb/api/mongo/responders/delete.py

@@ -0,0 +1,105 @@
+from mindsdb.api.mongo.classes import Responder
+import mindsdb.api.mongo.functions as helpers
+from mindsdb_sql_parser.ast import Identifier
+from mindsdb_sql_parser.ast.mindsdb import DropPredictor, DropJob, DropMLEngine
+from mindsdb.interfaces.jobs.jobs_controller import JobsController
+
+from mindsdb.api.mongo.classes.query_sql import run_sql_command
+
+
+class Responce(Responder):
+    when = {'delete': helpers.is_true}
+
+    def result(self, query, request_env, mindsdb_env, session):
+        try:
+            res = self._result(query, request_env, mindsdb_env)
+        except Exception as e:
+            res = {
+                'n': 0,
+                'writeErrors': [{
+                    'index': 0,
+                    'code': 0,
+                    'errmsg': str(e)
+                }],
+                'ok': 1
+            }
+        return res
+
+    def _result(self, query, request_env, mindsdb_env):
+        table = query['delete']
+        if table == 'predictors':
+            table = 'models'
+
+        project_name = request_env['database']
+
+        allowed_tables = ('models', 'jobs', 'ml_engines')
+        if table not in allowed_tables:
+            raise Exception(f"Only removing from this collections is supported: {', '.join(allowed_tables)}")
+
+        if len(query['deletes']) != 1:
+            raise Exception("Should be only one argument in REMOVE operation")
+
+        obj_name, obj_id = None, None
+
+        delete_filter = query['deletes'][0]['q']
+        if '_id' in delete_filter:
+            # get name of object
+            obj_id = helpers.objectid_to_int(delete_filter['_id'])
+
+        if 'name' in delete_filter:
+            obj_name = delete_filter['name']
+
+        version = None
+        if 'version' in delete_filter:
+            version = delete_filter['version']
+
+        if obj_name is None and obj_id is None:
+            raise Exception("Can't find object to delete, use filter by name or _id")
+
+        if obj_name is None:
+            if table == 'models':
+                model_id = obj_id >> 20
+                version = obj_id & (2 ** 20 - 1)
+
+                models = mindsdb_env['model_controller'].get_models(
+                    ml_handler_name=None,
+                    project_name=project_name
+                )
+                for model in models:
+                    if model['id'] == model_id:
+                        obj_name = model['name']
+                        break
+                if obj_name is None:
+                    raise Exception("Can't find model by _id")
+            elif table == 'jobs':
+                jobs_controller = JobsController()
+                for job in jobs_controller.get_list(project_name):
+                    if job['id'] == obj_id:
+                        obj_name = job['name']
+                        break
+
+        # delete model
+        if table == 'models':
+            if version is None:
+                ast_query = DropPredictor(Identifier(parts=[project_name, obj_name]))
+                run_sql_command(request_env, ast_query)
+            else:
+                # delete model version
+                ast_query = DropPredictor(Identifier(parts=[project_name, obj_name, str(version)]))
+                run_sql_command(request_env, ast_query)
+
+        elif table == 'jobs':
+            ast_query = DropJob(Identifier(parts=[project_name, obj_name]))
+            run_sql_command(request_env, ast_query)
+
+        elif table == 'ml_engines':
+            ast_query = DropMLEngine(Identifier(parts=[obj_name]))
+            run_sql_command(request_env, ast_query)
+
+        return {
+            'n': 1,
+            'ok': 1
+        }
+
+
+responder = Responce()

mindsdb/api/mongo/responders/describe.py

@@ -0,0 +1,23 @@
+from mindsdb_sql_parser.ast import Describe, Identifier
+
+from mindsdb.api.mongo.classes import Responder
+import mindsdb.api.mongo.functions as helpers
+from mindsdb.api.mongo.classes.query_sql import run_sql_command
+
+
+class Responce(Responder):
+    when = {'describe': helpers.is_true}
+
+    def result(self, query, request_env, mindsdb_env, session):
+        db = query['$db']
+        model = query['describe']
+
+        ast_query = Describe(Identifier(model))
+
+        data = run_sql_command(request_env, ast_query)
+        res = {'data': data, 'ns': f"{db}.{model}"}
+
+        return res
+
+
+responder = Responce()

mindsdb/api/mongo/responders/end_sessions.py

@@ -0,0 +1,13 @@
+from mindsdb.api.mongo.classes import Responder
+import mindsdb.api.mongo.functions as helpers
+
+
+class Responce(Responder):
+    when = {'endSessions': helpers.is_true}
+
+    result = {
+        'ok': 1
+    }
+
+
+responder = Responce()

mindsdb/api/mongo/responders/find.py

@@ -0,0 +1,175 @@
+from bson.int64 import Int64
+
+from mindsdb_sql_parser.ast import Join, Select, Identifier, Describe, Show, Constant
+import mindsdb.api.mongo.functions as helpers
+from mindsdb.api.mongo.classes import Responder
+from mindsdb.api.mongo.utilities.mongodb_ast import MongoToAst
+from mindsdb.interfaces.jobs.jobs_controller import JobsController
+
+from mindsdb.api.mongo.classes.query_sql import run_sql_command
+
+
+def find_to_ast(query, database):
+    mongoToAst = MongoToAst()
+
+    collection = [database, query['find']]
+
+    if not query.get('singleBatch') and 'collection' in query.get('filter'):
+        # JOIN mode
+
+        # upper query
+        ast_query = mongoToAst.find(
+            collection=collection,
+            projection=query.get('projection'),
+            sort=query.get('sort'),
+            limit=query.get('limit'),
+            skip=query.get('skip'),
+        )
+
+        # table_query
+        collection = query['filter']['collection']
+        filter = query['filter'].get('query', {})
+        table_select = mongoToAst.find(
+            collection=collection,
+            filter=filter,
+        )
+        table_select.parentheses = True
+        table_select.alias = Identifier(query['filter']['collection'])
+        table_select.alias.parts = [table_select.alias.parts[-1]]
+        if 'limit' in query:
+            table_select.limit = Constant(query['limit'])
+
+        modifiers = query['filter'].get('modifiers')
+        if modifiers is not None and hasattr(ast_query, 'modifiers'):
+            for modifier in modifiers:
+                table_select.modifiers.append(modifier)
+
+        # convert to join
+        right_table = ast_query.from_table
+
+        ast_join = Join(
+            left=table_select,
+            right=right_table,
+            join_type='join'
+        )
+        ast_query.from_table = ast_join
+
+    else:
+        # is single table
+        ast_query = mongoToAst.find(
+            collection=collection,
+            filter=query.get('filter'),
+            projection=query.get('projection'),
+            sort=query.get('sort'),
+            limit=query.get('limit'),
+            skip=query.get('skip'),
+        )
+        modifiers = query['filter'].get('modifiers')
+        if modifiers is not None and hasattr(ast_query, 'modifiers'):
+            for modifier in modifiers:
+                ast_query.modifiers.append(modifier)
+    return ast_query
+
+
+class Responce(Responder):
+    when = {'find': helpers.is_true}
+
+    def result(self, query, request_env, mindsdb_env, session):
+        database = request_env['database']
+        project_name = request_env['database']
+
+        if database == 'config':
+            # return nothing
+            return {
+                'cursor': {
+                    'id': Int64(0),
+                    'ns': f"{database}.$cmd.{query['find']}",
+                    'firstBatch': []
+                },
+                'ok': 1
+            }
+
+        # system queries
+        elif query['find'] == 'system.version':
+            # For studio3t
+            data = [{
+                "_id": "featureCompatibilityVersion",
+                "version": "3.6"
+            }]
+            cursor = {
+                'id': Int64(0),
+                'ns': f"system.version.$cmd.{query['find']}",
+                'firstBatch': data
+            }
+            return {
+                'cursor': cursor,
+                'ok': 1
+            }
+
+        elif query['find'] == 'ml_engines':
+            ast_query = Show('ml_engines')
+
+        else:
+            ast_query = find_to_ast(query, database)
+
+        # add _id for objects
+        table_name = None
+        obj_idx = {}
+        if isinstance(ast_query, Select) and ast_query.from_table is not None:
+            if isinstance(ast_query.from_table, Identifier):
+                table_name = ast_query.from_table.parts[-1].lower()
+
+                if table_name == 'models':
+
+                    models = mindsdb_env['model_controller'].get_models(
+                        ml_handler_name=None,
+                        project_name=project_name
+                    )
+
+                    for model in models:
+                        obj_idx[model['name']] = model['id']
+
+                    # is select from model without where
+                    if table_name in obj_idx and ast_query.where is None:
+                        # replace query to describe model
+                        ast_query = Describe(ast_query.from_table)
+                elif table_name == 'jobs':
+                    jobs_controller = JobsController()
+                    for job in jobs_controller.get_list(project_name):
+                        obj_idx[job['name']] = job['id']
+
+        data = run_sql_command(request_env, ast_query)
+
+        if table_name == 'models':
+            # for models and models_versions _id is:
+            #   - first 20 bytes is version
+            #   - next bytes is model id
+
+            for row in data:
+                model_id = obj_idx.get(row.get('NAME'))
+                if model_id is not None:
+                    obj_id = model_id << 20
+
+                    obj_id += row.get('VERSION', 0)
+
+                    row['_id'] = helpers.int_to_objectid(obj_id)
+        elif table_name == 'jobs':
+            for row in data:
+                obj_id = obj_idx.get(row.get('NAME'))
+                if obj_id is not None:
+                    row['_id'] = helpers.int_to_objectid(obj_id)
+
+        db = mindsdb_env['config']['api']['mongodb']['database']
+
+        cursor = {
+            'id': Int64(0),
+            'ns': f"{db}.$cmd.{query['find']}",
+            'firstBatch': data
+        }
+        return {
+            'cursor': cursor,
+            'ok': 1
+        }
+
+
+responder = Responce()

mindsdb/api/mongo/responders/get_cmd_line_opts.py

@@ -0,0 +1,18 @@
+from mindsdb.api.mongo.classes import Responder
+import mindsdb.api.mongo.functions as helpers
+
+
+class Responce(Responder):
+    when = {'getCmdLineOpts': helpers.is_true}
+
+    result = {
+        'argv': [
+            'mongod'
+        ],
+        'parsed': {
+        },
+        'ok': 1
+    }
+
+
+responder = Responce()

mindsdb/api/mongo/responders/get_free_monitoring_status.py

@@ -0,0 +1,14 @@
+from mindsdb.api.mongo.classes import Responder
+import mindsdb.api.mongo.functions as helpers
+
+
+class Responce(Responder):
+    when = {'getFreeMonitoringStatus': helpers.is_true}
+
+    result = {
+        'state': 'undecided',
+        'ok': 1
+    }
+
+
+responder = Responce()

mindsdb/api/mongo/responders/get_parameter.py

@@ -0,0 +1,23 @@
+from mindsdb.api.mongo.classes import Responder
+import mindsdb.api.mongo.functions as helpers
+
+
+class Responce(Responder):
+    when = {'getParameter': helpers.is_true}
+
+    def result(self, query, request_env, mindsdb_env, session):
+
+        if query['featureCompatibilityVersion']:
+            res = {
+                "featureCompatibilityVersion":
+                    {
+                        "version": "3.6"
+                    },
+                "ok": 1.0
+            }
+        else:
+            raise NotImplementedError(f'Unknown parameter {query}')
+        return res
+
+
+responder = Responce()

mindsdb/api/mongo/responders/getlog.py

@@ -0,0 +1,14 @@
+from mindsdb.api.mongo.classes import Responder
+import mindsdb.api.mongo.functions as helpers
+
+
+class Responce(Responder):
+    when = {'getLog': helpers.is_true}
+
+    result = {
+        "log": [],
+        "ok": 1
+    }
+
+
+responder = Responce()

mindsdb/api/mongo/responders/host_info.py

@@ -0,0 +1,28 @@
+from mindsdb.api.mongo.classes import Responder
+import mindsdb.api.mongo.functions as helpers
+
+
+class Responce(Responder):
+    when = {'hostInfo': helpers.is_true}
+
+    # NOTE answer is not full, but looks like this info is enough
+    result = {
+        'system': {
+            'cpuAddrSize': 64,
+            'memSizeMB': 15951,
+            'numCores': 4,
+            'cpuArch': "x86_64",
+            'numaEnabled': False
+        },
+        'os': {
+            'type': "Linux",
+            'name': "Ubuntu",
+            'version': "20.04"
+        },
+        'extra': {
+        },
+        'ok': 1
+    }
+
+
+responder = Responce()

mindsdb/api/mongo/responders/insert.py

@@ -0,0 +1,268 @@
+import pandas as pd
+from mindsdb_sql_parser.ast import Constant, Identifier, Insert, OrderBy, TableColumn
+from mindsdb_sql_parser.ast.mindsdb import (
+    CreateJob,
+    CreateMLEngine,
+    CreatePredictor,
+    FinetunePredictor,
+    RetrainPredictor,
+)
+
+import mindsdb.api.mongo.functions as helpers
+from mindsdb.api.mongo.classes import Responder
+from mindsdb.api.mongo.classes.query_sql import run_sql_command
+from mindsdb.api.mongo.responders.aggregate import aggregate_to_ast
+from mindsdb.api.mongo.responders.find import find_to_ast
+from mindsdb.api.mongo.utilities.mongodb_parser import MongodbParser
+from mindsdb.integrations.libs.response import HandlerStatusResponse
+from mindsdb.utilities import log
+
+logger = log.getLogger(__name__)
+
+
+class Responce(Responder):
+    when = {"insert": helpers.is_true}
+
+    def result(self, query, request_env, mindsdb_env, session):
+        try:
+            res = self._result(query, request_env, mindsdb_env)
+        except Exception as e:
+            logger.error(e)
+            res = {
+                "n": 0,
+                "writeErrors": [{"index": 0, "code": 0, "errmsg": str(e)}],
+                "ok": 1,
+            }
+        return res
+
+    def _insert_database(self, query, request_env, mindsdb_env):
+
+        for doc in query["documents"]:
+            if "_id" in doc:
+                del doc["_id"]
+            for field in ("name", "engine", "connection_args"):
+                if field not in doc:
+                    raise Exception(f"'{field}' must be specified")
+
+            status = HandlerStatusResponse(success=False)
+            try:
+                handler = mindsdb_env["integration_controller"].create_tmp_handler(
+                    name=doc['name'],
+                    engine=doc["engine"],
+                    connection_args=doc["connection_args"]
+                )
+                status = handler.check_connection()
+            except Exception as e:
+                status.error_message = str(e)
+
+            if status.success is False:
+                raise Exception(f"Can't connect to db: {status.error_message}")
+
+            integration = mindsdb_env["integration_controller"].get(doc["name"])
+            if integration is not None:
+                raise Exception(f"Database '{doc['name']}' already exists.")
+
+        for doc in query["documents"]:
+            mindsdb_env["integration_controller"].add(
+                doc["name"], doc["engine"], doc["connection_args"]
+            )
+
+    def _insert_model(self, query, request_env, mindsdb_env):
+        predictors_columns = [
+            "name",
+            "status",
+            "accuracy",
+            "predict",
+            "select_data_query",
+            "training_options",
+            "connection",
+        ]
+
+        if len(query["documents"]) != 1:
+            raise Exception("Must be inserted just one predictor at time")
+
+        for doc in query["documents"]:
+            if "_id" in doc:
+                del doc["_id"]
+
+            action = doc.pop("action", "create").lower()
+
+            bad_columns = [x for x in doc if x not in predictors_columns]
+            if len(bad_columns) > 0:
+                raise Exception(
+                    f"Is no possible insert this columns to 'predictors' collection: {', '.join(bad_columns)}"
+                )
+
+            if "name" not in doc:
+                raise Exception("Please, specify 'name' field")
+
+            if "predict" not in doc:
+                predict = None
+                if action == "create":
+                    raise Exception("Please, specify 'predict' field")
+            else:
+                predict = doc["predict"]
+                if not isinstance(predict, list):
+                    predict = [Identifier(x.strip()) for x in predict.split(",")]
+
+            order_by = None
+            group_by = None
+            ts_settings = {}
+
+            kwargs = doc.get("training_options", {})
+
+            if "timeseries_settings" in kwargs:
+                ts_settings = kwargs.pop("timeseries_settings")
+
+                # mongo shell client sends int as float. need to convert it to int
+                for key in ("window", "horizon"):
+                    val = ts_settings.get(key)
+                    if val is not None:
+                        ts_settings[key] = int(val)
+
+                if "order_by" in ts_settings:
+                    order_by = ts_settings["order_by"]
+                    if not isinstance(order_by, list):
+                        order_by = [order_by]
+
+                    order_by = [OrderBy(Identifier(x)) for x in order_by]
+                if "group_by" in ts_settings:
+                    group_by = [Identifier(x) for x in ts_settings.get("group_by", [])]
+
+            using = dict(kwargs)
+
+            select_data_query = doc.get("select_data_query")
+            integration_name = None
+            if "connection" in doc:
+                integration_name = Identifier(doc["connection"])
+
+            Class = CreatePredictor
+            if action == "retrain":
+                Class = RetrainPredictor
+            elif action == "finetune":
+                Class = FinetunePredictor
+
+            create_predictor_ast = Class(
+                name=Identifier(f"{request_env['database']}.{doc['name']}"),
+                integration_name=integration_name,
+                query_str=select_data_query,
+                targets=predict,
+                order_by=order_by,
+                group_by=group_by,
+                window=ts_settings.get("window"),
+                horizon=ts_settings.get("horizon"),
+                using=using,
+            )
+
+            run_sql_command(request_env, create_predictor_ast)
+
+    def _insert_job(self, query, request_env, mindsdb_env):
+        for doc in query["documents"]:
+
+            query_str = doc["query"]
+            # try parse as mongo
+            parser = MongodbParser()
+            try:
+                mql = parser.from_string(query_str)
+
+                method = mql.pipeline[0]["method"]
+                if method == "find":
+                    args = mql.pipeline[0]["args"]
+                    projection = None
+                    if len(args) > 1:
+                        projection = args[1]
+                    query = {
+                        "find": mql.collection,
+                        "filter": args[0],
+                        "projection": projection,
+                    }
+
+                    for step in mql.pipeline[1:]:
+                        if step["method"] == "limit":
+                            query["limit"] = step["args"][0]
+                        if step["method"] == "skip":
+                            query["skip"] = step["args"][0]
+                        if step["method"] == "sort":
+                            query["sort"] = step["args"][0]
+                    # TODO implement group modifiers
+                    ast_query = find_to_ast(
+                        query, request_env.get("database", "mindsdb")
+                    )
+
+                    # to string
+                    query_str = ast_query.to_string()
+                elif method == "aggregate":
+                    query = {
+                        "aggregate": mql.collection,
+                        "pipeline": mql.pipeline[0]["args"][0],
+                    }
+                    ast_query = aggregate_to_ast(
+                        query, request_env.get("database", "mindsdb")
+                    )
+                    query_str = ast_query.to_string()
+            except Exception:
+                # keep query
+                pass
+
+            repeat_str = doc.get("schedule_str").lower().lstrip("every ")
+
+            ast_query = CreateJob(
+                name=Identifier(doc["name"]),
+                start_str=doc.get("start_at"),
+                end_str=doc.get("end_at"),
+                query_str=query_str,
+                repeat_str=repeat_str,
+            )
+            run_sql_command(request_env, ast_query)
+
+    def _insert_ml_engine(self, query, request_env, mindsdb_env):
+        for doc in query["documents"]:
+
+            ast_query = CreateMLEngine(
+                name=Identifier(doc["name"]),
+                handler=doc["handler"],
+                params=doc.get("params"),
+            )
+
+            run_sql_command(request_env, ast_query)
+
+    def _result(self, query, request_env, mindsdb_env):
+        table = query["insert"]
+
+        if table == "databases":
+            self._insert_database(query, request_env, mindsdb_env)
+
+        elif table in ["predictors", "models"]:
+            self._insert_model(query, request_env, mindsdb_env)
+
+        elif table == "jobs":
+            self._insert_job(query, request_env, mindsdb_env)
+
+        elif table == "ml_engines":
+            self._insert_ml_engine(query, request_env, mindsdb_env)
+
+        else:
+            # regular insert
+
+            df = pd.DataFrame(query["documents"])
+            if "_id" in df.columns:
+                df = df.drop("_id", axis=1)
+
+            data = df.to_dict("split")
+            values = []
+            for row in data["data"]:
+                values.append([Constant(i) for i in row])
+            ast_query = Insert(
+                table=Identifier(table),
+                columns=[TableColumn(c) for c in data["columns"]],
+                values=values,
+            )
+
+            run_sql_command(request_env, ast_query)
+
+        result = {"n": len(query["documents"]), "ok": 1}
+
+        return result
+
+
+responder = Responce()