MindsDB 25.4.5.0__py3-none-any.whl → 25.5.4.0__py3-none-any.whl

mindsdb/api/executor/sql_query/steps/subselect_step.py

@@ -55,7 +55,7 @@ class SubSelectStepCall(BaseStepCall):
 
             for col_name in query_cols:
                 if col_name not in result_cols:
-                    result.add_column(Column(col_name))
+                    result.add_column(Column(name=col_name))
 
         # inject previous step values
         if isinstance(query, Select):
@@ -69,12 +69,10 @@ class SubSelectStepCall(BaseStepCall):
         df = result.to_df()
         res = query_df(df, query, session=self.session)
 
-        result2 = ResultSet()
         # get database from first column
         database = result.columns[0].database
-        result2.from_df(res, database, table_name)
 
-        return result2
+        return ResultSet.from_df(res, database, table_name)
 
 
 class QueryStepCall(BaseStepCall):
@@ -86,7 +84,7 @@ class QueryStepCall(BaseStepCall):
 
         if step.from_table is not None:
             if isinstance(step.from_table, pd.DataFrame):
-                result_set = ResultSet().from_df(step.from_table)
+                result_set = ResultSet.from_df(step.from_table)
             else:
                 result_set = self.steps_data[step.from_table.step_num]
         else:
@@ -208,6 +206,4 @@ class QueryStepCall(BaseStepCall):
         query.from_table = Identifier('df_table')
         res = query_df(df, query, session=self.session)
 
-        data = ResultSet().from_df_cols(res, col_names, strict=False)
-
-        return data
+        return ResultSet.from_df_cols(df=res, columns_dict=col_names, strict=False)

mindsdb/api/executor/sql_query/steps/union_step.py

@@ -48,6 +48,4 @@ class UnionStepCall(BaseStepCall):
         })
         resp_df.replace({np.nan: None}, inplace=True)
 
-        data = ResultSet().from_df_cols(resp_df, col_names=names)
-
-        return data
+        return ResultSet.from_df_cols(df=resp_df, columns_dict=names)

mindsdb/api/http/initialize.py

@@ -4,9 +4,11 @@ import mimetypes
 import threading
 import traceback
 import webbrowser
+
 from pathlib import Path
 from http import HTTPStatus
 
+
 import requests
 from flask import Flask, url_for, make_response, request, send_from_directory
 from flask.json import dumps
@@ -50,17 +52,37 @@ from mindsdb.utilities.json_encoder import CustomJSONProvider
 from mindsdb.utilities.ps import is_pid_listen_port, wait_func_is_true
 from mindsdb.utilities.sentry import sentry_sdk  # noqa: F401
 from mindsdb.utilities.otel import trace  # noqa: F401
-from opentelemetry.instrumentation.flask import FlaskInstrumentor  # noqa: F401
-from opentelemetry.instrumentation.requests import RequestsInstrumentor  # noqa: F401
 
 logger = log.getLogger(__name__)
 
 
+class _NoOpFlaskInstrumentor:
+    def instrument_app(self, app):
+        pass
+
+
+class _NoOpRequestsInstrumentor:
+    def instrument(self):
+        pass
+
+
+try:
+    from opentelemetry.instrumentation.flask import FlaskInstrumentor
+    from opentelemetry.instrumentation.requests import RequestsInstrumentor
+except ImportError:
+    logger.debug(
+        "OpenTelemetry is not avaiable. Please run `pip install -r requirements/requirements-opentelemetry.txt` to use it."
+    )
+    FlaskInstrumentor = _NoOpFlaskInstrumentor
+    RequestsInstrumentor = _NoOpRequestsInstrumentor
+
+
 class Swagger_Api(Api):
     """
     This is a modification of the base Flask Restplus Api class due to the issue described here
     https://github.com/noirbizarre/flask-restplus/issues/223
     """
+
     @property
     def specs_url(self):
         return url_for(self.endpoint("specs"), _external=False)
@@ -75,7 +97,10 @@ def custom_output_json(data, code, headers=None):
 def get_last_compatible_gui_version() -> Version:
     logger.debug("Getting last compatible frontend..")
     try:
-        res = requests.get('https://mindsdb-web-builds.s3.amazonaws.com/compatible-config.json', timeout=5)
+        res = requests.get(
+            "https://mindsdb-web-builds.s3.amazonaws.com/compatible-config.json",
+            timeout=5,
+        )
     except (ConnectionError, requests.exceptions.ConnectionError) as e:
         logger.error(f"Is no connection. {e}")
         return False
@@ -101,13 +126,16 @@ def get_last_compatible_gui_version() -> Version:
         gui_versions = {}
         max_mindsdb_lv = None
         max_gui_lv = None
-        for el in versions['mindsdb']:
-            if el['mindsdb_version'] is None:
-                gui_lv = parse_version(el['gui_version'])
+        for el in versions["mindsdb"]:
+            if el["mindsdb_version"] is None:
+                gui_lv = parse_version(el["gui_version"])
             else:
-                mindsdb_lv = parse_version(el['mindsdb_version'])
-                gui_lv = parse_version(el['gui_version'])
-                if mindsdb_lv.base_version not in gui_versions or gui_lv > gui_versions[mindsdb_lv.base_version]:
+                mindsdb_lv = parse_version(el["mindsdb_version"])
+                gui_lv = parse_version(el["gui_version"])
+                if (
+                    mindsdb_lv.base_version not in gui_versions
+                    or gui_lv > gui_versions[mindsdb_lv.base_version]
+                ):
                     gui_versions[mindsdb_lv.base_version] = gui_lv
                 if max_mindsdb_lv is None or max_mindsdb_lv < mindsdb_lv:
                     max_mindsdb_lv = mindsdb_lv
@@ -122,7 +150,11 @@ def get_last_compatible_gui_version() -> Version:
         elif current_mindsdb_lv > all_mindsdb_lv[-1]:
             gui_version_lv = max_gui_lv
         else:
-            lower_versions = {key: value for key, value in gui_versions.items() if parse_version(key) < current_mindsdb_lv}
+            lower_versions = {
+                key: value
+                for key, value in gui_versions.items()
+                if parse_version(key) < current_mindsdb_lv
+            }
             if len(lower_versions) == 0:
                 gui_version_lv = gui_versions[all_mindsdb_lv[0].base_version]
             else:
@@ -139,12 +171,12 @@ def get_last_compatible_gui_version() -> Version:
 def get_current_gui_version() -> Version:
     logger.debug("Getting current frontend version..")
     config = Config()
-    static_path = Path(config['paths']['static'])
-    version_txt_path = static_path.joinpath('version.txt')
+    static_path = Path(config["paths"]["static"])
+    version_txt_path = static_path.joinpath("version.txt")
 
     current_gui_version = None
     if version_txt_path.is_file():
-        with open(version_txt_path, 'rt') as f:
+        with open(version_txt_path, "rt") as f:
             current_gui_version = f.readline()
 
     current_gui_lv = (
@@ -160,7 +192,7 @@ def initialize_static():
     config = Config()
     last_gui_version_lv = get_last_compatible_gui_version()
     current_gui_version_lv = get_current_gui_version()
-    required_gui_version = config['gui'].get('version')
+    required_gui_version = config["gui"].get("version")
 
     if required_gui_version is not None:
         required_gui_version_lv = parse_version(required_gui_version)
@@ -176,7 +208,10 @@ def initialize_static():
             return False
 
         # ignore versions like '23.9.2.2'
-        if current_gui_version_lv is not None and len(current_gui_version_lv.release) < 3:
+        if (
+            current_gui_version_lv is not None
+            and len(current_gui_version_lv.release) < 3
+        ):
             if current_gui_version_lv == last_gui_version_lv:
                 return True
         logger.debug("Updating gui..")
@@ -187,37 +222,40 @@ def initialize_static():
 
 
 def initialize_app(config, no_studio):
-    static_root = config['paths']['static']
+    static_root = config["paths"]["static"]
     logger.debug(f"Static route: {static_root}")
-    gui_exists = Path(static_root).joinpath('index.html').is_file()
+    gui_exists = Path(static_root).joinpath("index.html").is_file()
     logger.debug(f"Does GUI already exist.. {'YES' if gui_exists else 'NO'}")
     init_static_thread = None
-    if (
-        no_studio is False
-        and (
-            config['gui']['autoupdate'] is True
-            or gui_exists is False
-        )
+    if no_studio is False and (
+        config["gui"]["autoupdate"] is True or gui_exists is False
     ):
-        init_static_thread = threading.Thread(target=initialize_static, name='initialize_static')
+        init_static_thread = threading.Thread(
+            target=initialize_static, name="initialize_static"
+        )
         init_static_thread.start()
 
+    # Wait for static initialization.
+    if not no_studio and init_static_thread is not None:
+        init_static_thread.join()
+
     app, api = initialize_flask(config, init_static_thread, no_studio)
     Compress(app)
+
     initialize_interfaces(app)
 
     if os.path.isabs(static_root) is False:
         static_root = os.path.join(os.getcwd(), static_root)
     static_root = Path(static_root)
 
-    @app.route('/', defaults={'path': ''}, methods=['GET'])
-    @app.route('/<path:path>', methods=['GET'])
+    @app.route("/", defaults={"path": ""}, methods=["GET"])
+    @app.route("/<path:path>", methods=["GET"])
     def root_index(path):
-        if path.startswith('api/'):
+        if path.startswith("api/"):
             return http_error(
                 HTTPStatus.NOT_FOUND,
-                'Not found',
-                'The endpoint you are trying to access does not exist on the server.'
+                "Not found",
+                "The endpoint you are trying to access does not exist on the server.",
             )
 
         # Normalize the path.
@@ -227,14 +265,14 @@ def initialize_app(config, no_studio):
         if not full_path.startswith(str(static_root)):
             return http_error(
                 HTTPStatus.FORBIDDEN,
-                'Forbidden',
-                'You are not allowed to access the requested resource.'
+                "Forbidden",
+                "You are not allowed to access the requested resource.",
             )
 
         if os.path.isfile(full_path):
             return send_from_directory(static_root, path)
         else:
-            return send_from_directory(static_root, 'index.html')
+            return send_from_directory(static_root, "index.html")
 
     protected_namespaces = [
         tab_ns,
@@ -253,7 +291,7 @@ def initialize_app(config, no_studio):
         skills_ns,
         agents_ns,
         jobs_ns,
-        knowledge_bases_ns
+        knowledge_bases_ns,
     ]
 
     for ns in protected_namespaces:
@@ -268,20 +306,20 @@ def initialize_app(config, no_studio):
         # pass through HTTP errors
         # NOTE flask_restx require 'message', also it modyfies 'application/problem+json' to 'application/json'
         if isinstance(e, HTTPException):
-            return ({
-                'title': e.name,
-                'detail': e.description,
-                'message': e.description
-            }, e.code, {
-                'Content-Type': 'application/problem+json'
-            })
-        return ({
-            'title': getattr(type(e), "__name__") or "Unknown error",
-            'detail': str(e),
-            'message': str(e)
-        }, 500, {
-            'Content-Type': 'application/problem+json'
-        })
+            return (
+                {"title": e.name, "detail": e.description, "message": e.description},
+                e.code,
+                {"Content-Type": "application/problem+json"},
+            )
+        return (
+            {
+                "title": getattr(type(e), "__name__") or "Unknown error",
+                "detail": str(e),
+                "message": str(e),
+            },
+            500,
+            {"Content-Type": "application/problem+json"},
+        )
 
     @app.teardown_appcontext
     def remove_session(*args, **kwargs):
@@ -295,31 +333,34 @@ def initialize_app(config, no_studio):
 
         # region routes where auth is required
         if (
-            config['auth']['http_auth_enabled'] is True
-            and any(request.path.startswith(f'/api{ns.path}') for ns in protected_namespaces)
+            config["auth"]["http_auth_enabled"] is True
+            and any(
+                request.path.startswith(f"/api{ns.path}") for ns in protected_namespaces
+            )
             and check_auth() is False
         ):
             return http_error(
-                HTTPStatus.UNAUTHORIZED, 'Unauthorized',
-                'Authorization is required to complete the request'
+                HTTPStatus.UNAUTHORIZED,
+                "Unauthorized",
+                "Authorization is required to complete the request",
             )
         # endregion
 
-        company_id = request.headers.get('company-id')
-        user_class = request.headers.get('user-class')
+        company_id = request.headers.get("company-id")
+        user_class = request.headers.get("user-class")
 
         try:
-            email_confirmed = int(request.headers.get('email-confirmed', 1))
+            email_confirmed = int(request.headers.get("email-confirmed", 1))
         except Exception:
             email_confirmed = 1
 
         try:
-            user_id = int(request.headers.get('user-id', 0))
+            user_id = int(request.headers.get("user-id", 0))
         except Exception:
             user_id = 0
 
         try:
-            session_id = request.cookies.get('session')
+            session_id = request.cookies.get("session")
         except Exception:
             session_id = "unknown"
 
@@ -349,10 +390,6 @@ def initialize_app(config, no_studio):
         ctx.user_class = user_class
         ctx.email_confirmed = email_confirmed
 
-    # Wait for static initialization.
-    if not no_studio and init_static_thread is not None:
-        init_static_thread.join()
-
     logger.debug("Done initializing app.")
     return app
 
@@ -360,13 +397,13 @@ def initialize_app(config, no_studio):
 def initialize_flask(config, init_static_thread, no_studio):
     logger.debug("Initializing flask..")
     # region required for windows https://github.com/mindsdb/mindsdb/issues/2526
-    mimetypes.add_type('text/css', '.css')
-    mimetypes.add_type('text/javascript', '.js')
+    mimetypes.add_type("text/css", ".css")
+    mimetypes.add_type("text/javascript", ".js")
     # endregion
 
     kwargs = {}
     if no_studio is not True:
-        static_path = os.path.join(config['paths']['static'], 'static/')
+        static_path = os.path.join(config["paths"]["static"], "static/")
         if os.path.isabs(static_path) is False:
             static_path = os.path.join(os.getcwd(), static_path)
         kwargs["static_url_path"] = "/static"
@@ -376,39 +413,35 @@ def initialize_flask(config, init_static_thread, no_studio):
     app = Flask(__name__, **kwargs)
     init_metrics(app)
 
-    # Instrument Flask app for OpenTelemetry
+    # Instrument Flask app and requests using either real or no-op instrumentors
     FlaskInstrumentor().instrument_app(app)
     RequestsInstrumentor().instrument()
 
-    app.config['SECRET_KEY'] = os.environ.get('FLASK_SECRET_KEY', secrets.token_hex(32))
-    app.config['SESSION_COOKIE_NAME'] = 'session'
-    app.config['PERMANENT_SESSION_LIFETIME'] = config['auth']['http_permanent_session_lifetime']
-    app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 60
-    app.config['SWAGGER_HOST'] = 'http://localhost:8000/mindsdb'
+    app.config["SECRET_KEY"] = os.environ.get("FLASK_SECRET_KEY", secrets.token_hex(32))
+    app.config["SESSION_COOKIE_NAME"] = "session"
+    app.config["PERMANENT_SESSION_LIFETIME"] = config["auth"][
+        "http_permanent_session_lifetime"
+    ]
+    app.config["SEND_FILE_MAX_AGE_DEFAULT"] = 60
+    app.config["SWAGGER_HOST"] = "http://localhost:8000/mindsdb"
     app.json = CustomJSONProvider()
 
-    authorizations = {
-        'apikey': {
-            'type': 'session',
-            'in': 'query',
-            'name': 'session'
-        }
-    }
+    authorizations = {"apikey": {"type": "session", "in": "query", "name": "session"}}
 
     logger.debug("Creating swagger API..")
     api = Swagger_Api(
         app,
         authorizations=authorizations,
-        security=['apikey'],
-        url_prefix=':8000',
-        prefix='/api',
-        doc='/doc/'
+        security=["apikey"],
+        url_prefix=":8000",
+        prefix="/api",
+        doc="/doc/",
     )
 
-    api.representations['application/json'] = custom_output_json
+    api.representations["application/json"] = custom_output_json
 
-    port = config['api']['http']['port']
-    host = config['api']['http']['host']
+    port = config["api"]["http"]["port"]
+    host = config["api"]["http"]["host"]
 
     # NOTE rewrite it, that hotfix to see GUI link
     if not no_studio:
@@ -421,9 +454,9 @@ def initialize_flask(config, init_static_thread, no_studio):
         pid = os.getpid()
         thread = threading.Thread(
             target=_open_webbrowser,
-            args=(url, pid, port, init_static_thread, config['paths']['static']),
+            args=(url, pid, port, init_static_thread, config["paths"]["static"]),
             daemon=True,
-            name='open_webbrowser'
+            name="open_webbrowser",
         )
         thread.start()
 

mindsdb/api/http/namespaces/analysis.py

@@ -77,9 +77,16 @@ class QueryAnalysis(Resource):
         if result.type != SQL_RESPONSE_TYPE.TABLE:
             return http_error(500, "Error", "Query does not return data")
 
-        column_names = [x["name"] for x in result.columns]
-        df = DataFrame(result.data, columns=column_names)
-        analysis = analyze_df(df)
+        column_names = [column.name for column in result.result_set.columns]
+        df = result.result_set.to_df()
+        try:
+            analysis = analyze_df(df)
+        except ImportError:
+            return {
+                'analysis': {},
+                'timestamp': time.time(),
+                'error': 'To use this feature, please install the "dataprep_ml" package.'
+            }
 
         query_tables = [
             table.to_string() for table in get_query_tables(ast)
@@ -88,7 +95,7 @@ class QueryAnalysis(Resource):
         return {
             "analysis": analysis,
             "column_names": column_names,
-            "row_count": len(result.data),
+            "row_count": len(result.result_set),
             "timestamp": time.time(),
             "tables": query_tables,
         }
@@ -107,6 +114,12 @@ class DataAnalysis(Resource):
         try:
             analysis = analyze_df(DataFrame(data, columns=column_names))
             return {"analysis": analysis, "timestamp": time.time()}
+        except ImportError:
+            return {
+                'analysis': {},
+                'timestamp': timestamp,
+                'error': 'To use this feature, please install the "dataprep_ml" package.'
+            }
         except Exception as e:
             # Don't want analysis exceptions to show up on UI.
             # TODO: Fix analysis so it doesn't throw exceptions at all.

mindsdb/api/http/namespaces/file.py

@@ -84,8 +84,14 @@ class File(Resource):
             parser.finalize()
             parser.close()
 
-            if file_object is not None and not file_object.closed:
-                file_object.close()
+            if file_object is not None:
+                if not file_object.closed:
+                    try:
+                        file_object.flush()
+                    except (AttributeError, ValueError, OSError):
+                        logger.debug("Failed to flush file_object before closing.", exc_info=True)
+                    file_object.close()
+                file_object = None
         else:
             data = request.json
 

mindsdb/api/http/namespaces/sql.py

@@ -8,6 +8,7 @@ import mindsdb.utilities.hooks as hooks
 import mindsdb.utilities.profiler as profiler
 from mindsdb.api.http.utils import http_error
 from mindsdb.api.http.namespaces.configs.sql import ns_conf
+from mindsdb.api.mysql.mysql_proxy.mysql_proxy import SQLAnswer
 from mindsdb.api.mysql.mysql_proxy.classes.fake_mysql_proxy import FakeMysqlProxy
 from mindsdb.api.executor.data_types.response_type import (
     RESPONSE_TYPE as SQL_RESPONSE_TYPE,
@@ -56,23 +57,8 @@ class Query(Resource):
             mysql_proxy = FakeMysqlProxy()
             mysql_proxy.set_context(context)
             try:
-                result = mysql_proxy.process_query(query)
-
-                if result.type == SQL_RESPONSE_TYPE.OK:
-                    query_response = {
-                        "type": SQL_RESPONSE_TYPE.OK,
-                        "affected_rows": result.affected_rows
-                    }
-                elif result.type == SQL_RESPONSE_TYPE.TABLE:
-                    data = result.data.to_lists(json_types=True)
-                    query_response = {
-                        "type": SQL_RESPONSE_TYPE.TABLE,
-                        "data": data,
-                        "column_names": [
-                            x["alias"] or x["name"] if "alias" in x else x["name"]
-                            for x in result.columns
-                        ],
-                    }
+                result: SQLAnswer = mysql_proxy.process_query(query)
+                query_response: dict = result.dump_http_response()
             except ExecutorException as e:
                 # classified error
                 error_type = "expected"
@@ -134,7 +120,7 @@ class ListDatabases(Resource):
         listing_query = "SHOW DATABASES"
         mysql_proxy = FakeMysqlProxy()
         try:
-            result = mysql_proxy.process_query(listing_query)
+            result: SQLAnswer = mysql_proxy.process_query(listing_query)
 
             # iterate over result.data and perform a query on each item to get the name of the tables
             if result.type == SQL_RESPONSE_TYPE.ERROR:
@@ -147,15 +133,15 @@ class ListDatabases(Resource):
                 listing_query_response = {"type": "ok"}
             elif result.type == SQL_RESPONSE_TYPE.TABLE:
                 listing_query_response = {
-                    "data": [
-                        {
-                            "name": x[0],
-                            "tables": mysql_proxy.process_query(
-                                "SHOW TABLES FROM `{}`".format(x[0])
-                            ).data,
-                        }
-                        for x in result.data
-                    ]
+                    "data": [{
+                        "name": db_row[0],
+                        "tables": [
+                            table_row[0]
+                            for table_row in mysql_proxy.process_query(
+                                "SHOW TABLES FROM `{}`".format(db_row[0])
+                            ).result_set.to_lists()
+                        ]
+                    } for db_row in result.result_set.to_lists()]
                 }
         except Exception as e:
             listing_query_response = {

mindsdb/api/http/namespaces/tree.py

@@ -94,7 +94,7 @@ class GetLeaf(Resource):
                 } for key, val in schemas.items()]
         elif db['type'] == 'system':
             system_db = ca.database_controller.get_system_db(db_name)
-            tables = system_db.get_tables()
+            tables = system_db.get_tree_tables()
             tables = [{
                 'name': table.name,
                 'class': table.kind,

mindsdb/api/http/start.py

@@ -1,3 +1,6 @@
+import gc
+gc.disable()
+
 from flask import Flask
 from waitress import serve
 
@@ -8,6 +11,8 @@ from mindsdb.utilities.config import config
 from mindsdb.utilities.functions import init_lexer_parsers
 from mindsdb.integrations.libs.ml_exec_base import process_cache
 
+gc.enable()
+
 logger = log.getLogger(__name__)
 
 
@@ -26,7 +31,7 @@ def start(verbose, no_studio, app: Flask = None):
     process_cache.init()
 
     if server_type == "waitress":
-        logger.debug("Serving HTTP app with waitress..")
+        logger.debug("Serving HTTP app with waitress...")
         serve(
             app,
             host='*' if host in ('', '0.0.0.0') else host,
@@ -34,7 +39,7 @@ def start(verbose, no_studio, app: Flask = None):
             **server_config
         )
     elif server_type == "flask":
-        logger.debug("Serving HTTP app with flask..")
+        logger.debug("Serving HTTP app with flask...")
         # that will 'disable access' log in console
 
         app.run(debug=False, port=port, host=host, **server_config)