MindsDB 25.5.3.0__py3-none-any.whl → 25.5.4.1__py3-none-any.whl

mindsdb/api/executor/sql_query/steps/fetch_dataframe_partition.py

@@ -8,7 +8,7 @@ from mindsdb.integrations.utilities.query_traversal import query_traversal
 from mindsdb.interfaces.query_context.context_controller import RunningQuery
 from mindsdb.api.executor.sql_query.result_set import ResultSet
 from mindsdb.utilities import log
-from mindsdb.utilities.config import Config
+from mindsdb.utilities.config import config
 from mindsdb.utilities.partitioning import get_max_thread_count, split_data_frame
 from mindsdb.api.executor.sql_query.steps.fetch_dataframe import get_table_alias, get_fill_param_fnc
 from mindsdb.utilities.context_executor import ContextThreadPoolExecutor
@@ -64,8 +64,6 @@ class FetchDataframePartitionCall(BaseStepCall):
         self.current_step_num = step.step_num
         self.substeps = step.steps
 
-        config = Config()
-
         # ml task queue enabled?
         use_threads, thread_count = False, None
         if config['ml_task_queue']['type'] == 'redis':
@@ -95,21 +93,8 @@ class FetchDataframePartitionCall(BaseStepCall):
         """
 
         results = []
-        while True:
-
-            # fetch batch
-            query2 = run_query.get_partition_query(self.current_step_num, query)
-            response = self.dn.query(
-                query=query2,
-                session=self.session
-            )
-            df = response.data_frame
-
-            if df is None or len(df) == 0:
-                break
 
-            # executing of sub steps can modify dataframe columns, lets memorise max tracking value
-            max_track_value = run_query.get_max_track_value(df)
+        for df in run_query.get_partitions(self.dn, self, query):
             try:
                 sub_data = self.exec_sub_steps(df)
                 results.append(sub_data)
@@ -119,8 +104,6 @@ class FetchDataframePartitionCall(BaseStepCall):
                 else:
                     raise e
 
-            run_query.set_progress(df, max_track_value)
-
         return self.concat_results(results)
 
     def concat_results(self, results: List[ResultSet]) -> ResultSet:
@@ -135,7 +118,7 @@ class FetchDataframePartitionCall(BaseStepCall):
 
         data = ResultSet()
         if len(df_list) > 0:
-            data.from_df_cols(pd.concat(df_list), col_names)
+            data = ResultSet.from_df_cols(pd.concat(df_list), col_names)
 
         return data
 
@@ -147,10 +130,7 @@ class FetchDataframePartitionCall(BaseStepCall):
         - substep are executed using result of previos step (like it is all fetched data is available)
         - the final result is returned and used outside to concatenate with results of other's batches
         """
-
-        input_data = ResultSet()
-
-        input_data.from_df(
+        input_data = ResultSet.from_df(
             df,
             table_name=self.table_alias[1],
             table_alias=self.table_alias[2],
@@ -191,22 +171,7 @@ class FetchDataframePartitionCall(BaseStepCall):
 
         with ContextThreadPoolExecutor(max_workers=thread_count) as executor:
 
-            while True:
-                # fetch batch
-                query2 = run_query.get_partition_query(self.current_step_num, query)
-                response = self.dn.query(
-                    query=query2,
-                    session=self.session
-                )
-                df = response.data_frame
-
-                if df is None or len(df) == 0:
-                    # TODO detect circles: data handler ignores condition and output is repeated
-
-                    # exit & stop workers
-                    break
-
-                max_track_value = run_query.get_max_track_value(df)
+            for df in run_query.get_partitions(self.dn, self, query):
 
                 # split into chunks and send to workers
                 futures = []
@@ -225,9 +190,5 @@ class FetchDataframePartitionCall(BaseStepCall):
                 if self.sql_query.stop_event is not None and self.sql_query.stop_event.is_set():
                     executor.shutdown()
                     raise RuntimeError('Query is interrupted')
-                # TODO
-                #  1. get next batch without updating track_value:
-                #    it allows to keep queue_in filled with data between fetching batches
-                run_query.set_progress(df, max_track_value)
 
         return self.concat_results(results)

mindsdb/api/executor/sql_query/steps/insert_step.py

@@ -1,17 +1,18 @@
 from mindsdb_sql_parser.ast import (
     Identifier,
 )
+
 from mindsdb.api.executor.planner.steps import (
     SaveToTable,
     InsertToTable,
     CreateTableStep
 )
-
 from mindsdb.api.executor.sql_query.result_set import ResultSet, Column
 from mindsdb.api.executor.exceptions import (
     NotSupportedYet,
     LogicError
 )
+from mindsdb.integrations.libs.response import INF_SCHEMA_COLUMNS_NAMES
 
 from .base import BaseStepCall
 
@@ -30,12 +31,32 @@ class InsertToTableCall(BaseStepCall):
             if step.is_replace:
                 is_replace = True
 
+        if len(step.table.parts) > 1:
+            integration_name = step.table.parts[0]
+            table_name = Identifier(parts=step.table.parts[1:])
+        else:
+            integration_name = self.context['database']
+            table_name = step.table
+
+        dn = self.session.datahub.get(integration_name)
+
+        if hasattr(dn, 'create_table') is False:
+            raise NotSupportedYet(f"Creating table in '{integration_name}' is not supported")
+
         if step.dataframe is not None:
             data = self.steps_data[step.dataframe.result.step_num]
         elif step.query is not None:
             data = ResultSet()
-            for col in step.query.columns:
-                data.add_column(Column(col.name))
+            if step.query.columns is None:
+                # Is query like: INSERT INTO table VALUES (...)
+                table_columns_df = dn.get_table_columns_df(str(table_name))
+                columns_names = table_columns_df[INF_SCHEMA_COLUMNS_NAMES.COLUMN_NAME].to_list()
+                for column_name in columns_names:
+                    data.add_column(Column(name=column_name))
+            else:
+                # Is query like: INSERT INTO table (column_name, ...) VALUES (...)
+                for col in step.query.columns:
+                    data.add_column(Column(name=col.name))
 
             records = []
             for row in step.query.values:
@@ -53,18 +74,6 @@ class InsertToTableCall(BaseStepCall):
         else:
             raise LogicError(f'Data not found for insert: {step}')
 
-        if len(step.table.parts) > 1:
-            integration_name = step.table.parts[0]
-            table_name = Identifier(parts=step.table.parts[1:])
-        else:
-            integration_name = self.context['database']
-            table_name = step.table
-
-        dn = self.session.datahub.get(integration_name)
-
-        if hasattr(dn, 'create_table') is False:
-            raise NotSupportedYet(f"Creating table in '{integration_name}' is not supported")
-
         #  del 'service' columns
         for col in data.find_columns('__mindsdb_row_id'):
             data.del_column(col)

mindsdb/api/executor/sql_query/steps/join_step.py

@@ -101,7 +101,7 @@ class JoinStepCall(BaseStepCall):
         resp_df.replace({np.nan: None}, inplace=True)
 
         names_a.update(names_b)
-        data = ResultSet().from_df_cols(resp_df, col_names=names_a)
+        data = ResultSet.from_df_cols(df=resp_df, columns_dict=names_a)
 
         for col in data.find_columns('__mindsdb_row_id'):
             data.del_column(col)

mindsdb/api/executor/sql_query/steps/project_step.py

@@ -83,4 +83,4 @@ class ProjectStepCall(BaseStepCall):
 
         res = query_df(df, query, session=self.session)
 
-        return ResultSet().from_df_cols(res, col_names, strict=False)
+        return ResultSet.from_df_cols(df=res, columns_dict=col_names, strict=False)

mindsdb/api/executor/sql_query/steps/sql_steps.py

@@ -38,4 +38,4 @@ class DataStepCall(BaseStepCall):
     def call(self, step):
         # create resultset
         df = pd.DataFrame(step.data)
-        return ResultSet().from_df(df, database='', table_name='')
+        return ResultSet.from_df(df, database='', table_name='')

mindsdb/api/executor/sql_query/steps/subselect_step.py

@@ -55,7 +55,7 @@ class SubSelectStepCall(BaseStepCall):
 
             for col_name in query_cols:
                 if col_name not in result_cols:
-                    result.add_column(Column(col_name))
+                    result.add_column(Column(name=col_name))
 
         # inject previous step values
         if isinstance(query, Select):
@@ -69,12 +69,10 @@ class SubSelectStepCall(BaseStepCall):
         df = result.to_df()
         res = query_df(df, query, session=self.session)
 
-        result2 = ResultSet()
         # get database from first column
         database = result.columns[0].database
-        result2.from_df(res, database, table_name)
 
-        return result2
+        return ResultSet.from_df(res, database, table_name)
 
 
 class QueryStepCall(BaseStepCall):
@@ -86,7 +84,7 @@ class QueryStepCall(BaseStepCall):
 
         if step.from_table is not None:
             if isinstance(step.from_table, pd.DataFrame):
-                result_set = ResultSet().from_df(step.from_table)
+                result_set = ResultSet.from_df(step.from_table)
             else:
                 result_set = self.steps_data[step.from_table.step_num]
         else:
@@ -208,6 +206,4 @@ class QueryStepCall(BaseStepCall):
         query.from_table = Identifier('df_table')
         res = query_df(df, query, session=self.session)
 
-        data = ResultSet().from_df_cols(res, col_names, strict=False)
-
-        return data
+        return ResultSet.from_df_cols(df=res, columns_dict=col_names, strict=False)

mindsdb/api/executor/sql_query/steps/union_step.py

@@ -48,6 +48,4 @@ class UnionStepCall(BaseStepCall):
         })
         resp_df.replace({np.nan: None}, inplace=True)
 
-        data = ResultSet().from_df_cols(resp_df, col_names=names)
-
-        return data
+        return ResultSet.from_df_cols(df=resp_df, columns_dict=names)

mindsdb/api/http/initialize.py

@@ -4,9 +4,11 @@ import mimetypes
 import threading
 import traceback
 import webbrowser
+
 from pathlib import Path
 from http import HTTPStatus
 
+
 import requests
 from flask import Flask, url_for, make_response, request, send_from_directory
 from flask.json import dumps
@@ -68,7 +70,9 @@ try:
     from opentelemetry.instrumentation.flask import FlaskInstrumentor
     from opentelemetry.instrumentation.requests import RequestsInstrumentor
 except ImportError:
-    logger.debug("OpenTelemetry is not avaiable. Please run `pip install -r requirements/requirements-opentelemetry.txt` to use it.")
+    logger.debug(
+        "OpenTelemetry is not avaiable. Please run `pip install -r requirements/requirements-opentelemetry.txt` to use it."
+    )
     FlaskInstrumentor = _NoOpFlaskInstrumentor
     RequestsInstrumentor = _NoOpRequestsInstrumentor
 
@@ -78,6 +82,7 @@ class Swagger_Api(Api):
     This is a modification of the base Flask Restplus Api class due to the issue described here
     https://github.com/noirbizarre/flask-restplus/issues/223
     """
+
     @property
     def specs_url(self):
         return url_for(self.endpoint("specs"), _external=False)
@@ -92,7 +97,10 @@ def custom_output_json(data, code, headers=None):
 def get_last_compatible_gui_version() -> Version:
     logger.debug("Getting last compatible frontend..")
     try:
-        res = requests.get('https://mindsdb-web-builds.s3.amazonaws.com/compatible-config.json', timeout=5)
+        res = requests.get(
+            "https://mindsdb-web-builds.s3.amazonaws.com/compatible-config.json",
+            timeout=5,
+        )
     except (ConnectionError, requests.exceptions.ConnectionError) as e:
         logger.error(f"Is no connection. {e}")
         return False
@@ -118,13 +126,16 @@ def get_last_compatible_gui_version() -> Version:
         gui_versions = {}
         max_mindsdb_lv = None
         max_gui_lv = None
-        for el in versions['mindsdb']:
-            if el['mindsdb_version'] is None:
-                gui_lv = parse_version(el['gui_version'])
+        for el in versions["mindsdb"]:
+            if el["mindsdb_version"] is None:
+                gui_lv = parse_version(el["gui_version"])
             else:
-                mindsdb_lv = parse_version(el['mindsdb_version'])
-                gui_lv = parse_version(el['gui_version'])
-                if mindsdb_lv.base_version not in gui_versions or gui_lv > gui_versions[mindsdb_lv.base_version]:
+                mindsdb_lv = parse_version(el["mindsdb_version"])
+                gui_lv = parse_version(el["gui_version"])
+                if (
+                    mindsdb_lv.base_version not in gui_versions
+                    or gui_lv > gui_versions[mindsdb_lv.base_version]
+                ):
                     gui_versions[mindsdb_lv.base_version] = gui_lv
                 if max_mindsdb_lv is None or max_mindsdb_lv < mindsdb_lv:
                     max_mindsdb_lv = mindsdb_lv
@@ -139,7 +150,11 @@ def get_last_compatible_gui_version() -> Version:
         elif current_mindsdb_lv > all_mindsdb_lv[-1]:
             gui_version_lv = max_gui_lv
         else:
-            lower_versions = {key: value for key, value in gui_versions.items() if parse_version(key) < current_mindsdb_lv}
+            lower_versions = {
+                key: value
+                for key, value in gui_versions.items()
+                if parse_version(key) < current_mindsdb_lv
+            }
             if len(lower_versions) == 0:
                 gui_version_lv = gui_versions[all_mindsdb_lv[0].base_version]
             else:
@@ -156,12 +171,12 @@ def get_last_compatible_gui_version() -> Version:
 def get_current_gui_version() -> Version:
     logger.debug("Getting current frontend version..")
     config = Config()
-    static_path = Path(config['paths']['static'])
-    version_txt_path = static_path.joinpath('version.txt')
+    static_path = Path(config["paths"]["static"])
+    version_txt_path = static_path.joinpath("version.txt")
 
     current_gui_version = None
     if version_txt_path.is_file():
-        with open(version_txt_path, 'rt') as f:
+        with open(version_txt_path, "rt") as f:
             current_gui_version = f.readline()
 
     current_gui_lv = (
@@ -177,7 +192,7 @@ def initialize_static():
     config = Config()
     last_gui_version_lv = get_last_compatible_gui_version()
     current_gui_version_lv = get_current_gui_version()
-    required_gui_version = config['gui'].get('version')
+    required_gui_version = config["gui"].get("version")
 
     if required_gui_version is not None:
         required_gui_version_lv = parse_version(required_gui_version)
@@ -193,7 +208,10 @@ def initialize_static():
             return False
 
         # ignore versions like '23.9.2.2'
-        if current_gui_version_lv is not None and len(current_gui_version_lv.release) < 3:
+        if (
+            current_gui_version_lv is not None
+            and len(current_gui_version_lv.release) < 3
+        ):
             if current_gui_version_lv == last_gui_version_lv:
                 return True
         logger.debug("Updating gui..")
@@ -204,37 +222,40 @@ def initialize_static():
 
 
 def initialize_app(config, no_studio):
-    static_root = config['paths']['static']
+    static_root = config["paths"]["static"]
     logger.debug(f"Static route: {static_root}")
-    gui_exists = Path(static_root).joinpath('index.html').is_file()
+    gui_exists = Path(static_root).joinpath("index.html").is_file()
     logger.debug(f"Does GUI already exist.. {'YES' if gui_exists else 'NO'}")
     init_static_thread = None
-    if (
-        no_studio is False
-        and (
-            config['gui']['autoupdate'] is True
-            or gui_exists is False
-        )
+    if no_studio is False and (
+        config["gui"]["autoupdate"] is True or gui_exists is False
     ):
-        init_static_thread = threading.Thread(target=initialize_static, name='initialize_static')
+        init_static_thread = threading.Thread(
+            target=initialize_static, name="initialize_static"
+        )
         init_static_thread.start()
 
+    # Wait for static initialization.
+    if not no_studio and init_static_thread is not None:
+        init_static_thread.join()
+
     app, api = initialize_flask(config, init_static_thread, no_studio)
     Compress(app)
+
     initialize_interfaces(app)
 
     if os.path.isabs(static_root) is False:
         static_root = os.path.join(os.getcwd(), static_root)
     static_root = Path(static_root)
 
-    @app.route('/', defaults={'path': ''}, methods=['GET'])
-    @app.route('/<path:path>', methods=['GET'])
+    @app.route("/", defaults={"path": ""}, methods=["GET"])
+    @app.route("/<path:path>", methods=["GET"])
     def root_index(path):
-        if path.startswith('api/'):
+        if path.startswith("api/"):
             return http_error(
                 HTTPStatus.NOT_FOUND,
-                'Not found',
-                'The endpoint you are trying to access does not exist on the server.'
+                "Not found",
+                "The endpoint you are trying to access does not exist on the server.",
             )
 
         # Normalize the path.
@@ -244,14 +265,14 @@ def initialize_app(config, no_studio):
         if not full_path.startswith(str(static_root)):
             return http_error(
                 HTTPStatus.FORBIDDEN,
-                'Forbidden',
-                'You are not allowed to access the requested resource.'
+                "Forbidden",
+                "You are not allowed to access the requested resource.",
             )
 
         if os.path.isfile(full_path):
             return send_from_directory(static_root, path)
         else:
-            return send_from_directory(static_root, 'index.html')
+            return send_from_directory(static_root, "index.html")
 
     protected_namespaces = [
         tab_ns,
@@ -270,7 +291,7 @@ def initialize_app(config, no_studio):
         skills_ns,
         agents_ns,
         jobs_ns,
-        knowledge_bases_ns
+        knowledge_bases_ns,
     ]
 
     for ns in protected_namespaces:
@@ -285,20 +306,20 @@ def initialize_app(config, no_studio):
         # pass through HTTP errors
         # NOTE flask_restx require 'message', also it modyfies 'application/problem+json' to 'application/json'
         if isinstance(e, HTTPException):
-            return ({
-                'title': e.name,
-                'detail': e.description,
-                'message': e.description
-            }, e.code, {
-                'Content-Type': 'application/problem+json'
-            })
-        return ({
-            'title': getattr(type(e), "__name__") or "Unknown error",
-            'detail': str(e),
-            'message': str(e)
-        }, 500, {
-            'Content-Type': 'application/problem+json'
-        })
+            return (
+                {"title": e.name, "detail": e.description, "message": e.description},
+                e.code,
+                {"Content-Type": "application/problem+json"},
+            )
+        return (
+            {
+                "title": getattr(type(e), "__name__") or "Unknown error",
+                "detail": str(e),
+                "message": str(e),
+            },
+            500,
+            {"Content-Type": "application/problem+json"},
+        )
 
     @app.teardown_appcontext
     def remove_session(*args, **kwargs):
@@ -312,31 +333,34 @@ def initialize_app(config, no_studio):
 
         # region routes where auth is required
         if (
-            config['auth']['http_auth_enabled'] is True
-            and any(request.path.startswith(f'/api{ns.path}') for ns in protected_namespaces)
+            config["auth"]["http_auth_enabled"] is True
+            and any(
+                request.path.startswith(f"/api{ns.path}") for ns in protected_namespaces
+            )
             and check_auth() is False
         ):
             return http_error(
-                HTTPStatus.UNAUTHORIZED, 'Unauthorized',
-                'Authorization is required to complete the request'
+                HTTPStatus.UNAUTHORIZED,
+                "Unauthorized",
+                "Authorization is required to complete the request",
             )
         # endregion
 
-        company_id = request.headers.get('company-id')
-        user_class = request.headers.get('user-class')
+        company_id = request.headers.get("company-id")
+        user_class = request.headers.get("user-class")
 
         try:
-            email_confirmed = int(request.headers.get('email-confirmed', 1))
+            email_confirmed = int(request.headers.get("email-confirmed", 1))
         except Exception:
             email_confirmed = 1
 
         try:
-            user_id = int(request.headers.get('user-id', 0))
+            user_id = int(request.headers.get("user-id", 0))
         except Exception:
             user_id = 0
 
         try:
-            session_id = request.cookies.get('session')
+            session_id = request.cookies.get("session")
         except Exception:
             session_id = "unknown"
 
@@ -366,10 +390,6 @@ def initialize_app(config, no_studio):
         ctx.user_class = user_class
         ctx.email_confirmed = email_confirmed
 
-    # Wait for static initialization.
-    if not no_studio and init_static_thread is not None:
-        init_static_thread.join()
-
     logger.debug("Done initializing app.")
     return app
 
@@ -377,13 +397,13 @@ def initialize_app(config, no_studio):
 def initialize_flask(config, init_static_thread, no_studio):
     logger.debug("Initializing flask..")
     # region required for windows https://github.com/mindsdb/mindsdb/issues/2526
-    mimetypes.add_type('text/css', '.css')
-    mimetypes.add_type('text/javascript', '.js')
+    mimetypes.add_type("text/css", ".css")
+    mimetypes.add_type("text/javascript", ".js")
     # endregion
 
     kwargs = {}
     if no_studio is not True:
-        static_path = os.path.join(config['paths']['static'], 'static/')
+        static_path = os.path.join(config["paths"]["static"], "static/")
         if os.path.isabs(static_path) is False:
             static_path = os.path.join(os.getcwd(), static_path)
         kwargs["static_url_path"] = "/static"
@@ -397,35 +417,31 @@ def initialize_flask(config, init_static_thread, no_studio):
     FlaskInstrumentor().instrument_app(app)
     RequestsInstrumentor().instrument()
 
-    app.config['SECRET_KEY'] = os.environ.get('FLASK_SECRET_KEY', secrets.token_hex(32))
-    app.config['SESSION_COOKIE_NAME'] = 'session'
-    app.config['PERMANENT_SESSION_LIFETIME'] = config['auth']['http_permanent_session_lifetime']
-    app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 60
-    app.config['SWAGGER_HOST'] = 'http://localhost:8000/mindsdb'
+    app.config["SECRET_KEY"] = os.environ.get("FLASK_SECRET_KEY", secrets.token_hex(32))
+    app.config["SESSION_COOKIE_NAME"] = "session"
+    app.config["PERMANENT_SESSION_LIFETIME"] = config["auth"][
+        "http_permanent_session_lifetime"
+    ]
+    app.config["SEND_FILE_MAX_AGE_DEFAULT"] = 60
+    app.config["SWAGGER_HOST"] = "http://localhost:8000/mindsdb"
     app.json = CustomJSONProvider()
 
-    authorizations = {
-        'apikey': {
-            'type': 'session',
-            'in': 'query',
-            'name': 'session'
-        }
-    }
+    authorizations = {"apikey": {"type": "session", "in": "query", "name": "session"}}
 
     logger.debug("Creating swagger API..")
     api = Swagger_Api(
         app,
         authorizations=authorizations,
-        security=['apikey'],
-        url_prefix=':8000',
-        prefix='/api',
-        doc='/doc/'
+        security=["apikey"],
+        url_prefix=":8000",
+        prefix="/api",
+        doc="/doc/",
     )
 
-    api.representations['application/json'] = custom_output_json
+    api.representations["application/json"] = custom_output_json
 
-    port = config['api']['http']['port']
-    host = config['api']['http']['host']
+    port = config["api"]["http"]["port"]
+    host = config["api"]["http"]["host"]
 
     # NOTE rewrite it, that hotfix to see GUI link
     if not no_studio:
@@ -438,9 +454,9 @@ def initialize_flask(config, init_static_thread, no_studio):
         pid = os.getpid()
         thread = threading.Thread(
             target=_open_webbrowser,
-            args=(url, pid, port, init_static_thread, config['paths']['static']),
+            args=(url, pid, port, init_static_thread, config["paths"]["static"]),
             daemon=True,
-            name='open_webbrowser'
+            name="open_webbrowser",
         )
         thread.start()
 

mindsdb/api/http/namespaces/analysis.py

@@ -77,8 +77,8 @@ class QueryAnalysis(Resource):
         if result.type != SQL_RESPONSE_TYPE.TABLE:
             return http_error(500, "Error", "Query does not return data")
 
-        column_names = [x["name"] for x in result.columns]
-        df = DataFrame(result.data, columns=column_names)
+        column_names = [column.name for column in result.result_set.columns]
+        df = result.result_set.to_df()
         try:
             analysis = analyze_df(df)
         except ImportError:
@@ -95,7 +95,7 @@ class QueryAnalysis(Resource):
         return {
             "analysis": analysis,
             "column_names": column_names,
-            "row_count": len(result.data),
+            "row_count": len(result.result_set),
             "timestamp": time.time(),
             "tables": query_tables,
         }