MindsDB 25.4.1.0__py3-none-any.whl → 25.4.2.0__py3-none-any.whl

mindsdb/api/executor/sql_query/sql_query.py

@@ -8,9 +8,9 @@
  * permission of MindsDB Inc
  *******************************************************
 """
-import re
 import inspect
 from textwrap import dedent
+from typing import Dict
 
 from mindsdb_sql_parser import parse_sql
 from mindsdb.api.executor.planner.steps import (
@@ -23,7 +23,7 @@ from mindsdb.api.executor.planner.exceptions import PlanningException
 from mindsdb.utilities.render.sqlalchemy_render import SqlalchemyRender
 from mindsdb.api.executor.planner import query_planner
 
-from mindsdb.api.executor.utilities.sql import query_df, get_query_models
+from mindsdb.api.executor.utilities.sql import get_query_models
 from mindsdb.interfaces.model.functions import get_model_record
 from mindsdb.api.executor.exceptions import (
     BadTableError,
@@ -38,8 +38,6 @@ from . import steps
 from .result_set import ResultSet, Column
 from . steps.base import BaseStepCall
 
-superset_subquery = re.compile(r'from[\s\n]*(\(.*\))[\s\n]*as[\s\n]*virtual_table', flags=re.IGNORECASE | re.MULTILINE | re.S)
-
 
 class SQLQuery:
 
@@ -59,23 +57,13 @@ class SQLQuery:
         }
 
         self.columns_list = None
-        self.steps_data = {}
+        self.steps_data: Dict[int, ResultSet] = {}
 
-        self.planner = None
+        self.planner: query_planner.QueryPlanner = None
         self.parameters = []
-        self.fetched_data = None
-
-        self.outer_query = None
+        self.fetched_data: ResultSet = None
 
         if isinstance(sql, str):
-            # region workaround for subqueries in superset
-            if 'as virtual_table' in sql.lower():
-                subquery = re.findall(superset_subquery, sql)
-                if isinstance(subquery, list) and len(subquery) == 1:
-                    subquery = subquery[0]
-                    self.outer_query = sql.replace(subquery, 'dataframe')
-                    sql = subquery.strip('()')
-            # endregion
             self.query = parse_sql(sql)
             self.context['query_str'] = sql
         else:
@@ -89,7 +77,6 @@ class SQLQuery:
         self.create_planner()
 
         if execute:
-            self.prepare_query(prepare=False)
             self.execute_query()
 
     @classmethod
@@ -190,55 +177,42 @@ class SQLQuery:
             default_namespace=database,
         )
 
-    def fetch(self, view='result_set'):
-        data = self.fetched_data
-
-        if view == 'dataframe':
-            result = data.to_df()
-        else:
-            result = data
+    def prepare_query(self):
+        """it is prepared statement call
+        """
+        try:
+            for step in self.planner.prepare_steps(self.query):
+                data = self.execute_step(step)
+                step.set_result(data)
+                self.steps_data[step.step_num] = data
+        except PlanningException as e:
+            raise LogicError(e)
 
-        return {
-            'success': True,
-            'result': result
-        }
+        statement_info = self.planner.get_statement_info()
 
-    def prepare_query(self, prepare=True):
-        if prepare:
-            # it is prepared statement call
-            try:
-                for step in self.planner.prepare_steps(self.query):
-                    data = self.execute_step(step)
-                    step.set_result(data)
-                    self.steps_data[step.step_num] = data
-            except PlanningException as e:
-                raise LogicError(e)
-
-            statement_info = self.planner.get_statement_info()
-
-            self.columns_list = []
-            for col in statement_info['columns']:
-                self.columns_list.append(
-                    Column(
-                        database=col['ds'],
-                        table_name=col['table_name'],
-                        table_alias=col['table_alias'],
-                        name=col['name'],
-                        alias=col['alias'],
-                        type=col['type']
-                    )
-                )
-
-            self.parameters = [
+        self.columns_list = []
+        for col in statement_info['columns']:
+            self.columns_list.append(
                 Column(
+                    database=col['ds'],
+                    table_name=col['table_name'],
+                    table_alias=col['table_alias'],
                     name=col['name'],
                     alias=col['alias'],
                     type=col['type']
                 )
-                for col in statement_info['parameters']
-            ]
-
-    def execute_query(self, params=None):
+            )
+
+        self.parameters = [
+            Column(
+                name=col['name'],
+                alias=col['alias'],
+                type=col['type']
+            )
+            for col in statement_info['parameters']
+        ]
+
+    def execute_query(self):
         if self.fetched_data is not None:
             # no need to execute
             return
@@ -246,7 +220,7 @@ class SQLQuery:
         step_result = None
         process_mark = None
         try:
-            steps = list(self.planner.execute_steps(params))
+            steps = list(self.planner.execute_steps())
             steps_classes = (x.__class__ for x in steps)
             predict_steps = (ApplyPredictorRowStep, ApplyPredictorStep, ApplyTimeseriesPredictorStep)
             if any(s in predict_steps for s in steps_classes):
@@ -270,27 +244,7 @@ class SQLQuery:
         if len(self.steps_data) == 0:
             return
 
-        try:
-            if self.outer_query is not None:
-                # workaround for subqueries in superset. remove it?
-                # +++
-                # ???
-
-                result = step_result
-                df = result.to_df()
-
-                df2 = query_df(df, self.outer_query)
-
-                result2 = ResultSet().from_df(df2, database='', table_name='')
-
-                self.columns_list = result2.columns
-                self.fetched_data = result2
-
-            else:
-                result = step_result
-                self.fetched_data = result
-        except Exception as e:
-            raise UnknownError("error in preparing result query step") from e
+        self.fetched_data = step_result
 
         try:
             if hasattr(self, 'columns_list') is False:

mindsdb/api/executor/sql_query/steps/delete_step.py

@@ -44,6 +44,5 @@ class DeleteStepCall(BaseStepCall):
 
         query_traversal(query.where, fill_params)
 
-        dn.query(query=query, session=self.session)
-
-        return ResultSet()
+        response = dn.query(query=query, session=self.session)
+        return ResultSet(affected_rows=response.affected_rows)

mindsdb/api/executor/sql_query/steps/fetch_dataframe.py

@@ -89,10 +89,11 @@ class FetchDataframeStepCall(BaseStepCall):
             table_alias = (self.context.get('database'), 'result', 'result')
 
             # fetch raw_query
-            df, columns_info = dn.query(
+            response = dn.query(
                 native_query=step.raw_query,
                 session=self.session
             )
+            df = response.data_frame
         else:
             table_alias = get_table_alias(step.query.from_table, self.context.get('database'))
 
@@ -104,13 +105,14 @@ class FetchDataframeStepCall(BaseStepCall):
 
             query, context_callback = query_context_controller.handle_db_context_vars(query, dn, self.session)
 
-            df, columns_info = dn.query(
+            response = dn.query(
                 query=query,
                 session=self.session
             )
+            df = response.data_frame
 
             if context_callback:
-                context_callback(df, columns_info)
+                context_callback(df, response.columns)
 
         result = ResultSet()
 

mindsdb/api/executor/sql_query/steps/insert_step.py

@@ -91,13 +91,13 @@ class InsertToTableCall(BaseStepCall):
             else:
                 col_names.add(col.alias)
 
-        dn.create_table(
+        response = dn.create_table(
             table_name=table_name,
             result_set=data,
             is_replace=is_replace,
             is_create=is_create
         )
-        return ResultSet()
+        return ResultSet(affected_rows=response.affected_rows)
 
 
 class SaveToTableCall(InsertToTableCall):

mindsdb/api/executor/sql_query/steps/prepare_steps.py

@@ -47,10 +47,10 @@ class GetTableColumnsCall(BaseStepCall):
         dn = self.session.datahub.get(step.namespace)
         ds_query = Select(from_table=Identifier(table), targets=[Star()], limit=Constant(0))
 
-        data, columns_info = dn.query(ds_query, session=self.session)
+        response = dn.query(ds_query, session=self.session)
 
         data = ResultSet()
-        for column in columns_info:
+        for column in response.columns:
             data.add_column(Column(
                 name=column['name'],
                 type=column.get('type'),

mindsdb/api/executor/sql_query/steps/subselect_step.py

@@ -3,13 +3,7 @@ from collections import defaultdict
 import pandas as pd
 
 from mindsdb_sql_parser.ast import (
-    Identifier,
-    Select,
-    Star,
-    Constant,
-    Parameter,
-    Function,
-    Variable
+    Identifier, Select, Star, Constant, Parameter, Function, Variable, BinaryOperation
 )
 
 from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import SERVER_VARIABLES
@@ -87,7 +81,7 @@ class QueryStepCall(BaseStepCall):
 
     bind = QueryStep
 
-    def call(self, step):
+    def call(self, step: QueryStep):
         query = step.query
 
         if step.from_table is not None:
@@ -190,6 +184,24 @@ class QueryStepCall(BaseStepCall):
         fill_params = get_fill_param_fnc(self.steps_data)
         query_traversal(query, fill_params)
 
+        if not step.strict_where:
+            # remove conditions with not-existed columns.
+            #   these conditions can be already used as input to model or knowledge base
+            #   but can be absent in their output
+
+            def remove_not_used_conditions(node, **kwargs):
+                # find last in where
+                if isinstance(node, BinaryOperation):
+                    for arg in node.args:
+                        if isinstance(arg, Identifier) and len(arg.parts) > 1:
+                            key = tuple(arg.parts[-2:])
+                            if key not in col_idx:
+                                # exclude
+                                node.args = [Constant(0), Constant(0)]
+                                node.op = '='
+
+            query_traversal(query.where, remove_not_used_conditions)
+
         query_traversal(query, check_fields)
         query.where = query_context_controller.remove_lasts(query.where)
 

mindsdb/api/executor/sql_query/steps/update_step.py

@@ -18,8 +18,6 @@ class UpdateToTableCall(BaseStepCall):
     bind = UpdateToTable
 
     def call(self, step):
-        data = ResultSet()
-
         if len(step.table.parts) > 1:
             integration_name = step.table.parts[0]
             table_name_parts = step.table.parts[1:]
@@ -85,8 +83,8 @@ class UpdateToTableCall(BaseStepCall):
 
             if result_step is None:
                 # run as is
-                dn.query(query=update_query, session=self.session)
-                return data
+                response = dn.query(query=update_query, session=self.session)
+                return ResultSet(affected_rows=response.affected_rows)
             result_data = self.steps_data[result_step.result.step_num]
 
             # link nodes with parameters for fast replacing with values
@@ -125,5 +123,5 @@ class UpdateToTableCall(BaseStepCall):
             for param_name, param in params_map_index:
                 param.value = row[param_name]
 
-            dn.query(query=update_query, session=self.session)
-        return data
+            response = dn.query(query=update_query, session=self.session)
+        return ResultSet(affected_rows=response.affected_rows)

mindsdb/api/http/namespaces/sql.py

@@ -59,7 +59,10 @@ class Query(Resource):
                 result = mysql_proxy.process_query(query)
 
                 if result.type == SQL_RESPONSE_TYPE.OK:
-                    query_response = {"type": SQL_RESPONSE_TYPE.OK}
+                    query_response = {
+                        "type": SQL_RESPONSE_TYPE.OK,
+                        "affected_rows": result.affected_rows
+                    }
                 elif result.type == SQL_RESPONSE_TYPE.TABLE:
                     data = result.data.to_lists(json_types=True)
                     query_response = {

mindsdb/api/mysql/mysql_proxy/data_types/mysql_packets/ok_packet.py

@@ -40,7 +40,7 @@ class OkPacket(Packet):
     def setup(self):
         eof = self._kwargs.get('eof', False)
         self.ok_header = Datum('int<1>', 0xFE if eof is True else 0)
-        self.affected_rows = Datum('int<lenenc>', self._kwargs.get('affected_rows', 0))
+        self.affected_rows = Datum('int<lenenc>', self._kwargs.get('affected_rows') or 0)
         self.last_insert_id = Datum('int<lenenc>', 0)
         status = self._kwargs.get('status', 0x0002)
         self.server_status = Datum('int<2>', status)

mindsdb/api/mysql/mysql_proxy/executor/mysql_executor.py

@@ -4,6 +4,7 @@ from mindsdb.api.executor.planner import utils as planner_utils
 import mindsdb.utilities.profiler as profiler
 from mindsdb.api.executor.sql_query.result_set import Column
 from mindsdb.api.executor.sql_query import SQLQuery
+from mindsdb.api.executor.data_types.answer import ExecuteAnswer
 from mindsdb.api.executor.command_executor import ExecuteCommands
 from mindsdb.api.mysql.mysql_proxy.utilities import ErSqlSyntaxError
 from mindsdb.utilities import log
@@ -12,37 +13,20 @@ logger = log.getLogger(__name__)
 
 
 class Executor:
-    """This class stores initial and intermediate params
-    between different steps of query execution. And it is also
-    creates a separate instance of ExecuteCommands to execute the current
-    query step.
-
-    IMPORTANT: A public API of this class is a contract.
-    And there are at least 2 classes strongly depend on it:
-        ExecuctorClient
-        ExecutorService.
-    These classes do the same work as Executor when
-    MindsDB works in 'modularity' mode.
-    Thus please make sure that IF you change the API,
-    you must update the API of these two classes as well!"""
-
     def __init__(self, session, sqlserver):
         self.session = session
         self.sqlserver = sqlserver
 
         self.query = None
 
-        # returned values
-        # all this attributes needs to be added in
-        # self.json() method
         self.columns = []
         self.params = []
         self.data = None
-        self.state_track = None
         self.server_status = None
         self.is_executed = False
         self.error_message = None
         self.error_code = None
+        self.executor_answer: ExecuteAnswer = None
 
         self.sql = ""
         self.sql_lower = ""
@@ -126,14 +110,7 @@ class Executor:
         if self.is_executed:
             return
 
-        ret = self.command_executor.execute_command(self.query)
-        self.error_code = ret.error_code
-        self.error_message = ret.error_message
+        executor_answer: ExecuteAnswer = self.command_executor.execute_command(self.query)
+        self.executor_answer = executor_answer
 
         self.is_executed = True
-
-        if ret.data is not None:
-            self.data = ret.data
-            self.columns = ret.data.columns
-
-        self.state_track = ret.state_track

mindsdb/api/mysql/mysql_proxy/libs/constants/mysql.py

@@ -94,6 +94,7 @@ class COMMANDS(object):
     COM_STMT_PREPARE = int('0x16', 0)
     COM_STMT_EXECUTE = int('0x17', 0)
     COM_STMT_FETCH = int('0x1c', 0)
+    COM_STMT_RESET = int('0x1a', 0)
     COM_STMT_CLOSE = int('0x19', 0)
     COM_FIELD_LIST = int('0x04', 0)  # deprecated
 

mindsdb/api/mysql/mysql_proxy/mysql_proxy.py

@@ -21,7 +21,8 @@ import sys
 import tempfile
 import traceback
 from functools import partial
-from typing import Dict, List
+from typing import Dict, List, Optional
+from dataclasses import dataclass
 
 from numpy import dtype as np_dtype
 from pandas.api import types as pd_types
@@ -71,6 +72,7 @@ from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import (
     TYPES,
     getConstName,
 )
+from mindsdb.api.executor.data_types.answer import ExecuteAnswer
 from mindsdb.api.executor.data_types.response_type import RESPONSE_TYPE
 from mindsdb.api.mysql.mysql_proxy.utilities import (
     ErWrongCharset,
@@ -93,24 +95,16 @@ def empty_fn():
     pass
 
 
+@dataclass
 class SQLAnswer:
-    def __init__(
-        self,
-        resp_type: RESPONSE_TYPE,
-        columns: List[Dict] = None,
-        data: List[Dict] = None,
-        status: int = None,
-        state_track: List[List] = None,
-        error_code: int = None,
-        error_message: str = None,
-    ):
-        self.resp_type = resp_type
-        self.columns = columns
-        self.data = data
-        self.status = status
-        self.state_track = state_track
-        self.error_code = error_code
-        self.error_message = error_message
+    resp_type: RESPONSE_TYPE = RESPONSE_TYPE.OK
+    columns: Optional[List[Dict]] = None
+    data: Optional[List[Dict]] = None   # resultSet ?
+    status: Optional[int] = None
+    state_track: Optional[List[List]] = None
+    error_code: Optional[int] = None
+    error_message: Optional[str] = None
+    affected_rows: Optional[int] = None
 
     @property
     def type(self):
@@ -333,7 +327,7 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
                 packages.append(self.last_packet())
             self.send_package_group(packages)
         elif answer.type == RESPONSE_TYPE.OK:
-            self.packet(OkPacket, state_track=answer.state_track).send()
+            self.packet(OkPacket, state_track=answer.state_track, affected_rows=answer.affected_rows).send()
         elif answer.type == RESPONSE_TYPE.ERROR:
             self.packet(
                 ErrPacket, err_code=answer.error_code, msg=answer.error_message
@@ -546,21 +540,23 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
     @profiler.profile()
     def process_query(self, sql):
         executor = Executor(session=self.session, sqlserver=self)
-
         executor.query_execute(sql)
+        executor_answer = executor.executor_answer
 
-        if executor.data is None:
+        if executor_answer.data is None:
             resp = SQLAnswer(
                 resp_type=RESPONSE_TYPE.OK,
-                state_track=executor.state_track,
+                state_track=executor_answer.state_track,
+                affected_rows=executor_answer.affected_rows
             )
         else:
             resp = SQLAnswer(
                 resp_type=RESPONSE_TYPE.TABLE,
-                state_track=executor.state_track,
-                columns=self.to_mysql_columns(executor.columns),
-                data=executor.data,
+                state_track=executor_answer.state_track,
+                columns=self.to_mysql_columns(executor_answer.data.columns),
+                data=executor_answer.data,
                 status=executor.server_status,
+                affected_rows=executor_answer.affected_rows
             )
 
         # Increment the counter and include metadata in attributes
@@ -604,18 +600,20 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
 
     def answer_stmt_execute(self, stmt_id, parameters):
         prepared_stmt = self.session.prepared_stmts[stmt_id]
-        executor = prepared_stmt["statement"]
+        executor: Executor = prepared_stmt["statement"]
 
         executor.stmt_execute(parameters)
 
-        if executor.data is None:
+        executor_answer: ExecuteAnswer = executor.executor_answer
+
+        if executor_answer.data is None:
             resp = SQLAnswer(
-                resp_type=RESPONSE_TYPE.OK, state_track=executor.state_track
+                resp_type=RESPONSE_TYPE.OK, state_track=executor_answer.state_track
             )
             return self.send_query_answer(resp)
 
         # TODO prepared_stmt['type'] == 'lock' is not used but it works
-        columns_def = self.to_mysql_columns(executor.columns)
+        columns_def = self.to_mysql_columns(executor_answer.data.columns)
         packages = [self.packet(ColumnCountPacket, count=len(columns_def))]
 
         packages.extend(self._get_column_defenition_packets(columns_def))
@@ -624,14 +622,14 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
             packages.append(self.packet(EofPacket, status=0x0062))
 
         # send all
-        for row in executor.data.to_lists():
+        for row in executor_answer.data.to_lists():
             packages.append(
                 self.packet(BinaryResultsetRowPacket, data=row, columns=columns_def)
             )
 
         server_status = executor.server_status or 0x0002
         packages.append(self.last_packet(status=server_status))
-        prepared_stmt["fetched"] += len(executor.data)
+        prepared_stmt["fetched"] += len(executor_answer.data)
 
         return self.send_package_group(packages)
 
@@ -639,23 +637,24 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
         prepared_stmt = self.session.prepared_stmts[stmt_id]
         executor = prepared_stmt["statement"]
         fetched = prepared_stmt["fetched"]
+        executor_answer: ExecuteAnswer = executor.executor_answer
 
-        if executor.data is None:
+        if executor_answer.data is None:
             resp = SQLAnswer(
-                resp_type=RESPONSE_TYPE.OK, state_track=executor.state_track
+                resp_type=RESPONSE_TYPE.OK, state_track=executor_answer.state_track
             )
             return self.send_query_answer(resp)
 
         packages = []
-        columns = self.to_mysql_columns(executor.columns)
-        for row in executor.data[fetched:limit].to_lists():
+        columns = self.to_mysql_columns(executor_answer.data.columns)
+        for row in executor_answer.data[fetched:limit].to_lists():
             packages.append(
                 self.packet(BinaryResultsetRowPacket, data=row, columns=columns)
             )
 
-        prepared_stmt["fetched"] += len(executor.data[fetched:limit])
+        prepared_stmt["fetched"] += len(executor_answer.data[fetched:limit])
 
-        if len(executor.data) <= limit + fetched:
+        if len(executor_answer.data) <= limit + fetched:
             status = sum(
                 [
                     SERVER_STATUS.SERVER_STATUS_AUTOCOMMIT,
@@ -772,6 +771,8 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
                 elif p.type.value == COMMANDS.COM_FIELD_LIST:
                     # this command is deprecated, but console client still use it.
                     response = SQLAnswer(RESPONSE_TYPE.OK)
+                elif p.type.value == COMMANDS.COM_STMT_RESET:
+                    response = SQLAnswer(RESPONSE_TYPE.OK)
                 else:
                     logger.warning("Command has no specific handler, return OK msg")
                     logger.debug(str(p))