MindsDB 25.9.2.0a1__py3-none-any.whl → 25.10.0rc1__py3-none-any.whl

mindsdb/api/executor/planner/plan_join.py

@@ -13,6 +13,8 @@ from mindsdb_sql_parser.ast import (
     Constant,
     NativeQuery,
     Parameter,
+    Function,
+    Last,
 )
 
 from mindsdb.integrations.utilities.query_traversal import query_traversal
@@ -220,17 +222,16 @@ class PlanJoinTablesQuery:
                 # try to use second arg, could be: 'x'=col
                 col_idx = 1
 
-        # check the case col <condition> constant, col between constant and constant
+        # check the case col <condition> value, col between value and value
         for i, arg in enumerate(node.args):
             if i == col_idx:
                 if not isinstance(arg, Identifier):
                     return
             else:
-                if not isinstance(arg, (Constant, Parameter)):
+                if not self.can_be_table_filter(arg):
                     return
 
         # checked, find table and store condition
-
         node2 = copy.deepcopy(node)
 
         arg1 = node2.args[col_idx]
@@ -248,6 +249,19 @@ class PlanJoinTablesQuery:
         node2._orig_node = node
         table_info.conditions.append(node2)
 
+    def can_be_table_filter(self, node):
+        """
+        Check if node can be used as a filter.
+        It can contain only: Constant, Parameter, Function (with Last)
+        """
+        if isinstance(node, (Constant, Parameter)):
+            return True
+        if isinstance(node, Function):
+            # `Last` must be in args
+            if not any(isinstance(arg, Last) for arg in node.args):
+                return False
+            return all([self.can_be_table_filter(arg) for arg in node.args])
+
     def check_query_conditions(self, query):
         # get conditions for tables
         binary_ops = []

mindsdb/api/executor/planner/query_prepare.py

@@ -286,26 +286,8 @@ class PreparedStatementPlanner:
 
                 if step.result_data is not None:
                     # save results
-
-                    if len(step.result_data["tables"]) > 0:
-                        table_info = step.result_data["tables"][0]
-                        columns_info = step.result_data["columns"][table_info]
-
-                        table.columns = []
-                        table.ds = table_info[0]
-                        for col in columns_info:
-                            if isinstance(col, tuple):
-                                # is predictor
-                                col = dict(name=col[0], type="str")
-                            table.columns.append(
-                                Column(
-                                    name=col["name"],
-                                    type=col["type"],
-                                )
-                            )
-
-                    # map by names
-                    table.columns_map = {i.name.upper(): i for i in table.columns}
+                    table.columns = step.result_data.columns
+                    table.columns_map = {column.name.upper(): column for column in step.result_data.columns}
 
         # === create columns list ===
         columns_result = []

mindsdb/api/executor/sql_query/sql_query.py

@@ -8,6 +8,7 @@
  * permission of MindsDB Inc
  *******************************************************
 """
+
 import inspect
 from textwrap import dedent
 from typing import Union, Dict
@@ -41,15 +42,21 @@ from mindsdb.utilities.context import context as ctx
 
 from . import steps
 from .result_set import ResultSet, Column
-from . steps.base import BaseStepCall
+from .steps.base import BaseStepCall
 
 
 class SQLQuery:
-
     step_handlers = {}
 
-    def __init__(self, sql: Union[ASTNode, str], session, execute: bool = True,
-                 database: str = None, query_id: int = None, stop_event=None):
+    def __init__(
+        self,
+        sql: Union[ASTNode, str],
+        session,
+        execute: bool = True,
+        database: str = None,
+        query_id: int = None,
+        stop_event=None,
+    ):
         self.session = session
 
         self.query_id = query_id
@@ -64,10 +71,7 @@ class SQLQuery:
         else:
             self.database = session.database
 
-        self.context = {
-            'database': None if self.database == '' else self.database.lower(),
-            'row_id': 0
-        }
+        self.context = {"database": None if self.database == "" else self.database.lower(), "row_id": 0}
 
         self.columns_list = None
         self.steps_data: Dict[int, ResultSet] = {}
@@ -82,14 +86,14 @@ class SQLQuery:
 
         if isinstance(sql, str):
             self.query = parse_sql(sql)
-            self.context['query_str'] = sql
+            self.context["query_str"] = sql
         else:
             self.query = sql
-            renderer = SqlalchemyRender('mysql')
+            renderer = SqlalchemyRender("mysql")
             try:
-                self.context['query_str'] = renderer.get_string(self.query, with_failback=True)
+                self.context["query_str"] = renderer.get_string(self.query, with_failback=True)
             except Exception:
-                self.context['query_str'] = str(self.query)
+                self.context["query_str"] = str(self.query)
 
         self.create_planner()
 
@@ -98,7 +102,6 @@ class SQLQuery:
 
     @classmethod
     def register_steps(cls):
-
         cls.step_handlers = {}
         for _, cl in inspect.getmembers(steps):
             if inspect.isclass(cl) and issubclass(cl, BaseStepCall):
@@ -115,13 +118,10 @@ class SQLQuery:
         query_tables = get_query_models(self.query, default_database=self.database)
 
         for project_name, table_name, table_version in query_tables:
-            args = {
-                'name': table_name,
-                'project_name': project_name
-            }
+            args = {"name": table_name, "project_name": project_name}
             if table_version is not None:
-                args['active'] = None
-                args['version'] = table_version
+                args["active"] = None
+                args["version"] = table_version
 
             model_record = get_model_record(**args)
             if model_record is None:
@@ -132,61 +132,65 @@ class SQLQuery:
                     continue
                 if agent is not None:
                     predictor = {
-                        'name': table_name,
-                        'integration_name': project_name,  # integration_name,
-                        'timeseries': False,
-                        'id': agent.id,
-                        'to_predict': 'answer',
+                        "name": table_name,
+                        "integration_name": project_name,  # integration_name,
+                        "timeseries": False,
+                        "id": agent.id,
+                        "to_predict": "answer",
                     }
                     predictor_metadata.append(predictor)
 
                 continue
 
-            if model_record.status == 'error':
-                dot_version_str = ''
-                and_version_str = ''
+            if model_record.status == "error":
+                dot_version_str = ""
+                and_version_str = ""
                 if table_version is not None:
-                    dot_version_str = f'.{table_version}'
-                    and_version_str = f' and version = {table_version}'
+                    dot_version_str = f".{table_version}"
+                    and_version_str = f" and version = {table_version}"
 
-                raise BadTableError(dedent(f'''\
+                raise BadTableError(
+                    dedent(f"""\
                     The model '{table_name}{dot_version_str}' cannot be used as it is currently in 'error' status.
                     For detailed information about the error, please execute the following command:
 
                         select error from information_schema.models where name = '{table_name}'{and_version_str};
-                '''))
+                """)
+                )
 
-            ts_settings = model_record.learn_args.get('timeseries_settings', {})
+            ts_settings = model_record.learn_args.get("timeseries_settings", {})
             predictor = {
-                'name': table_name,
-                'integration_name': project_name,   # integration_name,
-                'timeseries': False,
-                'id': model_record.id,
-                'to_predict': model_record.to_predict,
+                "name": table_name,
+                "integration_name": project_name,  # integration_name,
+                "timeseries": False,
+                "id": model_record.id,
+                "to_predict": model_record.to_predict,
             }
-            if ts_settings.get('is_timeseries') is True:
-                window = ts_settings.get('window')
-                order_by = ts_settings.get('order_by')
+            if ts_settings.get("is_timeseries") is True:
+                window = ts_settings.get("window")
+                order_by = ts_settings.get("order_by")
                 if isinstance(order_by, list):
                     order_by = order_by[0]
-                group_by = ts_settings.get('group_by')
+                group_by = ts_settings.get("group_by")
                 if isinstance(group_by, list) is False and group_by is not None:
                     group_by = [group_by]
-                predictor.update({
-                    'timeseries': True,
-                    'window': window,
-                    'horizon': ts_settings.get('horizon'),
-                    'order_by_column': order_by,
-                    'group_by_columns': group_by
-                })
+                predictor.update(
+                    {
+                        "timeseries": True,
+                        "window": window,
+                        "horizon": ts_settings.get("horizon"),
+                        "order_by_column": order_by,
+                        "group_by_columns": group_by,
+                    }
+                )
 
-            predictor['model_types'] = model_record.data.get('dtypes', {})
+            predictor["model_types"] = model_record.data.get("dtypes", {})
 
             predictor_metadata.append(predictor)
 
-        database = None if self.database == '' else self.database.lower()
+        database = None if self.database == "" else self.database.lower()
 
-        self.context['predictor_metadata'] = predictor_metadata
+        self.context["predictor_metadata"] = predictor_metadata
         self.planner = query_planner.QueryPlanner(
             self.query,
             integrations=databases,
@@ -195,38 +199,32 @@ class SQLQuery:
         )
 
     def prepare_query(self):
-        """it is prepared statement call
-        """
+        """it is prepared statement call"""
         try:
             for step in self.planner.prepare_steps(self.query):
                 data = self.execute_step(step)
                 step.set_result(data)
                 self.steps_data[step.step_num] = data
         except PlanningException as e:
-            raise LogicError(e)
+            raise LogicError(e) from e
 
         statement_info = self.planner.get_statement_info()
 
         self.columns_list = []
-        for col in statement_info['columns']:
+        for col in statement_info["columns"]:
             self.columns_list.append(
                 Column(
-                    database=col['ds'],
-                    table_name=col['table_name'],
-                    table_alias=col['table_alias'],
-                    name=col['name'],
-                    alias=col['alias'],
-                    type=col['type']
+                    database=col["ds"],
+                    table_name=col["table_name"],
+                    table_alias=col["table_alias"],
+                    name=col["name"],
+                    alias=col["alias"],
+                    type=col["type"],
                 )
             )
 
         self.parameters = [
-            Column(
-                name=col['name'],
-                alias=col['alias'],
-                type=col['type']
-            )
-            for col in statement_info['parameters']
+            Column(name=col["name"], alias=col["alias"], type=col["type"]) for col in statement_info["parameters"]
         ]
 
     def execute_query(self):
@@ -237,14 +235,16 @@ class SQLQuery:
         try:
             steps = list(self.planner.execute_steps())
         except PlanningException as e:
-            raise LogicError(e)
+            raise LogicError(e) from e
 
         if self.planner.plan.is_resumable:
             # create query
             if self.query_id is not None:
                 self.run_query = query_context_controller.get_query(self.query_id)
             else:
-                self.run_query = query_context_controller.create_query(self.context['query_str'], database=self.database)
+                self.run_query = query_context_controller.create_query(
+                    self.context["query_str"], database=self.database
+                )
 
             if self.planner.plan.is_async and ctx.task_id is None:
                 # add to task
@@ -265,9 +265,9 @@ class SQLQuery:
             steps_classes = (x.__class__ for x in steps)
             predict_steps = (ApplyPredictorRowStep, ApplyPredictorStep, ApplyTimeseriesPredictorStep)
             if any(s in predict_steps for s in steps_classes):
-                process_mark = create_process_mark('predict')
+                process_mark = create_process_mark("predict")
             for step in steps:
-                with profiler.Context(f'step: {step.__class__.__name__}'):
+                with profiler.Context(f"step: {step.__class__.__name__}"):
                     step_result = self.execute_step(step)
                 self.steps_data[step.step_num] = step_result
         except Exception as e:
@@ -282,7 +282,7 @@ class SQLQuery:
                 ctx.run_query_id = None
         finally:
             if process_mark is not None:
-                delete_process_mark('predict', process_mark)
+                delete_process_mark("predict", process_mark)
 
         # save updated query
         self.query = self.planner.query
@@ -294,14 +294,14 @@ class SQLQuery:
         self.fetched_data = step_result
 
         try:
-            if hasattr(self, 'columns_list') is False:
+            if hasattr(self, "columns_list") is False:
                 # how it becomes False?
                 self.columns_list = self.fetched_data.columns
 
             if self.columns_list is None:
                 self.columns_list = self.fetched_data.columns
 
-            for col in self.fetched_data.find_columns('__mindsdb_row_id'):
+            for col in self.fetched_data.find_columns("__mindsdb_row_id"):
                 self.fetched_data.del_column(col)
 
         except Exception as e:

mindsdb/api/executor/sql_query/steps/fetch_dataframe.py

@@ -87,8 +87,7 @@ class FetchDataframeStepCall(BaseStepCall):
         if query is None:
             table_alias = (self.context.get("database"), "result", "result")
 
-            # fetch raw_query
-            response: DataHubResponse = dn.query(native_query=step.raw_query, session=self.session)
+            response: DataHubResponse = dn.query(step.raw_query, session=self.session)
             df = response.data_frame
         else:
             if isinstance(step.query, (Union, Intersect)):

mindsdb/api/executor/sql_query/steps/subselect_step.py

@@ -199,7 +199,6 @@ class QueryStepCall(BaseStepCall):
             #   but can be absent in their output
 
             def remove_not_used_conditions(node, **kwargs):
-                # find last in where
                 if isinstance(node, BinaryOperation):
                     for arg in node.args:
                         if isinstance(arg, Identifier) and len(arg.parts) > 1:

mindsdb/api/executor/utilities/functions.py

@@ -23,15 +23,15 @@ def download_file(url):
         parse_result = urllib.parse.urlparse(url)
         scheme = parse_result.scheme
     except ValueError:
-        raise Exception(f'Invalid url: {url}')
+        raise Exception(f"Invalid url: {url}")
     except Exception as e:
-        raise Exception(f'URL parsing error: {e}')
-    temp_dir = tempfile.mkdtemp(prefix='mindsdb_file_download_')
-    if scheme == '':
+        raise Exception(f"URL parsing error: {e}") from e
+    temp_dir = tempfile.mkdtemp(prefix="mindsdb_file_download_")
+    if scheme == "":
         raise Exception(f"Unknown url schema: {url}")
 
     response = requests.get(url)
-    temp_file_path = Path(temp_dir).joinpath('file')
-    with open(str(temp_file_path), 'wb')as file:
+    temp_file_path = Path(temp_dir).joinpath("file")
+    with open(str(temp_file_path), "wb") as file:
         file.write(response.content)
     return str(temp_file_path)

mindsdb/api/executor/utilities/sql.py

@@ -4,13 +4,14 @@ from typing import List
 import duckdb
 from duckdb import InvalidInputException
 import numpy as np
+import orjson
 
 from mindsdb_sql_parser import parse_sql
 from mindsdb_sql_parser.ast import ASTNode, Select, Identifier, Function, Constant
 
 from mindsdb.integrations.utilities.query_traversal import query_traversal
 from mindsdb.utilities import log
-from mindsdb.utilities.exception import format_db_error_message
+from mindsdb.utilities.exception import QueryError
 from mindsdb.utilities.functions import resolve_table_identifier, resolve_model_identifier
 from mindsdb.utilities.json_encoder import CustomJSONEncoder
 from mindsdb.utilities.render.sqlalchemy_render import SqlalchemyRender
@@ -64,6 +65,9 @@ def query_df_with_type_infer_fallback(query_str: str, dataframes: dict, user_fun
     Returns:
         pandas.DataFrame
         pandas.columns
+
+    Raises:
+        QueryError: Raised when DuckDB fails to execute the query
     """
 
     try:
@@ -86,9 +90,17 @@ def query_df_with_type_infer_fallback(query_str: str, dataframes: dict, user_fun
             else:
                 raise exception
             description = con.description
+    except InvalidInputException as e:
+        raise QueryError(
+            db_type="DuckDB",
+            db_error_msg=f"DuckDB failed to execute query, likely due to inability to determine column data types. Details: {e}",
+            failed_query=query_str,
+            is_external=False,
+            is_expected=False,
+        ) from e
     except Exception as e:
-        raise Exception(
-            format_db_error_message(db_type="DuckDB", db_error_msg=str(e), failed_query=query_str, is_external=False)
+        raise QueryError(
+            db_type="DuckDB", db_error_msg=str(e), failed_query=query_str, is_external=False, is_expected=False
         ) from e
 
     return result_df, description
@@ -164,7 +176,13 @@ def query_df(df, query, session=None):
         query_str = str(query)
 
     if isinstance(query_ast, Select) is False or isinstance(query_ast.from_table, Identifier) is False:
-        raise Exception("Only 'SELECT from TABLE' statements supported for internal query")
+        raise QueryError(
+            db_type="DuckDB",
+            db_error_msg="Only 'SELECT from TABLE' statements supported for internal query",
+            failed_query=query_str,
+            is_external=False,
+            is_expected=False,
+        )
 
     table_name = query_ast.from_table.parts[0]
     query_ast.from_table.parts = ["df"]
@@ -214,27 +232,26 @@ def query_df(df, query, session=None):
             custom_functions_list = [] if user_functions is None else list(user_functions.functions.keys())
             all_functions_list = duckdb_functions_and_kw_list + custom_functions_list
             if len(all_functions_list) > 0 and fnc_name not in all_functions_list:
-                raise Exception(
-                    format_db_error_message(
-                        db_type="DuckDB",
-                        db_error_msg=(
-                            f"Unknown function: '{fnc_name}'. This function is not recognized during internal query processing.\n"
-                            "Please use DuckDB-supported functions instead."
-                        ),
-                        failed_query=query_str,
-                        is_external=False,
-                    )
+                raise QueryError(
+                    db_type="DuckDB",
+                    db_error_msg=(
+                        f"Unknown function: '{fnc_name}'. This function is not recognized during internal query processing.\n"
+                        "Please use DuckDB-supported functions instead."
+                    ),
+                    failed_query=query_str,
+                    is_external=False,
+                    is_expected=False,
                 )
 
     query_traversal(query_ast, adapt_query)
 
-    # convert json columns
-    encoder = CustomJSONEncoder()
-
     def _convert(v):
         if isinstance(v, dict) or isinstance(v, list):
             try:
-                return encoder.encode(v)
+                default_encoder = CustomJSONEncoder().default
+                return orjson.dumps(
+                    v, default=default_encoder, option=orjson.OPT_SERIALIZE_NUMPY | orjson.OPT_PASSTHROUGH_DATETIME
+                ).decode("utf-8")
             except Exception:
                 pass
         return v
@@ -245,8 +262,8 @@ def query_df(df, query, session=None):
     render = SqlalchemyRender("postgres")
     try:
         query_str = render.get_string(query_ast, with_failback=False)
-    except Exception as e:
-        logger.error(f"Exception during query casting to 'postgres' dialect. Query: {str(query)}. Error: {e}")
+    except Exception:
+        logger.exception(f"Exception during query casting to 'postgres' dialect. Query:\n{str(query)}.\nError:")
         query_str = render.get_string(query_ast, with_failback=True)
 
     # workaround to prevent duckdb.TypeMismatchException

mindsdb/api/http/gui.py

@@ -30,8 +30,8 @@ def download_gui(destignation, version):
     try:
         for r in resources:
             get_resources(r)
-    except Exception as e:
-        logger.error(f"Error during downloading files from s3: {e}")
+    except Exception:
+        logger.exception("Error during downloading files from s3:")
         return False
 
     static_folder = destignation
@@ -39,12 +39,8 @@ def download_gui(destignation, version):
     ZipFile(dist_zip_path).extractall(static_folder)
 
     if static_folder.joinpath("dist").is_dir():
-        shutil.move(
-            str(destignation.joinpath("dist").joinpath("index.html")), static_folder
-        )
-        shutil.move(
-            str(destignation.joinpath("dist").joinpath("assets")), static_folder
-        )
+        shutil.move(str(destignation.joinpath("dist").joinpath("index.html")), static_folder)
+        shutil.move(str(destignation.joinpath("dist").joinpath("assets")), static_folder)
         shutil.rmtree(destignation.joinpath("dist"))
 
     os.remove(dist_zip_path)
@@ -74,9 +70,7 @@ def update_static(gui_version: Version):
     config = Config()
     static_path = Path(config["paths"]["static"])
 
-    logger.info(
-        f"New version of GUI available ({gui_version.base_version}). Downloading..."
-    )
+    logger.info(f"New version of GUI available ({gui_version.base_version}). Downloading...")
 
     temp_dir = tempfile.mkdtemp(prefix="mindsdb_gui_files_")
     success = download_gui(temp_dir, gui_version.base_version)