MindsDB 25.7.4.0__py3-none-any.whl → 25.8.2.0__py3-none-any.whl

mindsdb/__about__.py

@@ -1,6 +1,6 @@
 __title__ = "MindsDB"
 __package_name__ = "mindsdb"
-__version__ = "25.7.4.0"
+__version__ = "25.8.2.0"
 __description__ = "MindsDB's AI SQL Server enables developers to build AI tools that need access to real-time data to perform their tasks"
 __email__ = "jorge@mindsdb.com"
 __author__ = "MindsDB Inc"

mindsdb/__main__.py

@@ -39,7 +39,7 @@ from mindsdb.utilities.starters import (
 )
 from mindsdb.utilities.ps import is_pid_listen_port, get_child_pids
 import mindsdb.interfaces.storage.db as db
-from mindsdb.utilities.fs import clean_process_marks, clean_unlinked_process_marks
+from mindsdb.utilities.fs import clean_process_marks, clean_unlinked_process_marks, create_pid_file, delete_pid_file
 from mindsdb.utilities.context import context as ctx
 from mindsdb.utilities.auth import register_oauth_client, get_aws_meta_data
 from mindsdb.utilities.sentry import sentry_sdk  # noqa: F401
@@ -335,6 +335,13 @@ if __name__ == "__main__":
         print(f"MindsDB {mindsdb_version}")
         sys.exit(0)
 
+    if config.cmd_args.update_gui:
+        from mindsdb.api.http.initialize import initialize_static
+
+        logger.info("Updating the GUI version")
+        initialize_static()
+        sys.exit(0)
+
     config.raise_warnings(logger=logger)
     os.environ["MINDSDB_RUNTIME"] = "1"
 
@@ -510,6 +517,8 @@ if __name__ == "__main__":
     if config.cmd_args.ml_task_queue_consumer is True:
         trunc_processes_struct[TrunkProcessEnum.ML_TASK_QUEUE].need_to_run = True
 
+    create_pid_file()
+
     for trunc_process_data in trunc_processes_struct.values():
         if trunc_process_data.started is True or trunc_process_data.need_to_run is False:
             continue
@@ -591,6 +600,7 @@ if __name__ == "__main__":
             ],
             return_exceptions=False,
         )
+        delete_pid_file()
 
     ioloop = asyncio.new_event_loop()
     ioloop.run_until_complete(wait_apis_start())

mindsdb/api/executor/command_executor.py

@@ -1195,11 +1195,17 @@ class ExecuteCommands:
                 msg = dedent(
                     f"""\
                     The '{handler_module_meta["name"]}' handler cannot be used. Reason is:
-                        {handler_module_meta["import"]["error_message"]}
+                        {handler_module_meta["import"]["error_message"] or msg}
                 """
                 )
                 is_cloud = self.session.config.get("cloud", False)
-                if is_cloud is False and "No module named" in handler_module_meta["import"]["error_message"]:
+                if (
+                    is_cloud is False
+                    # NOTE: BYOM may raise these errors if there is an error in the user's code,
+                    # therefore error_message will be None
+                    and handler_module_meta["name"] != "byom"
+                    and "No module named" in handler_module_meta["import"]["error_message"]
+                ):
                     logger.info(get_handler_install_message(handler_module_meta["name"]))
             ast_drop = DropMLEngine(name=Identifier(name))
             self.answer_drop_ml_engine(ast_drop)
@@ -1342,24 +1348,12 @@ class ExecuteCommands:
                 from_table=NativeQuery(integration=statement.from_table, query=statement.query_str),
             )
             query_str = query.to_string()
-        else:
-            query = parse_sql(query_str)
-
-        if isinstance(query, Select):
-            # check create view sql
-            query.limit = Constant(1)
-
-            query_context_controller.set_context(query_context_controller.IGNORE_CONTEXT)
-            try:
-                SQLQuery(query, session=self.session, database=database_name)
-            finally:
-                query_context_controller.release_context(query_context_controller.IGNORE_CONTEXT)
 
         project = self.session.database_controller.get_project(project_name)
 
         if isinstance(statement, CreateView):
             try:
-                project.create_view(view_name, query=query_str)
+                project.create_view(view_name, query=query_str, session=self.session)
             except EntityExistsError:
                 if getattr(statement, "if_not_exists", False) is False:
                     raise

mindsdb/api/executor/sql_query/steps/fetch_dataframe.py

@@ -50,29 +50,26 @@ def get_table_alias(table_obj, default_db_name):
 
 def get_fill_param_fnc(steps_data):
     def fill_params(node, callstack=None, **kwargs):
-        if isinstance(node, Parameter):
-            rs = steps_data[node.value.step_num]
-            items = [Constant(i) for i in rs.get_column_values(col_idx=0)]
-
-            is_single_item = True
-            if callstack:
-                node_prev = callstack[0]
-                if isinstance(node_prev, BinaryOperation):
-                    # Check case: 'something IN Parameter()'
-                    if node_prev.op.lower() == "in" and node_prev.args[1] is node:
-                        is_single_item = False
-
-            if is_single_item and len(items) == 1:
-                # extract one value for option 'col=(subselect)'
-                node = items[0]
-            else:
-                node = Tuple(items)
-            return node
-
-        if isinstance(node, Parameter):
-            rs = steps_data[node.value.step_num]
-            items = [Constant(i) for i in rs.get_column_values(col_idx=0)]
-            return Tuple(items)
+        if not isinstance(node, Parameter):
+            return
+
+        rs = steps_data[node.value.step_num]
+        items = [Constant(i) for i in rs.get_column_values(col_idx=0)]
+
+        is_single_item = True
+        if callstack:
+            node_prev = callstack[0]
+            if isinstance(node_prev, BinaryOperation):
+                # Check case: 'something IN Parameter()'
+                if node_prev.op.lower() == "in" and node_prev.args[1] is node:
+                    is_single_item = False
+
+        if is_single_item and len(items) == 1:
+            # extract one value for option 'col=(subselect)'
+            node = items[0]
+        else:
+            node = Tuple(items)
+        return node
 
     return fill_params
 
@@ -115,7 +112,7 @@ class FetchDataframeStepCall(BaseStepCall):
 
         # if query registered, set progress
         if self.sql_query.run_query is not None:
-            self.sql_query.run_query.set_progress(df, None)
+            self.sql_query.run_query.set_progress(processed_rows=len(df))
         return ResultSet.from_df(
             df,
             table_name=table_alias[1],

mindsdb/api/executor/sql_query/steps/fetch_dataframe_partition.py

@@ -97,6 +97,7 @@ class FetchDataframePartitionCall(BaseStepCall):
         for df in run_query.get_partitions(self.dn, self, query):
             try:
                 sub_data = self.exec_sub_steps(df)
+                run_query.set_progress(processed_rows=len(df))
                 results.append(sub_data)
             except Exception as e:
                 if on_error == "skip":
@@ -175,17 +176,22 @@ class FetchDataframePartitionCall(BaseStepCall):
                 # split into chunks and send to workers
                 futures = []
                 for df2 in split_data_frame(df, partition_size):
-                    futures.append(executor.submit(self.exec_sub_steps, df2))
+                    futures.append([executor.submit(self.exec_sub_steps, df2), len(df2)])
 
-                for future in futures:
+                error = None
+                for future, rows_count in futures:
                     try:
                         results.append(future.result())
+                        run_query.set_progress(processed_rows=rows_count)
                     except Exception as e:
                         if on_error == "skip":
                             logger.error(e)
                         else:
                             executor.shutdown()
-                            raise e
+                            error = e
+
+                if error:
+                    raise error
                 if self.sql_query.stop_event is not None and self.sql_query.stop_event.is_set():
                     executor.shutdown()
                     raise RuntimeError("Query is interrupted")

mindsdb/api/executor/sql_query/steps/subselect_step.py

@@ -2,7 +2,15 @@ from collections import defaultdict
 
 import pandas as pd
 
-from mindsdb_sql_parser.ast import Identifier, Select, Star, Constant, Parameter, Function, Variable, BinaryOperation
+from mindsdb_sql_parser.ast import (
+    Identifier,
+    Select,
+    Star,
+    Constant,
+    Function,
+    Variable,
+    BinaryOperation,
+)
 
 from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import SERVER_VARIABLES
 from mindsdb.api.executor.planner.step_result import Result
@@ -52,13 +60,8 @@ class SubSelectStepCall(BaseStepCall):
 
         # inject previous step values
         if isinstance(query, Select):
-
-            def inject_values(node, **kwargs):
-                if isinstance(node, Parameter) and isinstance(node.value, Result):
-                    prev_result = self.steps_data[node.value.step_num]
-                    return Constant(prev_result.get_column_values(col_idx=0)[0])
-
-            query_traversal(query, inject_values)
+            fill_params = get_fill_param_fnc(self.steps_data)
+            query_traversal(query, fill_params)
 
         df = result.to_df()
         res = query_df(df, query, session=self.session)

mindsdb/api/executor/utilities/mysql_to_duckdb_functions.py

@@ -0,0 +1,264 @@
+from mindsdb_sql_parser.ast import Identifier, Function, Constant, BinaryOperation
+
+
+def adapt_char_fn(node: Function) -> Function | None:
+    """Replace MySQL's multy-arg CHAR call to chain of DuckDB's CHR calls
+
+    Example:
+        CHAR(77, 78, 79) => CHR(77) || CHR(78) || CHR(79)
+
+    Args:
+        node (Function): Function node to adapt
+
+    Returns:
+        Function | None: Adapted function node
+    """
+    if len(node.args) == 1:
+        node.op = "chr"
+        return node
+
+    acc = None
+    for arg in node.args:
+        fn = Function(op="chr", args=[arg])
+        if acc is None:
+            acc = fn
+            continue
+        acc = BinaryOperation("||", args=[acc, fn])
+
+    acc.parentheses = True
+    acc.alias = node.alias
+    return acc
+
+
+def adapt_locate_fn(node: Function) -> Function | None:
+    """Replace MySQL's LOCATE (or INSTR) call to DuckDB's STRPOS call
+
+    Example:
+        LOCATE('bar', 'foobarbar') => STRPOS('foobarbar', 'bar')
+        INSTR('foobarbar', 'bar') => STRPOS('foobarbar', 'bar')
+        LOCATE('bar', 'foobarbar', 3) => ValueError (there is no analogue in DuckDB)
+
+    Args:
+        node (Function): Function node to adapt
+
+    Returns:
+        Function | None: Adapted function node
+
+    Raises:
+        ValueError: If the function has 3 arguments
+    """
+    if len(node.args) == 3:
+        raise ValueError("MySQL LOCATE function with 3 arguments is not supported")
+    if node.op == "locate":
+        node.args = [node.args[1], node.args[0]]
+    elif node.op == "insrt":
+        node.args = [node.args[0], node.args[1]]
+    node.op = "strpos"
+
+
+def adapt_unhex_fn(node: Function) -> None:
+    """Check MySQL's UNHEX function call arguments to ensure they are strings,
+    because DuckDB's UNHEX accepts only string arguments, while MySQL's UNHEX can accept integer arguments.
+    NOTE: if return dataframe from duckdb then unhex values are array - this may be an issue
+
+    Args:
+        node (Function): Function node to adapt
+
+    Returns:
+        None
+
+    Raises:
+        ValueError: If the function argument is not a string
+    """
+    for arg in node.args:
+        if not isinstance(arg, (str, bytes)):
+            raise ValueError("MySQL UNHEX function argument must be a string")
+
+
+def adapt_format_fn(node: Function) -> None:
+    """Adapt MySQL's FORMAT function to DuckDB's FORMAT function
+
+    Example:
+        FORMAT(1234567.89, 0) => FORMAT('{:,.0f}', 1234567.89)
+        FORMAT(1234567.89, 2) => FORMAT('{:,.2f}', 1234567.89)
+        FORMAT(name, 2) => FORMAT('{:,.2f}', name)
+        FORMAT('{:.2f}', 1234567.89) => FORMAT('{:,.2f}', 1234567.89)  # no changes for original style
+
+    Args:
+        node (Function): Function node to adapt
+
+    Returns:
+        None
+
+    Raises:
+        ValueError: If MySQL's function has 3rd 'locale' argument, like FORMAT(12332.2, 2, 'de_DE')
+    """
+    match node.args[0], node.args[1]:
+        case Constant(value=(int() | float())), Constant(value=int()):
+            ...
+        case Identifier(), Constant(value=int()):
+            ...
+        case _:
+            return node
+
+    if len(node.args) > 2:
+        raise ValueError("'locale' argument of 'format' function is not supported")
+    decimal_places = node.args[1].value
+
+    if isinstance(node.args[0], Constant):
+        node.args[1].value = node.args[0].value
+        node.args[0].value = f"{{:,.{decimal_places}f}}"
+    else:
+        node.args[1] = node.args[0]
+        node.args[0] = Constant(f"{{:,.{decimal_places}f}}")
+
+
+def adapt_sha2_fn(node: Function) -> None:
+    """Adapt MySQL's SHA2 function to DuckDB's SHA256 function
+
+    Example:
+        SHA2('test', 256) => SHA256('test')
+
+    Args:
+        node (Function): Function node to adapt
+
+    Returns:
+        None
+
+    Raises:
+        ValueError: If the function has more than 1 argument or the argument is not 256
+    """
+    if len(node.args) > 1 and node.args[1].value != 256:
+        raise ValueError("Only sha256 is supported")
+    node.op = "sha256"
+    node.args = [node.args[0]]
+
+
+def adapt_length_fn(node: Function) -> None:
+    """Adapt MySQL's LENGTH function to DuckDB's STRLEN function
+    NOTE: duckdb also have LENGTH, therefore it can not be used
+
+    Example:
+        LENGTH('test') => STRLEN('test')
+
+    Args:
+        node (Function): Function node to adapt
+
+    Returns:
+        None
+    """
+    node.op = "strlen"
+
+
+def adapt_regexp_substr_fn(node: Function) -> None:
+    """Adapt MySQL's REGEXP_SUBSTR function to DuckDB's REGEXP_EXTRACT function
+
+    Example:
+        REGEXP_SUBSTR('foobarbar', 'bar', 1, 1) => REGEXP_EXTRACT('foobarbar', 'bar')
+
+    Args:
+        node (Function): Function node to adapt
+
+    Returns:
+        None
+
+    Raises:
+        ValueError: If the function has more than 2 arguments or 3rd or 4th argument is not 1
+    """
+    if (
+        len(node.args) == 3
+        and node.args[2].value != 1
+        or len(node.args) == 4
+        and (node.args[3].value != 1 or node.args[2].value != 1)
+        or len(node.args) > 4
+    ):
+        raise ValueError("Only 2 arguments are supported for REGEXP_SUBSTR function")
+    node.args = node.args[:2]
+    node.op = "regexp_extract"
+
+
+def adapt_substring_index_fn(node: Function) -> BinaryOperation | Function:
+    """Adapt MySQL's SUBSTRING_INDEX function to DuckDB's SPLIT_PART function
+
+    Example:
+        SUBSTRING_INDEX('a.b.c.d', '.', 1) => SPLIT_PART('a.b.c.d', '.', 1)
+        SUBSTRING_INDEX('a.b.c.d', '.', 2) => CONCAT_WS('.', SPLIT_PART('a.b.c.d', '.', 1), SPLIT_PART('a.b.c.d', '.', 2))
+
+    Args:
+        node (Function): Function node to adapt
+
+    Returns:
+        BinaryOperation | Function: Binary operation node or function node
+
+    Raises:
+        ValueError: If the function has more than 3 arguments or the 3rd argument is not 1
+    """
+    if len(node.args[1].value) > 1:
+        raise ValueError("Only one car in separator")
+
+    if node.args[2].value == 1:
+        node.op = "split_part"
+        return node
+
+    acc = [node.args[1]]
+    for i in range(node.args[2].value):
+        fn = Function(op="split_part", args=[node.args[0], node.args[1], Constant(i + 1)])
+        acc.append(fn)
+
+    acc = Function(op="concat_ws", args=acc)
+    acc.alias = node.alias
+    return acc
+
+
+def adapt_curtime_fn(node: Function) -> BinaryOperation:
+    """Adapt MySQL's CURTIME function to DuckDB's GET_CURRENT_TIME function.
+    To get the same type as MySQL's CURTIME function, we need to cast the result to time type.
+
+    Example:
+        CURTIME() => GET_CURRENT_TIME()::time
+
+    Args:
+        node (Function): Function node to adapt
+
+    Returns:
+        BinaryOperation: Binary operation node
+    """
+    return BinaryOperation("::", args=[Function(op="get_current_time", args=[]), Identifier("time")], alias=node.alias)
+
+
+def adapt_timestampdiff_fn(node: Function) -> None:
+    """Adapt MySQL's TIMESTAMPDIFF function to DuckDB's DATE_DIFF function
+    NOTE: Looks like cast string args to timestamp works in most cases, but there may be some exceptions.
+
+    Example:
+        TIMESTAMPDIFF(YEAR, '2000-02-01', '2003-05-01') => DATE_DIFF('year', timestamp '2000-02-01', timestamp '2003-05-01')
+
+    Args:
+        node (Function): Function node to adapt
+
+    Returns:
+        None
+    """
+    node.op = "date_diff"
+    node.args[0] = Constant(node.args[0].parts[0])
+    node.args[1] = BinaryOperation(" ", args=[Identifier("timestamp"), node.args[1]])
+    node.args[2] = BinaryOperation(" ", args=[Identifier("timestamp"), node.args[2]])
+
+
+def adapt_extract_fn(node: Function) -> None:
+    """Adapt MySQL's EXTRACT function to DuckDB's EXTRACT function
+    TODO: multi-part args, like YEAR_MONTH, is not supported yet
+    NOTE: Looks like adding 'timestamp' works in most cases, but there may be some exceptions.
+
+    Example:
+        EXTRACT(YEAR FROM '2000-02-01') => EXTRACT('year' from timestamp '2000-02-01')
+
+    Args:
+        node (Function): Function node to adapt
+
+    Returns:
+        None
+    """
+    node.args[0] = Constant(node.args[0].parts[0])
+    if not isinstance(node.from_arg, Identifier):
+        node.from_arg = BinaryOperation(" ", args=[Identifier("timestamp"), node.from_arg])

mindsdb/api/executor/utilities/sql.py

@@ -14,6 +14,19 @@ from mindsdb.utilities.exception import format_db_error_message
 from mindsdb.utilities.functions import resolve_table_identifier, resolve_model_identifier
 from mindsdb.utilities.json_encoder import CustomJSONEncoder
 from mindsdb.utilities.render.sqlalchemy_render import SqlalchemyRender
+from mindsdb.api.executor.utilities.mysql_to_duckdb_functions import (
+    adapt_char_fn,
+    adapt_locate_fn,
+    adapt_unhex_fn,
+    adapt_format_fn,
+    adapt_sha2_fn,
+    adapt_length_fn,
+    adapt_regexp_substr_fn,
+    adapt_substring_index_fn,
+    adapt_curtime_fn,
+    adapt_timestampdiff_fn,
+    adapt_extract_fn,
+)
 
 logger = log.getLogger(__name__)
 
@@ -185,6 +198,23 @@ def query_df(df, query, session=None):
         if isinstance(node, Function):
             fnc_name = node.op.lower()
 
+            mysql_to_duck_fn_map = {
+                "char": adapt_char_fn,
+                "locate": adapt_locate_fn,
+                "insrt": adapt_locate_fn,
+                "unhex": adapt_unhex_fn,
+                "format": adapt_format_fn,
+                "sha2": adapt_sha2_fn,
+                "length": adapt_length_fn,
+                "regexp_substr": adapt_regexp_substr_fn,
+                "substring_index": adapt_substring_index_fn,
+                "curtime": adapt_curtime_fn,
+                "timestampdiff": adapt_timestampdiff_fn,
+                "extract": adapt_extract_fn,
+            }
+            if fnc_name in mysql_to_duck_fn_map:
+                return mysql_to_duck_fn_map[fnc_name](node)
+
             if fnc_name == "database" and len(node.args) == 0:
                 if session is not None:
                     cur_db = session.database

mindsdb/api/http/initialize.py

@@ -203,7 +203,8 @@ def initialize_static():
         logger.debug("Updating gui..")
         success = update_static(last_gui_version_lv)
 
-    db.session.close()
+    if db.session:
+        db.session.close()
     return success