MindsDB 25.6.4.0__py3-none-any.whl → 25.7.2.0__py3-none-any.whl

mindsdb/api/executor/planner/query_prepare.py

@@ -8,7 +8,7 @@ from mindsdb.api.executor.planner import utils
 
 def to_string(identifier):
     # alternative to AST.to_string() but without quoting
-    return '.'.join(identifier.parts)
+    return ".".join(identifier.parts)
 
 
 class Table:
@@ -32,7 +32,6 @@ class Column:
     def __init__(self, node=None, table=None, name=None, type=None):
         alias = None
         if node is not None:
-
             if isinstance(node, ast.Identifier):
                 # set name
                 name = node.parts[-1]  # ???
@@ -67,26 +66,25 @@ class Statement:
         self.offset = 0
 
 
-class PreparedStatementPlanner():
-
+class PreparedStatementPlanner:
     def __init__(self, planner):
         self.planner = planner
 
     def get_type_of_var(self, v):
         if isinstance(v, str):
-            return 'str'
+            return "str"
         elif isinstance(v, float):
-            return 'float'
+            return "float"
         elif isinstance(v, int):
-            return 'integer'
+            return "integer"
 
-        return 'str'
+        return "str"
 
     def get_statement_info(self):
         stmt = self.planner.statement
 
         if stmt is None:
-            raise PlanningException('Statement is not prepared')
+            raise PlanningException("Statement is not prepared")
 
         columns_result = []
 
@@ -95,45 +93,45 @@ class PreparedStatementPlanner():
             if column.table is not None:
                 table = column.table.name
                 ds = column.table.ds
-            columns_result.append(dict(
-                alias=column.alias,
-                type=column.type,
-                name=column.name,
-                table_name=table,
-                table_alias=table,
-                ds=ds,
-            ))
+            columns_result.append(
+                dict(
+                    alias=column.alias,
+                    type=column.type,
+                    name=column.name,
+                    table_name=table,
+                    table_alias=table,
+                    ds=ds,
+                )
+            )
 
         parameters = []
         for param in stmt.params:
-            name = '?'
-            parameters.append(dict(
-                alias=name,
-                type='str',
-                name=name,
-            ))
-
-        return {
-            'parameters': parameters,
-            'columns': columns_result
-        }
+            name = "?"
+            parameters.append(
+                dict(
+                    alias=name,
+                    type="str",
+                    name=name,
+                )
+            )
 
-    def get_table_of_column(self, t):
+        return {"parameters": parameters, "columns": columns_result}
 
+    def get_table_of_column(self, t):
         tables_map = self.planner.statement.tables_map
 
         # get tables to check
         if len(t.parts) > 1:
             # try to find table
             table_parts = t.parts[:-1]
-            table_name = '.'.join(table_parts)
+            table_name = ".".join(table_parts)
             if table_name in tables_map:
                 return tables_map[table_name]
 
             elif len(table_parts) > 1:
                 # maybe datasource is 1st part
                 table_parts = table_parts[1:]
-                table_name = '.'.join(table_parts)
+                table_name = ".".join(table_parts)
                 if table_name in tables_map:
                     return tables_map[table_name]
 
@@ -158,14 +156,10 @@ class PreparedStatementPlanner():
             # in reverse order
             for p in table.parts[::-1]:
                 parts.insert(0, p)
-                keys.append('.'.join(parts))
+                keys.append(".".join(parts))
 
         # remember table
-        tbl = Table(
-            ds=ds,
-            node=table,
-            is_predictor=is_predictor
-        )
+        tbl = Table(ds=ds, node=table, is_predictor=is_predictor)
         tbl.keys = keys
 
         return tbl
@@ -189,7 +183,6 @@ class PreparedStatementPlanner():
         stmt.tables_map = {}
         stmt.tables_lvl1 = []
         if query.from_table is not None:
-
             if isinstance(query.from_table, ast.Join):
                 # get all tables
                 join_tables = utils.convert_join_to_list(query.from_table)
@@ -198,21 +191,17 @@ class PreparedStatementPlanner():
 
             if isinstance(query.from_table, ast.Select):
                 # nested select, get only last select
-                join_tables = [
-                    dict(
-                        table=utils.get_deepest_select(query.from_table).from_table
-                    )
-                ]
+                join_tables = [dict(table=utils.get_deepest_select(query.from_table).from_table)]
 
             for i, join_table in enumerate(join_tables):
-                table = join_table['table']
+                table = join_table["table"]
                 if isinstance(table, ast.Identifier):
                     tbl = self.table_from_identifier(table)
 
                     if tbl.is_predictor:
                         # Is the last table?
                         if i + 1 < len(join_tables):
-                            raise PlanningException('Predictor must be last table in query')
+                            raise PlanningException("Predictor must be last table in query")
 
                     stmt.tables_lvl1.append(tbl)
                     for key in tbl.keys:
@@ -225,13 +214,12 @@ class PreparedStatementPlanner():
         # is there any predictors at other levels?
         lvl1_predictors = [i for i in stmt.tables_lvl1 if i.is_predictor]
         if len(query_predictors) != len(lvl1_predictors):
-            raise PlanningException('Predictor is not at first level')
+            raise PlanningException("Predictor is not at first level")
 
         # === get targets ===
         columns = []
         get_all_tables = False
         for t in query.targets:
-
             column = Column(t)
 
             # column alias
@@ -264,10 +252,10 @@ class PreparedStatementPlanner():
                 column.type = self.get_type_of_var(t.value)
             elif isinstance(t, ast.Function):
                 # mysql function
-                if t.op == 'connection_id':
-                    column.type = 'integer'
+                if t.op == "connection_id":
+                    column.type = "integer"
                 else:
-                    column.type = 'str'
+                    column.type = "str"
             else:
                 # TODO go down into lower level.
                 #  It can be function, operation, select.
@@ -276,7 +264,7 @@ class PreparedStatementPlanner():
                 # TODO add several known types for function, i.e ABS-int
 
                 # TODO TypeCast - as casted type
-                column.type = 'str'
+                column.type = "str"
 
             if alias is not None:
                 column.alias = alias
@@ -299,28 +287,25 @@ class PreparedStatementPlanner():
                 if step.result_data is not None:
                     # save results
 
-                    if len(step.result_data['tables']) > 0:
-                        table_info = step.result_data['tables'][0]
-                        columns_info = step.result_data['columns'][table_info]
+                    if len(step.result_data["tables"]) > 0:
+                        table_info = step.result_data["tables"][0]
+                        columns_info = step.result_data["columns"][table_info]
 
                         table.columns = []
                         table.ds = table_info[0]
                         for col in columns_info:
                             if isinstance(col, tuple):
                                 # is predictor
-                                col = dict(name=col[0], type='str')
+                                col = dict(name=col[0], type="str")
                             table.columns.append(
                                 Column(
-                                    name=col['name'],
-                                    type=col['type'],
+                                    name=col["name"],
+                                    type=col["type"],
                                 )
                             )
 
                     # map by names
-                    table.columns_map = {
-                        i.name.upper(): i
-                        for i in table.columns
-                    }
+                    table.columns_map = {i.name.upper(): i for i in table.columns}
 
         # === create columns list ===
         columns_result = []
@@ -329,7 +314,7 @@ class PreparedStatementPlanner():
                 # add data from all tables
                 for table in stmt.tables_lvl1:
                     if table.columns is None:
-                        raise PlanningException(f'Table is not found {table.name}')
+                        raise PlanningException(f"Table is not found {table.name}")
 
                     for col in table.columns:
                         # col = {name: 'col', type: 'str'}
@@ -354,7 +339,7 @@ class PreparedStatementPlanner():
                             column.type = table.columns_map[col_name].type
                         else:
                             # continue
-                            raise PlanningException(f'Column not found {col_name}')
+                            raise PlanningException(f"Column not found {col_name}")
 
                 else:
                     # table is not found, looking for in all tables
@@ -368,11 +353,11 @@ class PreparedStatementPlanner():
 
             # forcing alias
             if column.alias is None:
-                column.alias = f'column_{i}'
+                column.alias = f"column_{i}"
 
             # forcing type
             if column.type is None:
-                column.type = 'str'
+                column.type = "str"
 
             columns_result.append(column)
 
@@ -393,28 +378,25 @@ class PreparedStatementPlanner():
         if step.result_data is not None:
             # save results
 
-            if len(step.result_data['tables']) > 0:
-                table_info = step.result_data['tables'][0]
-                columns_info = step.result_data['columns'][table_info]
+            if len(step.result_data["tables"]) > 0:
+                table_info = step.result_data["tables"][0]
+                columns_info = step.result_data["columns"][table_info]
 
                 table.columns = []
                 table.ds = table_info[0]
                 for col in columns_info:
                     if isinstance(col, tuple):
                         # is predictor
-                        col = dict(name=col[0], type='str')
+                        col = dict(name=col[0], type="str")
                     table.columns.append(
                         Column(
-                            name=col['name'],
-                            type=col['type'],
+                            name=col["name"],
+                            type=col["type"],
                         )
                     )
 
                 # map by names
-                table.columns_map = {
-                    i.name.upper(): i
-                    for i in table.columns
-                }
+                table.columns_map = {i.name.upper(): i for i in table.columns}
 
         # save results
         columns_result = []
@@ -430,7 +412,7 @@ class PreparedStatementPlanner():
 
             if column.type is None:
                 # forcing type
-                column.type = 'str'
+                column.type = "str"
 
             columns_result.append(column)
 
@@ -440,13 +422,12 @@ class PreparedStatementPlanner():
         stmt = self.planner.statement
 
         stmt.columns = [
-            Column(name='Variable_name', type='str'),
-            Column(name='Value', type='str'),
+            Column(name="Variable_name", type="str"),
+            Column(name="Value", type="str"),
         ]
         return []
 
     def prepare_steps(self, query):
-
         stmt = Statement()
         self.planner.statement = stmt
 
@@ -476,7 +457,6 @@ class PreparedStatementPlanner():
         if isinstance(query, ast.Show):
             return self.prepare_show(query)
         else:
-
             # do nothing
             return []
             # raise NotImplementedError(query.__name__)
@@ -496,7 +476,6 @@ class PreparedStatementPlanner():
         query = self.planner.query
 
         if params is not None:
-
             if len(params) != len(stmt.params):
                 raise PlanningException("Count of execution parameters don't match prepared statement")
 
@@ -508,12 +487,14 @@ class PreparedStatementPlanner():
         stmt.params = None
 
         if (
-                isinstance(query, ast.Select)
-                or isinstance(query, ast.Union)
-                or isinstance(query, ast.CreateTable)
-                or isinstance(query, ast.Insert)
-                or isinstance(query, ast.Update)
-                or isinstance(query, ast.Delete)
+            isinstance(query, ast.Select)
+            or isinstance(query, ast.Union)
+            or isinstance(query, ast.CreateTable)
+            or isinstance(query, ast.Insert)
+            or isinstance(query, ast.Update)
+            or isinstance(query, ast.Delete)
+            or isinstance(query, ast.Intersect)
+            or isinstance(query, ast.Except)
         ):
             return self.plan_query(query)
         else:

mindsdb/api/executor/sql_query/steps/fetch_dataframe.py

@@ -6,6 +6,8 @@ from mindsdb_sql_parser.ast import (
     Parameter,
     BinaryOperation,
     Tuple,
+    Union,
+    Intersect,
 )
 
 from mindsdb.api.executor.planner.steps import FetchDataframeStep
@@ -92,7 +94,10 @@ class FetchDataframeStepCall(BaseStepCall):
             response: DataHubResponse = dn.query(native_query=step.raw_query, session=self.session)
             df = response.data_frame
         else:
-            table_alias = get_table_alias(step.query.from_table, self.context.get("database"))
+            if isinstance(step.query, (Union, Intersect)):
+                table_alias = ["", "", ""]
+            else:
+                table_alias = get_table_alias(step.query.from_table, self.context.get("database"))
 
             # TODO for information_schema we have 'database' = 'mindsdb'
 

mindsdb/api/executor/sql_query/steps/union_step.py

@@ -9,7 +9,6 @@ from .base import BaseStepCall
 
 
 class UnionStepCall(BaseStepCall):
-
     bind = UnionStep
 
     def call(self, step):
@@ -19,7 +18,8 @@ class UnionStepCall(BaseStepCall):
         # count of columns have to match
         if len(left_result.columns) != len(right_result.columns):
             raise WrongArgumentError(
-                f'UNION columns count mismatch: {len(left_result.columns)} != {len(right_result.columns)} ')
+                f"UNION columns count mismatch: {len(left_result.columns)} != {len(right_result.columns)} "
+            )
 
         # types have to match
         # TODO: return checking type later
@@ -33,19 +33,21 @@ class UnionStepCall(BaseStepCall):
         table_a, names = left_result.to_df_cols()
         table_b, _ = right_result.to_df_cols()
 
-        op = 'UNION ALL'
-        if step.unique:
-            op = 'UNION'
+        if step.operation.lower() == "intersect":
+            op = "INTERSECT"
+        else:
+            op = "UNION"
+
+        if step.unique is not True:
+            op += " ALL"
+
         query = f"""
             SELECT * FROM table_a
             {op}
             SELECT * FROM table_b
         """
 
-        resp_df, _description = query_df_with_type_infer_fallback(query, {
-            'table_a': table_a,
-            'table_b': table_b
-        })
+        resp_df, _description = query_df_with_type_infer_fallback(query, {"table_a": table_a, "table_b": table_b})
         resp_df.replace({np.nan: None}, inplace=True)
 
         return ResultSet.from_df_cols(df=resp_df, columns_dict=names)

mindsdb/api/executor/utilities/sql.py

@@ -6,13 +6,14 @@ from duckdb import InvalidInputException
 import numpy as np
 
 from mindsdb_sql_parser import parse_sql
-from mindsdb.utilities.render.sqlalchemy_render import SqlalchemyRender
-from mindsdb.integrations.utilities.query_traversal import query_traversal
 from mindsdb_sql_parser.ast import ASTNode, Select, Identifier, Function, Constant
-from mindsdb.utilities.functions import resolve_table_identifier, resolve_model_identifier
 
+from mindsdb.integrations.utilities.query_traversal import query_traversal
 from mindsdb.utilities import log
+from mindsdb.utilities.exception import format_db_error_message
+from mindsdb.utilities.functions import resolve_table_identifier, resolve_model_identifier
 from mindsdb.utilities.json_encoder import CustomJSONEncoder
+from mindsdb.utilities.render.sqlalchemy_render import SqlalchemyRender
 
 logger = log.getLogger(__name__)
 
@@ -64,29 +65,85 @@ def query_df_with_type_infer_fallback(query_str: str, dataframes: dict, user_fun
         pandas.columns
     """
 
-    with duckdb.connect(database=":memory:") as con:
-        if user_functions:
-            user_functions.register(con)
-
-        for name, value in dataframes.items():
-            con.register(name, value)
-
-        exception = None
-        for sample_size in [1000, 10000, 1000000]:
-            try:
-                con.execute(f"set global pandas_analyze_sample={sample_size};")
-                result_df = con.execute(query_str).fetchdf()
-            except InvalidInputException as e:
-                exception = e
+    try:
+        with duckdb.connect(database=":memory:") as con:
+            if user_functions:
+                user_functions.register(con)
+
+            for name, value in dataframes.items():
+                con.register(name, value)
+
+            exception = None
+            for sample_size in [1000, 10000, 1000000]:
+                try:
+                    con.execute(f"set global pandas_analyze_sample={sample_size};")
+                    result_df = con.execute(query_str).fetchdf()
+                except InvalidInputException as e:
+                    exception = e
+                else:
+                    break
             else:
-                break
-        else:
-            raise exception
-        description = con.description
+                raise exception
+            description = con.description
+    except Exception as e:
+        raise Exception(
+            format_db_error_message(db_type="DuckDB", db_error_msg=str(e), failed_query=query_str, is_external=False)
+        ) from e
 
     return result_df, description
 
 
+_duckdb_functions_and_kw_list = None
+
+
+def get_duckdb_functions_and_kw_list() -> list[str] | None:
+    """Returns a list of all functions and keywords supported by DuckDB.
+    The list is merge of:
+     - list of duckdb's functions: 'select * from duckdb_functions()' or 'pragma functions'
+     - ist of keywords, because of some functions are just sintax-sugar
+       and not present in the duckdb_functions (like 'if()').
+     - hardcoded list of window_functions, because there are no way to get if from duckdb,
+       and they are not present in the duckdb_functions()
+
+    Returns:
+        list[str] | None: List of supported functions and keywords, or None if unable to retrieve the list.
+    """
+    global _duckdb_functions_and_kw_list
+    window_functions_list = [
+        "cume_dist",
+        "dense_rank",
+        "first_value",
+        "lag",
+        "last_value",
+        "lead",
+        "nth_value",
+        "ntile",
+        "percent_rank",
+        "rank_dense",
+        "rank",
+        "row_number",
+    ]
+    if _duckdb_functions_and_kw_list is None:
+        try:
+            df, _ = query_df_with_type_infer_fallback(
+                """
+                select distinct name
+                from (
+                    select function_name as name from duckdb_functions()
+                    union all
+                    select keyword_name as name from duckdb_keywords()
+                ) ta;
+            """,
+                dataframes={},
+            )
+            df.columns = [name.lower() for name in df.columns]
+            _duckdb_functions_and_kw_list = df["name"].drop_duplicates().str.lower().to_list() + window_functions_list
+        except Exception as e:
+            logger.warning(f"Unable to get DuckDB functions list: {e}")
+
+    return _duckdb_functions_and_kw_list
+
+
 def query_df(df, query, session=None):
     """Perform simple query ('select' from one table, without subqueries and joins) on DataFrame.
 
@@ -100,8 +157,10 @@ def query_df(df, query, session=None):
 
     if isinstance(query, str):
         query_ast = parse_sql(query)
+        query_str = query
     else:
         query_ast = copy.deepcopy(query)
+        query_str = str(query)
 
     if isinstance(query_ast, Select) is False or isinstance(query_ast.from_table, Identifier) is False:
         raise Exception("Only 'SELECT from TABLE' statements supported for internal query")
@@ -125,6 +184,7 @@ def query_df(df, query, session=None):
                 return node
         if isinstance(node, Function):
             fnc_name = node.op.lower()
+
             if fnc_name == "database" and len(node.args) == 0:
                 if session is not None:
                     cur_db = session.database
@@ -142,6 +202,22 @@ def query_df(df, query, session=None):
                 if user_functions is not None:
                     user_functions.check_function(node)
 
+            duckdb_functions_and_kw_list = get_duckdb_functions_and_kw_list() or []
+            custom_functions_list = [] if user_functions is None else list(user_functions.functions.keys())
+            all_functions_list = duckdb_functions_and_kw_list + custom_functions_list
+            if len(all_functions_list) > 0 and fnc_name not in all_functions_list:
+                raise Exception(
+                    format_db_error_message(
+                        db_type="DuckDB",
+                        db_error_msg=(
+                            f"Unknown function: '{fnc_name}'. This function is not recognized during internal query processing.\n"
+                            "Please use DuckDB-supported functions instead."
+                        ),
+                        failed_query=query_str,
+                        is_external=False,
+                    )
+                )
+
     query_traversal(query_ast, adapt_query)
 
     # convert json columns