MindsDB 25.6.4.0__py3-none-any.whl → 25.7.1.0__py3-none-any.whl

mindsdb/__about__.py

@@ -1,6 +1,6 @@
 __title__ = "MindsDB"
 __package_name__ = "mindsdb"
-__version__ = "25.6.4.0"
+__version__ = "25.7.1.0"
 __description__ = "MindsDB's AI SQL Server enables developers to build AI tools that need access to real-time data to perform their tasks"
 __email__ = "jorge@mindsdb.com"
 __author__ = "MindsDB Inc"

mindsdb/api/executor/command_executor.py

@@ -36,6 +36,8 @@ from mindsdb_sql_parser.ast import (
     Tuple,
     Function,
     Variable,
+    Intersect,
+    Except,
 )
 
 # typed models
@@ -580,9 +582,6 @@ class ExecuteCommands:
                 return ret
             query = SQLQuery(statement, session=self.session, database=database_name)
             return self.answer_select(query)
-        elif statement_type is Union:
-            query = SQLQuery(statement, session=self.session, database=database_name)
-            return self.answer_select(query)
         elif statement_type is Explain:
             return self.answer_show_columns(statement.target, database_name=database_name)
         elif statement_type is CreateTable:
@@ -627,6 +626,9 @@ class ExecuteCommands:
             return self.answer_create_kb_index(statement, database_name)
         elif statement_type is EvaluateKnowledgeBase:
             return self.answer_evaluate_kb(statement, database_name)
+        elif statement_type in (Union, Intersect, Except):
+            query = SQLQuery(statement, session=self.session, database=database_name)
+            return self.answer_select(query)
         else:
             logger.warning(f"Unknown SQL statement: {sql}")
             raise NotSupportedYet(f"Unknown SQL statement: {sql}")
@@ -1554,9 +1556,9 @@ class ExecuteCommands:
         if is_full:
             targets.extend(
                 [
-                    Constant("COLLATION", alias=Identifier("Collation")),
-                    Constant("PRIVILEGES", alias=Identifier("Privileges")),
-                    Constant("COMMENT", alias=Identifier("Comment")),
+                    Constant(None, alias=Identifier("Collation")),
+                    Constant("select", alias=Identifier("Privileges")),
+                    Constant(None, alias=Identifier("Comment")),
                 ]
             )
         new_statement = Select(

mindsdb/api/executor/datahub/datanodes/information_schema_datanode.py

@@ -177,7 +177,7 @@ class InformationSchemaDataNode(DataNode):
         if table_name not in self.tables:
             raise exc.TableNotExistError(f"Table information_schema.{table_name} does not exists")
         table_columns_names = self.tables[table_name].columns
-        df = pd.DataFrame([[table_columns_names]], columns=[INF_SCHEMA_COLUMNS_NAMES.COLUMN_NAME])
+        df = pd.DataFrame(pd.Series(table_columns_names, name=INF_SCHEMA_COLUMNS_NAMES.COLUMN_NAME))
         for column_name in astuple(INF_SCHEMA_COLUMNS_NAMES):
             if column_name == INF_SCHEMA_COLUMNS_NAMES.COLUMN_NAME:
                 continue

mindsdb/api/executor/datahub/datanodes/integration_datanode.py

@@ -1,6 +1,5 @@
 import time
 import inspect
-from textwrap import dedent
 from dataclasses import astuple
 from typing import Iterable, List
 
@@ -21,6 +20,7 @@ from mindsdb.integrations.utilities.utils import get_class_name
 from mindsdb.metrics import metrics
 from mindsdb.utilities import log
 from mindsdb.utilities.profiler import profiler
+from mindsdb.utilities.exception import format_db_error_message
 from mindsdb.api.executor.datahub.datanodes.system_tables import infer_mysql_type
 
 logger = log.getLogger(__name__)
@@ -244,18 +244,16 @@ class IntegrationDataNode(DataNode):
             failed_sql_query = native_query
             if query is not None:
                 failed_sql_query = query.to_string()
+
             raise Exception(
-                dedent(f"""\
-                Failed to execute external database query during query processing.
-                
-                Database Details:
-                - Name: {self.integration_handler.name}
-                - Type: {self.integration_handler.__class__.name}
-                
-                Error: {result.error_message}
-                Failed Query: {failed_sql_query}
-            """)
+                format_db_error_message(
+                    db_name=self.integration_handler.name,
+                    db_type=self.integration_handler.__class__.name,
+                    db_error_msg=result.error_message,
+                    failed_query=failed_sql_query,
+                )
             )
+
         if result.type == RESPONSE_TYPE.OK:
             return DataHubResponse(affected_rows=result.affected_rows)
 

mindsdb/api/executor/datahub/datanodes/system_tables.py

@@ -623,7 +623,7 @@ class MetaColumnStatisticsTable(Table):
             columns = record.meta_columns
 
             for column in columns:
-                column_statistics = column.meta_column_statistics[0]
+                column_statistics = column.meta_column_statistics[0] if column.meta_column_statistics else None
 
                 item = {
                     "TABLE_SCHEMA": database_name,

mindsdb/api/executor/planner/query_prepare.py

@@ -8,7 +8,7 @@ from mindsdb.api.executor.planner import utils
 
 def to_string(identifier):
     # alternative to AST.to_string() but without quoting
-    return '.'.join(identifier.parts)
+    return ".".join(identifier.parts)
 
 
 class Table:
@@ -32,7 +32,6 @@ class Column:
     def __init__(self, node=None, table=None, name=None, type=None):
         alias = None
         if node is not None:
-
             if isinstance(node, ast.Identifier):
                 # set name
                 name = node.parts[-1]  # ???
@@ -67,26 +66,25 @@ class Statement:
         self.offset = 0
 
 
-class PreparedStatementPlanner():
-
+class PreparedStatementPlanner:
     def __init__(self, planner):
         self.planner = planner
 
     def get_type_of_var(self, v):
         if isinstance(v, str):
-            return 'str'
+            return "str"
         elif isinstance(v, float):
-            return 'float'
+            return "float"
         elif isinstance(v, int):
-            return 'integer'
+            return "integer"
 
-        return 'str'
+        return "str"
 
     def get_statement_info(self):
         stmt = self.planner.statement
 
         if stmt is None:
-            raise PlanningException('Statement is not prepared')
+            raise PlanningException("Statement is not prepared")
 
         columns_result = []
 
@@ -95,45 +93,45 @@ class PreparedStatementPlanner():
             if column.table is not None:
                 table = column.table.name
                 ds = column.table.ds
-            columns_result.append(dict(
-                alias=column.alias,
-                type=column.type,
-                name=column.name,
-                table_name=table,
-                table_alias=table,
-                ds=ds,
-            ))
+            columns_result.append(
+                dict(
+                    alias=column.alias,
+                    type=column.type,
+                    name=column.name,
+                    table_name=table,
+                    table_alias=table,
+                    ds=ds,
+                )
+            )
 
         parameters = []
         for param in stmt.params:
-            name = '?'
-            parameters.append(dict(
-                alias=name,
-                type='str',
-                name=name,
-            ))
-
-        return {
-            'parameters': parameters,
-            'columns': columns_result
-        }
+            name = "?"
+            parameters.append(
+                dict(
+                    alias=name,
+                    type="str",
+                    name=name,
+                )
+            )
 
-    def get_table_of_column(self, t):
+        return {"parameters": parameters, "columns": columns_result}
 
+    def get_table_of_column(self, t):
         tables_map = self.planner.statement.tables_map
 
         # get tables to check
         if len(t.parts) > 1:
             # try to find table
             table_parts = t.parts[:-1]
-            table_name = '.'.join(table_parts)
+            table_name = ".".join(table_parts)
             if table_name in tables_map:
                 return tables_map[table_name]
 
             elif len(table_parts) > 1:
                 # maybe datasource is 1st part
                 table_parts = table_parts[1:]
-                table_name = '.'.join(table_parts)
+                table_name = ".".join(table_parts)
                 if table_name in tables_map:
                     return tables_map[table_name]
 
@@ -158,14 +156,10 @@ class PreparedStatementPlanner():
             # in reverse order
             for p in table.parts[::-1]:
                 parts.insert(0, p)
-                keys.append('.'.join(parts))
+                keys.append(".".join(parts))
 
         # remember table
-        tbl = Table(
-            ds=ds,
-            node=table,
-            is_predictor=is_predictor
-        )
+        tbl = Table(ds=ds, node=table, is_predictor=is_predictor)
         tbl.keys = keys
 
         return tbl
@@ -189,7 +183,6 @@ class PreparedStatementPlanner():
         stmt.tables_map = {}
         stmt.tables_lvl1 = []
         if query.from_table is not None:
-
             if isinstance(query.from_table, ast.Join):
                 # get all tables
                 join_tables = utils.convert_join_to_list(query.from_table)
@@ -198,21 +191,17 @@ class PreparedStatementPlanner():
 
             if isinstance(query.from_table, ast.Select):
                 # nested select, get only last select
-                join_tables = [
-                    dict(
-                        table=utils.get_deepest_select(query.from_table).from_table
-                    )
-                ]
+                join_tables = [dict(table=utils.get_deepest_select(query.from_table).from_table)]
 
             for i, join_table in enumerate(join_tables):
-                table = join_table['table']
+                table = join_table["table"]
                 if isinstance(table, ast.Identifier):
                     tbl = self.table_from_identifier(table)
 
                     if tbl.is_predictor:
                         # Is the last table?
                         if i + 1 < len(join_tables):
-                            raise PlanningException('Predictor must be last table in query')
+                            raise PlanningException("Predictor must be last table in query")
 
                     stmt.tables_lvl1.append(tbl)
                     for key in tbl.keys:
@@ -225,13 +214,12 @@ class PreparedStatementPlanner():
         # is there any predictors at other levels?
         lvl1_predictors = [i for i in stmt.tables_lvl1 if i.is_predictor]
         if len(query_predictors) != len(lvl1_predictors):
-            raise PlanningException('Predictor is not at first level')
+            raise PlanningException("Predictor is not at first level")
 
         # === get targets ===
         columns = []
         get_all_tables = False
         for t in query.targets:
-
             column = Column(t)
 
             # column alias
@@ -264,10 +252,10 @@ class PreparedStatementPlanner():
                 column.type = self.get_type_of_var(t.value)
             elif isinstance(t, ast.Function):
                 # mysql function
-                if t.op == 'connection_id':
-                    column.type = 'integer'
+                if t.op == "connection_id":
+                    column.type = "integer"
                 else:
-                    column.type = 'str'
+                    column.type = "str"
             else:
                 # TODO go down into lower level.
                 #  It can be function, operation, select.
@@ -276,7 +264,7 @@ class PreparedStatementPlanner():
                 # TODO add several known types for function, i.e ABS-int
 
                 # TODO TypeCast - as casted type
-                column.type = 'str'
+                column.type = "str"
 
             if alias is not None:
                 column.alias = alias
@@ -299,28 +287,25 @@ class PreparedStatementPlanner():
                 if step.result_data is not None:
                     # save results
 
-                    if len(step.result_data['tables']) > 0:
-                        table_info = step.result_data['tables'][0]
-                        columns_info = step.result_data['columns'][table_info]
+                    if len(step.result_data["tables"]) > 0:
+                        table_info = step.result_data["tables"][0]
+                        columns_info = step.result_data["columns"][table_info]
 
                         table.columns = []
                         table.ds = table_info[0]
                         for col in columns_info:
                             if isinstance(col, tuple):
                                 # is predictor
-                                col = dict(name=col[0], type='str')
+                                col = dict(name=col[0], type="str")
                             table.columns.append(
                                 Column(
-                                    name=col['name'],
-                                    type=col['type'],
+                                    name=col["name"],
+                                    type=col["type"],
                                 )
                             )
 
                     # map by names
-                    table.columns_map = {
-                        i.name.upper(): i
-                        for i in table.columns
-                    }
+                    table.columns_map = {i.name.upper(): i for i in table.columns}
 
         # === create columns list ===
         columns_result = []
@@ -329,7 +314,7 @@ class PreparedStatementPlanner():
                 # add data from all tables
                 for table in stmt.tables_lvl1:
                     if table.columns is None:
-                        raise PlanningException(f'Table is not found {table.name}')
+                        raise PlanningException(f"Table is not found {table.name}")
 
                     for col in table.columns:
                         # col = {name: 'col', type: 'str'}
@@ -354,7 +339,7 @@ class PreparedStatementPlanner():
                             column.type = table.columns_map[col_name].type
                         else:
                             # continue
-                            raise PlanningException(f'Column not found {col_name}')
+                            raise PlanningException(f"Column not found {col_name}")
 
                 else:
                     # table is not found, looking for in all tables
@@ -368,11 +353,11 @@ class PreparedStatementPlanner():
 
             # forcing alias
             if column.alias is None:
-                column.alias = f'column_{i}'
+                column.alias = f"column_{i}"
 
             # forcing type
             if column.type is None:
-                column.type = 'str'
+                column.type = "str"
 
             columns_result.append(column)
 
@@ -393,28 +378,25 @@ class PreparedStatementPlanner():
         if step.result_data is not None:
             # save results
 
-            if len(step.result_data['tables']) > 0:
-                table_info = step.result_data['tables'][0]
-                columns_info = step.result_data['columns'][table_info]
+            if len(step.result_data["tables"]) > 0:
+                table_info = step.result_data["tables"][0]
+                columns_info = step.result_data["columns"][table_info]
 
                 table.columns = []
                 table.ds = table_info[0]
                 for col in columns_info:
                     if isinstance(col, tuple):
                         # is predictor
-                        col = dict(name=col[0], type='str')
+                        col = dict(name=col[0], type="str")
                     table.columns.append(
                         Column(
-                            name=col['name'],
-                            type=col['type'],
+                            name=col["name"],
+                            type=col["type"],
                         )
                     )
 
                 # map by names
-                table.columns_map = {
-                    i.name.upper(): i
-                    for i in table.columns
-                }
+                table.columns_map = {i.name.upper(): i for i in table.columns}
 
         # save results
         columns_result = []
@@ -430,7 +412,7 @@ class PreparedStatementPlanner():
 
             if column.type is None:
                 # forcing type
-                column.type = 'str'
+                column.type = "str"
 
             columns_result.append(column)
 
@@ -440,13 +422,12 @@ class PreparedStatementPlanner():
         stmt = self.planner.statement
 
         stmt.columns = [
-            Column(name='Variable_name', type='str'),
-            Column(name='Value', type='str'),
+            Column(name="Variable_name", type="str"),
+            Column(name="Value", type="str"),
         ]
         return []
 
     def prepare_steps(self, query):
-
         stmt = Statement()
         self.planner.statement = stmt
 
@@ -476,7 +457,6 @@ class PreparedStatementPlanner():
         if isinstance(query, ast.Show):
             return self.prepare_show(query)
         else:
-
             # do nothing
             return []
             # raise NotImplementedError(query.__name__)
@@ -496,7 +476,6 @@ class PreparedStatementPlanner():
         query = self.planner.query
 
         if params is not None:
-
             if len(params) != len(stmt.params):
                 raise PlanningException("Count of execution parameters don't match prepared statement")
 
@@ -508,12 +487,14 @@ class PreparedStatementPlanner():
         stmt.params = None
 
         if (
-                isinstance(query, ast.Select)
-                or isinstance(query, ast.Union)
-                or isinstance(query, ast.CreateTable)
-                or isinstance(query, ast.Insert)
-                or isinstance(query, ast.Update)
-                or isinstance(query, ast.Delete)
+            isinstance(query, ast.Select)
+            or isinstance(query, ast.Union)
+            or isinstance(query, ast.CreateTable)
+            or isinstance(query, ast.Insert)
+            or isinstance(query, ast.Update)
+            or isinstance(query, ast.Delete)
+            or isinstance(query, ast.Intersect)
+            or isinstance(query, ast.Except)
         ):
             return self.plan_query(query)
         else:

mindsdb/api/executor/sql_query/steps/fetch_dataframe.py

@@ -6,6 +6,8 @@ from mindsdb_sql_parser.ast import (
     Parameter,
     BinaryOperation,
     Tuple,
+    Union,
+    Intersect,
 )
 
 from mindsdb.api.executor.planner.steps import FetchDataframeStep
@@ -92,7 +94,10 @@ class FetchDataframeStepCall(BaseStepCall):
             response: DataHubResponse = dn.query(native_query=step.raw_query, session=self.session)
             df = response.data_frame
         else:
-            table_alias = get_table_alias(step.query.from_table, self.context.get("database"))
+            if isinstance(step.query, (Union, Intersect)):
+                table_alias = ["", "", ""]
+            else:
+                table_alias = get_table_alias(step.query.from_table, self.context.get("database"))
 
             # TODO for information_schema we have 'database' = 'mindsdb'
 

mindsdb/api/executor/sql_query/steps/union_step.py

@@ -9,7 +9,6 @@ from .base import BaseStepCall
 
 
 class UnionStepCall(BaseStepCall):
-
     bind = UnionStep
 
     def call(self, step):
@@ -19,7 +18,8 @@ class UnionStepCall(BaseStepCall):
         # count of columns have to match
         if len(left_result.columns) != len(right_result.columns):
             raise WrongArgumentError(
-                f'UNION columns count mismatch: {len(left_result.columns)} != {len(right_result.columns)} ')
+                f"UNION columns count mismatch: {len(left_result.columns)} != {len(right_result.columns)} "
+            )
 
         # types have to match
         # TODO: return checking type later
@@ -33,19 +33,21 @@ class UnionStepCall(BaseStepCall):
         table_a, names = left_result.to_df_cols()
         table_b, _ = right_result.to_df_cols()
 
-        op = 'UNION ALL'
-        if step.unique:
-            op = 'UNION'
+        if step.operation.lower() == "intersect":
+            op = "INTERSECT"
+        else:
+            op = "UNION"
+
+        if step.unique is not True:
+            op += " ALL"
+
         query = f"""
             SELECT * FROM table_a
             {op}
             SELECT * FROM table_b
         """
 
-        resp_df, _description = query_df_with_type_infer_fallback(query, {
-            'table_a': table_a,
-            'table_b': table_b
-        })
+        resp_df, _description = query_df_with_type_infer_fallback(query, {"table_a": table_a, "table_b": table_b})
         resp_df.replace({np.nan: None}, inplace=True)
 
         return ResultSet.from_df_cols(df=resp_df, columns_dict=names)

mindsdb/api/http/namespaces/file.py

@@ -3,6 +3,7 @@ import shutil
 import tarfile
 import tempfile
 import zipfile
+from urllib.parse import urlparse
 
 import multipart
 import requests
@@ -13,7 +14,7 @@ from flask_restx import Resource
 from mindsdb.api.http.namespaces.configs.files import ns_conf
 from mindsdb.api.http.utils import http_error
 from mindsdb.metrics.metrics import api_endpoint_metrics
-from mindsdb.utilities.config import Config
+from mindsdb.utilities.config import config
 from mindsdb.utilities.context import context as ctx
 from mindsdb.utilities import log
 from mindsdb.utilities.security import is_private_url, clear_filename, validate_urls
@@ -105,31 +106,55 @@ class File(Resource):
 
         if data.get("source_type") == "url":
             url = data["source"]
-            config = Config()
-            allowed_urls = config.get("file_upload_domains", [])
-            if allowed_urls and not validate_urls(url, allowed_urls):
-                return http_error(400, "Invalid File URL source.", f"Allowed hosts are: {', '.join(allowed_urls)}.")
+            try:
+                url = urlparse(url)
+                if not (url.scheme and url.netloc):
+                    raise ValueError()
+                url = url.geturl()
+            except Exception:
+                return http_error(
+                    400,
+                    "Invalid URL",
+                    f"The URL is not valid: {data['source']}",
+                )
+
+            url_file_upload_enabled = config["url_file_upload"]["enabled"]
+            if url_file_upload_enabled is False:
+                return http_error(400, "URL file upload is disabled.", "URL file upload is disabled.")
+
+            allowed_origins = config["url_file_upload"]["allowed_origins"]
+            disallowed_origins = config["url_file_upload"]["disallowed_origins"]
+
+            if validate_urls(url, allowed_origins, disallowed_origins) is False:
+                return http_error(
+                    400,
+                    "Invalid URL",
+                    "URL is not allowed for security reasons. Allowed hosts are: "
+                    f"{', '.join(allowed_origins) if allowed_origins else 'not specified'}.",
+                )
+
             data["file"] = clear_filename(data["name"])
             is_cloud = config.get("cloud", False)
-            if is_cloud and is_private_url(url):
-                return http_error(400, f"URL is private: {url}")
-
-            if is_cloud is True and ctx.user_class != 1:
-                info = requests.head(url)
-                file_size = info.headers.get("Content-Length")
-                try:
-                    file_size = int(file_size)
-                except Exception:
-                    pass
-
-                if file_size is None:
-                    return http_error(
-                        400,
-                        "Error getting file info",
-                        "Сan't determine remote file size",
-                    )
-                if file_size > MAX_FILE_SIZE:
-                    return http_error(400, "File is too big", f"Upload limit for file is {MAX_FILE_SIZE >> 20} MB")
+            if is_cloud:
+                if is_private_url(url):
+                    return http_error(400, f"URL is private: {url}")
+
+                if ctx.user_class != 1:
+                    info = requests.head(url, timeout=30)
+                    file_size = info.headers.get("Content-Length")
+                    try:
+                        file_size = int(file_size)
+                    except Exception:
+                        pass
+
+                    if file_size is None:
+                        return http_error(
+                            400,
+                            "Error getting file info",
+                            "Сan't determine remote file size",
+                        )
+                    if file_size > MAX_FILE_SIZE:
+                        return http_error(400, "File is too big", f"Upload limit for file is {MAX_FILE_SIZE >> 20} MB")
             with requests.get(url, stream=True) as r:
                 if r.status_code != 200:
                     return http_error(400, "Error getting file", f"Got status code: {r.status_code}")