MindsDB 25.6.3.1__py3-none-any.whl → 25.7.1.0__py3-none-any.whl

mindsdb/__about__.py

@@ -1,6 +1,6 @@
 __title__ = "MindsDB"
 __package_name__ = "mindsdb"
-__version__ = "25.6.3.1"
+__version__ = "25.7.1.0"
 __description__ = "MindsDB's AI SQL Server enables developers to build AI tools that need access to real-time data to perform their tasks"
 __email__ = "jorge@mindsdb.com"
 __author__ = "MindsDB Inc"

mindsdb/api/executor/command_executor.py

@@ -36,6 +36,8 @@ from mindsdb_sql_parser.ast import (
     Tuple,
     Function,
     Variable,
+    Intersect,
+    Except,
 )
 
 # typed models
@@ -580,9 +582,6 @@ class ExecuteCommands:
                 return ret
             query = SQLQuery(statement, session=self.session, database=database_name)
             return self.answer_select(query)
-        elif statement_type is Union:
-            query = SQLQuery(statement, session=self.session, database=database_name)
-            return self.answer_select(query)
         elif statement_type is Explain:
             return self.answer_show_columns(statement.target, database_name=database_name)
         elif statement_type is CreateTable:
@@ -627,6 +626,9 @@ class ExecuteCommands:
             return self.answer_create_kb_index(statement, database_name)
         elif statement_type is EvaluateKnowledgeBase:
             return self.answer_evaluate_kb(statement, database_name)
+        elif statement_type in (Union, Intersect, Except):
+            query = SQLQuery(statement, session=self.session, database=database_name)
+            return self.answer_select(query)
         else:
             logger.warning(f"Unknown SQL statement: {sql}")
             raise NotSupportedYet(f"Unknown SQL statement: {sql}")
@@ -1554,9 +1556,9 @@ class ExecuteCommands:
         if is_full:
             targets.extend(
                 [
-                    Constant("COLLATION", alias=Identifier("Collation")),
-                    Constant("PRIVILEGES", alias=Identifier("Privileges")),
-                    Constant("COMMENT", alias=Identifier("Comment")),
+                    Constant(None, alias=Identifier("Collation")),
+                    Constant("select", alias=Identifier("Privileges")),
+                    Constant(None, alias=Identifier("Comment")),
                 ]
             )
         new_statement = Select(

mindsdb/api/executor/datahub/datanodes/information_schema_datanode.py

@@ -15,12 +15,38 @@ from mindsdb.integrations.libs.response import INF_SCHEMA_COLUMNS_NAMES
 from mindsdb.utilities import log
 
 from .system_tables import (
-    SchemataTable, TablesTable, ColumnsTable, EventsTable, RoutinesTable,
-    PluginsTable, EnginesTable, KeyColumnUsageTable, StatisticsTable,
-    CharacterSetsTable, CollationsTable)
+    SchemataTable,
+    TablesTable,
+    MetaTablesTable,
+    ColumnsTable,
+    MetaColumnsTable,
+    EventsTable,
+    RoutinesTable,
+    PluginsTable,
+    EnginesTable,
+    MetaTableConstraintsTable,
+    KeyColumnUsageTable,
+    MetaColumnUsageTable,
+    StatisticsTable,
+    MetaColumnStatisticsTable,
+    CharacterSetsTable,
+    CollationsTable,
+    MetaHandlerInfoTable,
+)
 from .mindsdb_tables import (
-    ModelsTable, DatabasesTable, MLEnginesTable, HandlersTable, JobsTable, QueriesTable,
-    ChatbotsTable, KBTable, SkillsTable, AgentsTable, ViewsTable, TriggersTable)
+    ModelsTable,
+    DatabasesTable,
+    MLEnginesTable,
+    HandlersTable,
+    JobsTable,
+    QueriesTable,
+    ChatbotsTable,
+    KBTable,
+    SkillsTable,
+    AgentsTable,
+    ViewsTable,
+    TriggersTable,
+)
 
 from mindsdb.api.executor.datahub.classes.tables_row import TablesRow
 
@@ -32,12 +58,35 @@ class InformationSchemaDataNode(DataNode):
     type = "INFORMATION_SCHEMA"
 
     tables_list = [
-        SchemataTable, TablesTable, ColumnsTable, EventsTable, RoutinesTable,
-        PluginsTable, EnginesTable, KeyColumnUsageTable, StatisticsTable,
-        CharacterSetsTable, CollationsTable,
-        ModelsTable, DatabasesTable, MLEnginesTable, HandlersTable, JobsTable,
-        ChatbotsTable, KBTable, SkillsTable, AgentsTable, ViewsTable, TriggersTable,
-        QueriesTable
+        SchemataTable,
+        TablesTable,
+        MetaTablesTable,
+        ColumnsTable,
+        MetaColumnsTable,
+        EventsTable,
+        RoutinesTable,
+        PluginsTable,
+        EnginesTable,
+        MetaTableConstraintsTable,
+        KeyColumnUsageTable,
+        MetaColumnUsageTable,
+        StatisticsTable,
+        MetaColumnStatisticsTable,
+        CharacterSetsTable,
+        CollationsTable,
+        ModelsTable,
+        DatabasesTable,
+        MLEnginesTable,
+        HandlersTable,
+        JobsTable,
+        ChatbotsTable,
+        KBTable,
+        SkillsTable,
+        AgentsTable,
+        ViewsTable,
+        TriggersTable,
+        QueriesTable,
+        MetaHandlerInfoTable,
     ]
 
     def __init__(self, session):
@@ -46,9 +95,7 @@ class InformationSchemaDataNode(DataNode):
         self.project_controller = ProjectController()
         self.database_controller = session.database_controller
 
-        self.persis_datanodes = {
-            'log': self.database_controller.logs_db_controller
-        }
+        self.persis_datanodes = {"log": self.database_controller.logs_db_controller}
 
         databases = self.database_controller.get_dict()
         if "files" in databases:
@@ -69,15 +116,13 @@ class InformationSchemaDataNode(DataNode):
         if name_lower == "information_schema":
             return self
 
-        if name_lower == 'log':
-            return self.database_controller.get_system_db('log')
+        if name_lower == "log":
+            return self.database_controller.get_system_db("log")
 
         if name_lower in self.persis_datanodes:
             return self.persis_datanodes[name_lower]
 
-        existing_databases_meta = (
-            self.database_controller.get_dict()
-        )  # filter_type='project'
+        existing_databases_meta = self.database_controller.get_dict()  # filter_type='project'
         database_name = None
         for key in existing_databases_meta:
             if key.lower() == name_lower:
@@ -130,11 +175,9 @@ class InformationSchemaDataNode(DataNode):
         """
         table_name = table_name.upper()
         if table_name not in self.tables:
-            raise exc.TableNotExistError(
-                f"Table information_schema.{table_name} does not exists"
-            )
+            raise exc.TableNotExistError(f"Table information_schema.{table_name} does not exists")
         table_columns_names = self.tables[table_name].columns
-        df = pd.DataFrame([[table_columns_names]], columns=[INF_SCHEMA_COLUMNS_NAMES.COLUMN_NAME])
+        df = pd.DataFrame(pd.Series(table_columns_names, name=INF_SCHEMA_COLUMNS_NAMES.COLUMN_NAME))
         for column_name in astuple(INF_SCHEMA_COLUMNS_NAMES):
             if column_name == INF_SCHEMA_COLUMNS_NAMES.COLUMN_NAME:
                 continue
@@ -153,9 +196,7 @@ class InformationSchemaDataNode(DataNode):
         """
         table_name = table_name.upper()
         if table_name not in self.tables:
-            raise exc.TableNotExistError(
-                f"Table information_schema.{table_name} does not exists"
-            )
+            raise exc.TableNotExistError(f"Table information_schema.{table_name} does not exists")
         return self.tables[table_name].columns
 
     def get_integrations_names(self):
@@ -168,25 +209,16 @@ class InformationSchemaDataNode(DataNode):
         return [x.lower() for x in projects]
 
     def get_tables(self):
-        return [
-            TablesRow(TABLE_NAME=name)
-            for name in self.tables.keys()
-        ]
+        return [TablesRow(TABLE_NAME=name) for name in self.tables.keys()]
 
     def get_tree_tables(self):
-        return {
-            name: table
-            for name, table in self.tables.items()
-            if table.visible
-        }
+        return {name: table for name, table in self.tables.items() if table.visible}
 
     def query(self, query: ASTNode, session=None) -> DataHubResponse:
         query_tables = [x[1] for x in get_query_tables(query)]
 
         if len(query_tables) != 1:
-            raise exc.BadTableError(
-                f"Only one table can be used in query to information_schema: {query}"
-            )
+            raise exc.BadTableError(f"Only one table can be used in query to information_schema: {query}")
 
         table_name = query_tables[0].upper()
 
@@ -195,7 +227,7 @@ class InformationSchemaDataNode(DataNode):
 
         tbl = self.tables[table_name]
 
-        if hasattr(tbl, 'get_data'):
+        if hasattr(tbl, "get_data"):
             dataframe = tbl.get_data(query=query, inf_schema=self, session=self.session)
         else:
             dataframe = self._get_empty_table(tbl)
@@ -203,11 +235,7 @@ class InformationSchemaDataNode(DataNode):
 
         columns_info = [{"name": k, "type": v} for k, v in data.dtypes.items()]
 
-        return DataHubResponse(
-            data_frame=data,
-            columns=columns_info,
-            affected_rows=0
-        )
+        return DataHubResponse(data_frame=data, columns=columns_info, affected_rows=0)
 
     def _get_empty_table(self, table):
         columns = table.columns

mindsdb/api/executor/datahub/datanodes/integration_datanode.py

@@ -20,6 +20,7 @@ from mindsdb.integrations.utilities.utils import get_class_name
 from mindsdb.metrics import metrics
 from mindsdb.utilities import log
 from mindsdb.utilities.profiler import profiler
+from mindsdb.utilities.exception import format_db_error_message
 from mindsdb.api.executor.datahub.datanodes.system_tables import infer_mysql_type
 
 logger = log.getLogger(__name__)
@@ -240,7 +241,19 @@ class IntegrationDataNode(DataNode):
             raise DBHandlerException(msg) from e
 
         if result.type == RESPONSE_TYPE.ERROR:
-            raise Exception(f"Error in {self.integration_name}: {result.error_message}")
+            failed_sql_query = native_query
+            if query is not None:
+                failed_sql_query = query.to_string()
+
+            raise Exception(
+                format_db_error_message(
+                    db_name=self.integration_handler.name,
+                    db_type=self.integration_handler.__class__.name,
+                    db_error_msg=result.error_message,
+                    failed_query=failed_sql_query,
+                )
+            )
+
         if result.type == RESPONSE_TYPE.OK:
             return DataHubResponse(affected_rows=result.affected_rows)
 

mindsdb/api/executor/datahub/datanodes/project_datanode.py

@@ -154,7 +154,7 @@ class ProjectDataNode(DataNode):
 
                 return DataHubResponse(data_frame=df, columns=columns_info)
 
-            raise EntityNotExistsError(f"Can't select from <{query_table}> in project")
+            raise EntityNotExistsError(f"Table '{query_table}' not found in database", self.project.name)
         else:
             raise NotImplementedError(f"Query not supported {query}")
 

mindsdb/api/executor/datahub/datanodes/system_tables.py

@@ -1,4 +1,4 @@
-from typing import Optional, Literal
+from typing import Optional, List, Literal
 from dataclasses import dataclass, fields
 
 import pandas as pd
@@ -8,6 +8,7 @@ from mindsdb.utilities import log
 from mindsdb.utilities.config import config
 from mindsdb.integrations.utilities.sql_utils import extract_comparison_conditions
 from mindsdb.integrations.libs.response import INF_SCHEMA_COLUMNS_NAMES
+from mindsdb.interfaces.data_catalog.data_catalog_reader import DataCatalogReader
 from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import MYSQL_DATA_TYPE, MYSQL_DATA_TYPE_COLUMNS_DEFAULT
 from mindsdb.api.executor.datahub.classes.tables_row import TABLES_ROW_TYPE, TablesRow
 
@@ -503,3 +504,315 @@ class CollationsTable(Table):
 
         df = pd.DataFrame(data, columns=cls.columns)
         return df
+
+
+# Data Catalog tables
+# TODO: Should these be placed in a separate schema?
+
+
+def _get_records_from_data_catalog(databases: List, tables: Optional[List[str]] = None) -> List:
+    """Get records from the data catalog based on the specified databases and tables."""
+    # TODO: Should we allow to query all databases?
+    if not databases:
+        raise ValueError("At least one database must be specified in the query.")
+
+    records = []
+    for database in databases:
+        data_catalog_reader = DataCatalogReader(database_name=database, table_names=tables)
+        records.extend(data_catalog_reader.read_metadata_as_records())
+
+    return records
+
+
+# TODO: Combine with existing 'TablesTable'?
+class MetaTablesTable(Table):
+    name = "META_TABLES"
+
+    columns = ["TABLE_CATALOG", "TABLE_SCHEMA", "TABLE_NAME", "TABLE_TYPE", "TABLE_DESCRIPTION", "ROW_COUNT"]
+
+    @classmethod
+    def get_data(cls, query: ASTNode = None, inf_schema=None, **kwargs):
+        databases, _ = _get_scope(query)
+
+        records = _get_records_from_data_catalog(databases)
+
+        data = []
+        for record in records:
+            item = {
+                "TABLE_CATALOG": "def",
+                "TABLE_SCHEMA": record.integration.name,
+                "TABLE_NAME": record.name,
+                "TABLE_TYPE": record.type,
+                "TABLE_DESCRIPTION": record.description or "",
+                "ROW_COUNT": record.row_count,
+            }
+            data.append(item)
+
+        df = pd.DataFrame(data, columns=cls.columns)
+        return df
+
+
+# TODO: Combine with existing 'ColumnsTable'?
+class MetaColumnsTable(Table):
+    name = "META_COLUMNS"
+
+    columns = [
+        "TABLE_CATALOG",
+        "TABLE_SCHEMA",
+        "TABLE_NAME",
+        "COLUMN_NAME",
+        "DATA_TYPE",
+        "COLUMN_DESCRIPTION",
+        "COLUMN_DEFAULT",
+        "IS_NULLABLE",
+    ]
+
+    @classmethod
+    def get_data(cls, query: ASTNode = None, inf_schema=None, **kwargs):
+        databases, tables = _get_scope(query)
+
+        records = _get_records_from_data_catalog(databases, tables)
+
+        data = []
+        for record in records:
+            database_name = record.integration.name
+            table_name = record.name
+            columns = record.meta_columns
+
+            for column in columns:
+                item = {
+                    "TABLE_CATALOG": "def",
+                    "TABLE_SCHEMA": database_name,
+                    "TABLE_NAME": table_name,
+                    "COLUMN_NAME": column.name,
+                    "DATA_TYPE": column.data_type,
+                    "COLUMN_DESCRIPTION": column.description or "",
+                    "COLUMN_DEFAULT": column.default_value,
+                    "IS_NULLABLE": "YES" if column.is_nullable else "NO",
+                }
+                data.append(item)
+
+        df = pd.DataFrame(data, columns=cls.columns)
+        return df
+
+
+class MetaColumnStatisticsTable(Table):
+    name = "META_COLUMN_STATISTICS"
+    columns = [
+        "TABLE_SCHEMA",
+        "TABLE_NAME",
+        "COLUMN_NAME",
+        "MOST_COMMON_VALS",
+        "MOST_COMMON_FREQS",
+        "NULL_FRAC",
+        "N_DISTINCT",
+        "MIN_VALUE",
+        "MAX_VALUE",
+    ]
+
+    @classmethod
+    def get_data(cls, query: ASTNode = None, inf_schema=None, **kwargs):
+        databases, tables = _get_scope(query)
+
+        records = _get_records_from_data_catalog(databases, tables)
+
+        data = []
+        for record in records:
+            database_name = record.integration.name
+            table_name = record.name
+            columns = record.meta_columns
+
+            for column in columns:
+                column_statistics = column.meta_column_statistics[0] if column.meta_column_statistics else None
+
+                item = {
+                    "TABLE_SCHEMA": database_name,
+                    "TABLE_NAME": table_name,
+                    "COLUMN_NAME": column.name,
+                }
+
+                if column_statistics:
+                    item.update(
+                        {
+                            "MOST_COMMON_VALS": column_statistics.most_common_values,
+                            "MOST_COMMON_FREQS": column_statistics.most_common_frequencies,
+                            "NULL_FRAC": column_statistics.null_percentage,
+                            "N_DISTINCT": column_statistics.distinct_values_count,
+                            "MIN_VALUE": column_statistics.minimum_value,
+                            "MAX_VALUE": column_statistics.maximum_value,
+                        }
+                    )
+
+                data.append(item)
+
+        df = pd.DataFrame(data, columns=cls.columns)
+        return df
+
+
+class MetaTableConstraintsTable(Table):
+    name = "META_TABLE_CONSTRAINTS"
+    columns = [
+        "CONSTRAINT_CATALOG",
+        "CONSTRAINT_SCHEMA",
+        "CONSTRAINT_NAME",
+        "TABLE_SCHEMA",
+        "TABLE_NAME",
+        "CONSTRAINT_TYPE",
+        "ENFORCED",
+    ]
+
+    @classmethod
+    def get_data(cls, query: ASTNode = None, inf_schema=None, **kwargs):
+        databases, tables = _get_scope(query)
+
+        records = _get_records_from_data_catalog(databases, tables)
+
+        data = []
+        for record in records:
+            database_name = record.integration.name
+            table_name = record.name
+            primary_keys = record.meta_primary_keys
+            foreign_keys_children = record.meta_foreign_keys_children
+            foreign_keys_parents = record.meta_foreign_keys_parents
+
+            for pk in primary_keys:
+                item = {
+                    "CONSTRAINT_CATALOG": "def",
+                    "CONSTRAINT_SCHEMA": database_name,
+                    "CONSTRAINT_NAME": pk.constraint_name,
+                    "TABLE_SCHEMA": database_name,
+                    "TABLE_NAME": table_name,
+                    "CONSTRAINT_TYPE": "PRIMARY KEY",
+                }
+                data.append(item)
+
+            for fk in foreign_keys_children:
+                item = {
+                    "CONSTRAINT_CATALOG": "def",
+                    "CONSTRAINT_SCHEMA": database_name,
+                    "CONSTRAINT_NAME": fk.constraint_name,
+                    "TABLE_SCHEMA": database_name,
+                    "TABLE_NAME": table_name,
+                    "CONSTRAINT_TYPE": "FOREIGN KEY",
+                }
+                data.append(item)
+
+            for fk in foreign_keys_parents:
+                item = {
+                    "CONSTRAINT_CATALOG": "def",
+                    "CONSTRAINT_SCHEMA": database_name,
+                    "CONSTRAINT_NAME": fk.constraint_name,
+                    "TABLE_SCHEMA": database_name,
+                    "TABLE_NAME": table_name,
+                    "CONSTRAINT_TYPE": "FOREIGN KEY",
+                }
+                data.append(item)
+
+        df = pd.DataFrame(data, columns=cls.columns)
+        return df
+
+
+class MetaColumnUsageTable(Table):
+    name = "META_KEY_COLUMN_USAGE"
+    columns = [
+        "CONSTRAINT_CATALOG",
+        "CONSTRAINT_SCHEMA",
+        "CONSTRAINT_NAME",
+        "TABLE_CATALOG",
+        "TABLE_SCHEMA",
+        "TABLE_NAME",
+        "COLUMN_NAME",
+        "ORDINAL_POSITION",
+        "POSITION_IN_UNIQUE_CONSTRAINT",
+        "REFERENCED_TABLE_SCHEMA",
+        "REFERENCED_TABLE_NAME",
+        "REFERENCED_COLUMN_NAME",
+    ]
+
+    @classmethod
+    def get_data(cls, query: ASTNode = None, inf_schema=None, **kwargs):
+        databases, tables = _get_scope(query)
+
+        records = _get_records_from_data_catalog(databases, tables)
+
+        data = []
+        for record in records:
+            database_name = record.integration.name
+            table_name = record.name
+            primary_keys = record.meta_primary_keys
+            foreign_keys_children = record.meta_foreign_keys_children
+            foreign_keys_parents = record.meta_foreign_keys_parents
+
+            for pk in primary_keys:
+                column = pk.meta_columns
+
+                item = {
+                    "CONSTRAINT_CATALOG": "def",
+                    "CONSTRAINT_SCHEMA": database_name,
+                    "CONSTRAINT_NAME": pk.constraint_name,
+                    "TABLE_CATALOG": "def",
+                    "TABLE_SCHEMA": database_name,
+                    "TABLE_NAME": table_name,
+                    "COLUMN_NAME": column.name,
+                    "ORDINAL_POSITION": pk.ordinal_position,
+                    "POSITION_IN_UNIQUE_CONSTRAINT": None,
+                    "REFERENCED_TABLE_SCHEMA": None,
+                    "REFERENCED_TABLE_NAME": None,
+                    "REFERENCED_COLUMN_NAME": None,
+                }
+                data.append(item)
+
+            for fk in foreign_keys_children:
+                item = {
+                    "CONSTRAINT_CATALOG": "def",
+                    "CONSTRAINT_SCHEMA": database_name,
+                    "CONSTRAINT_NAME": fk.constraint_name,
+                    "TABLE_CATALOG": "def",
+                    "TABLE_SCHEMA": database_name,
+                    "TABLE_NAME": table_name,
+                    "COLUMN_NAME": fk.child_column.name,
+                    "ORDINAL_POSITION": None,
+                    "POSITION_IN_UNIQUE_CONSTRAINT": None,
+                    "REFERENCED_TABLE_SCHEMA": fk.parent_table.integration.name if fk.parent_table else None,
+                    "REFERENCED_TABLE_NAME": fk.parent_table.name if fk.parent_table else None,
+                    "REFERENCED_COLUMN_NAME": fk.parent_column.name if fk.parent_column else None,
+                }
+                data.append(item)
+
+            for fk in foreign_keys_parents:
+                item = {
+                    "CONSTRAINT_CATALOG": "def",
+                    "CONSTRAINT_SCHEMA": database_name,
+                    "CONSTRAINT_NAME": fk.constraint_name,
+                    "TABLE_CATALOG": "def",
+                    "TABLE_SCHEMA": database_name,
+                    "TABLE_NAME": table_name,
+                    "COLUMN_NAME": fk.child_column.name,
+                    "ORDINAL_POSITION": None,
+                    "POSITION_IN_UNIQUE_CONSTRAINT": None,
+                    "REFERENCED_TABLE_SCHEMA": fk.child_table.integration.name if fk.child_table else None,
+                    "REFERENCED_TABLE_NAME": fk.child_table.name if fk.child_table else None,
+                    "REFERENCED_COLUMN_NAME": fk.parent_column.name if fk.child_column else None,
+                }
+                data.append(item)
+
+        df = pd.DataFrame(data, columns=cls.columns)
+        return df
+
+
+class MetaHandlerInfoTable(Table):
+    name = "META_HANDLER_INFO"
+    columns = ["HANDLER_INFO", "TABLE_SCHEMA"]
+
+    @classmethod
+    def get_data(cls, query: ASTNode = None, inf_schema=None, **kwargs):
+        databases, tables = _get_scope(query)
+
+        data = []
+        for database in databases:
+            data_catalog_reader = DataCatalogReader(database_name=database, table_names=tables)
+            handler_info = data_catalog_reader.get_handler_info()
+            data.append({"HANDLER_INFO": str(handler_info), "TABLE_SCHEMA": database})
+
+        df = pd.DataFrame(data, columns=cls.columns)
+        return df

mindsdb/api/executor/planner/plan_join.py

@@ -158,7 +158,7 @@ class PlanJoinTablesQuery:
                 integration = self.planner.default_namespace
 
         if integration is None and not hasattr(table, "sub_select"):
-            raise PlanningException(f"Integration not found for: {table}")
+            raise PlanningException(f"Database not found for: {table}")
 
         sub_select = getattr(table, "sub_select", None)
 

mindsdb/api/executor/planner/query_planner.py

@@ -255,12 +255,18 @@ class QueryPlanner:
 
         database = self.default_namespace
 
+        err_msg_suffix = ""
         if len(parts) > 1:
             if parts[0].lower() in self.databases:
                 database = parts.pop(0).lower()
+            else:
+                err_msg_suffix = f"'{parts[0].lower()}' is not valid database name."
 
         if database is None:
-            raise PlanningException(f"Integration not found for: {node}")
+            raise PlanningException(
+                f"Invalid or missing database name for identifier '{node}'. {err_msg_suffix}\n"
+                "Query must include a valid database name prefix in format: 'database_name.table_name' or 'database_name.schema_name.table_name'"
+            )
 
         return database, Identifier(parts=parts, alias=alias)