MindsDB 25.4.1.0__py3-none-any.whl → 25.4.2.1__py3-none-any.whl

mindsdb/__about__.py

@@ -1,6 +1,6 @@
 __title__ = 'MindsDB'
 __package_name__ = 'mindsdb'
-__version__ = '25.4.1.0'
+__version__ = '25.4.2.1'
 __description__ = "MindsDB's AI SQL Server enables developers to build AI tools that need access to real-time data to perform their tasks"
 __email__ = "jorge@mindsdb.com"
 __author__ = 'MindsDB Inc'

mindsdb/api/executor/command_executor.py

@@ -34,6 +34,7 @@ from mindsdb_sql_parser.ast import (
     Update,
     Use,
     Tuple,
+    Function,
 )
 
 # typed models
@@ -164,18 +165,17 @@ class ExecuteCommands:
         self.datahub = session.datahub
 
     @profiler.profile()
-    def execute_command(self, statement, database_name: str = None) -> ExecuteAnswer:
-        sql = None
-        if isinstance(statement, ASTNode):
-            sql = statement.to_string()
-        sql_lower = sql.lower()
+    def execute_command(self, statement: ASTNode, database_name: str = None) -> ExecuteAnswer:
+        sql: str = statement.to_string()
+        sql_lower: str = sql.lower()
 
         if database_name is None:
             database_name = self.session.database
 
-        if type(statement) is CreateDatabase:
+        statement_type = type(statement)
+        if statement_type is CreateDatabase:
             return self.answer_create_database(statement)
-        elif type(statement) is CreateMLEngine:
+        elif statement_type is CreateMLEngine:
             name = statement.name.parts[-1]
 
             return self.answer_create_ml_engine(
@@ -184,16 +184,16 @@ class ExecuteCommands:
                 params=statement.params,
                 if_not_exists=getattr(statement, "if_not_exists", False)
             )
-        elif type(statement) is DropMLEngine:
+        elif statement_type is DropMLEngine:
             return self.answer_drop_ml_engine(statement)
-        elif type(statement) is DropPredictor:
+        elif statement_type is DropPredictor:
             return self.answer_drop_model(statement, database_name)
 
-        elif type(statement) is DropTables:
+        elif statement_type is DropTables:
             return self.answer_drop_tables(statement, database_name)
-        elif type(statement) is DropDatasource or type(statement) is DropDatabase:
+        elif statement_type is DropDatasource or statement_type is DropDatabase:
             return self.answer_drop_database(statement)
-        elif type(statement) is Describe:
+        elif statement_type is Describe:
             # NOTE in sql 'describe table' is same as 'show columns'
             obj_type = statement.type
 
@@ -202,11 +202,11 @@ class ExecuteCommands:
             else:
                 return self.answer_describe_object(obj_type.upper(), statement.value, database_name)
 
-        elif type(statement) is RetrainPredictor:
+        elif statement_type is RetrainPredictor:
             return self.answer_retrain_predictor(statement, database_name)
-        elif type(statement) is FinetunePredictor:
+        elif statement_type is FinetunePredictor:
             return self.answer_finetune_predictor(statement, database_name)
-        elif type(statement) is Show:
+        elif statement_type is Show:
             sql_category = statement.category.lower()
             if hasattr(statement, "modes"):
                 if isinstance(statement.modes, list) is False:
@@ -504,13 +504,13 @@ class ExecuteCommands:
                 return self.answer_select(query)
             else:
                 raise NotSupportedYet(f"Statement not implemented: {sql}")
-        elif type(statement) in (
+        elif statement_type in (
             StartTransaction,
             CommitTransaction,
             RollbackTransaction,
         ):
             return ExecuteAnswer()
-        elif type(statement) is Set:
+        elif statement_type is Set:
             category = (statement.category or "").lower()
             if category == "" and isinstance(statement.name, Identifier):
                 param = statement.name.parts[0].lower()
@@ -565,85 +565,118 @@ class ExecuteCommands:
                     f"SQL statement is not processable, return OK package: {sql}"
                 )
                 return ExecuteAnswer()
-        elif type(statement) is Use:
+        elif statement_type is Use:
             db_name = statement.value.parts[-1]
             self.change_default_db(db_name)
             return ExecuteAnswer()
-        elif type(statement) in (
+        elif statement_type in (
             CreatePredictor,
             CreateAnomalyDetectionModel,  # we may want to specialize these in the future
         ):
             return self.answer_create_predictor(statement, database_name)
-        elif type(statement) is CreateView:
+        elif statement_type is CreateView:
             return self.answer_create_view(statement, database_name)
-        elif type(statement) is DropView:
+        elif statement_type is DropView:
             return self.answer_drop_view(statement, database_name)
-        elif type(statement) is Delete:
-            SQLQuery(statement, session=self.session, execute=True, database=database_name)
-            return ExecuteAnswer()
-
-        elif type(statement) is Insert:
-            SQLQuery(statement, session=self.session, execute=True, database=database_name)
-            return ExecuteAnswer()
-        elif type(statement) is Update:
-            SQLQuery(statement, session=self.session, execute=True, database=database_name)
-            return ExecuteAnswer()
+        elif statement_type is Delete:
+            query = SQLQuery(statement, session=self.session, database=database_name)
+            return ExecuteAnswer(
+                affected_rows=query.fetched_data.affected_rows
+            )
+        elif statement_type is Insert:
+            query = SQLQuery(statement, session=self.session, database=database_name)
+            return ExecuteAnswer(
+                affected_rows=query.fetched_data.affected_rows
+            )
+        elif statement_type is Update:
+            query = SQLQuery(statement, session=self.session, database=database_name)
+            return ExecuteAnswer(
+                affected_rows=query.fetched_data.affected_rows
+            )
         elif (
-            type(statement) is Alter
+            statement_type is Alter
             and ("disable keys" in sql_lower)
             or ("enable keys" in sql_lower)
         ):
             return ExecuteAnswer()
-        elif type(statement) is Select:
+        elif statement_type is Select:
+            ret = self.exec_service_function(statement, database_name)
+            if ret is not None:
+                return ret
             query = SQLQuery(statement, session=self.session, database=database_name)
             return self.answer_select(query)
-        elif type(statement) is Union:
+        elif statement_type is Union:
             query = SQLQuery(statement, session=self.session, database=database_name)
             return self.answer_select(query)
-        elif type(statement) is Explain:
+        elif statement_type is Explain:
             return self.answer_show_columns(statement.target, database_name=database_name)
-        elif type(statement) is CreateTable:
+        elif statement_type is CreateTable:
             return self.answer_create_table(statement, database_name)
         # -- jobs --
-        elif type(statement) is CreateJob:
+        elif statement_type is CreateJob:
             return self.answer_create_job(statement, database_name)
-        elif type(statement) is DropJob:
+        elif statement_type is DropJob:
             return self.answer_drop_job(statement, database_name)
         # -- triggers --
-        elif type(statement) is CreateTrigger:
+        elif statement_type is CreateTrigger:
             return self.answer_create_trigger(statement, database_name)
-        elif type(statement) is DropTrigger:
+        elif statement_type is DropTrigger:
             return self.answer_drop_trigger(statement, database_name)
         # -- chatbots
-        elif type(statement) is CreateChatBot:
+        elif statement_type is CreateChatBot:
             return self.answer_create_chatbot(statement, database_name)
-        elif type(statement) is UpdateChatBot:
+        elif statement_type is UpdateChatBot:
             return self.answer_update_chatbot(statement, database_name)
-        elif type(statement) is DropChatBot:
+        elif statement_type is DropChatBot:
             return self.answer_drop_chatbot(statement, database_name)
-        elif type(statement) is CreateKnowledgeBase:
+        elif statement_type is CreateKnowledgeBase:
             return self.answer_create_kb(statement, database_name)
-        elif type(statement) is DropKnowledgeBase:
+        elif statement_type is DropKnowledgeBase:
             return self.answer_drop_kb(statement, database_name)
-        elif type(statement) is CreateSkill:
+        elif statement_type is CreateSkill:
             return self.answer_create_skill(statement, database_name)
-        elif type(statement) is DropSkill:
+        elif statement_type is DropSkill:
             return self.answer_drop_skill(statement, database_name)
-        elif type(statement) is UpdateSkill:
+        elif statement_type is UpdateSkill:
             return self.answer_update_skill(statement, database_name)
-        elif type(statement) is CreateAgent:
+        elif statement_type is CreateAgent:
             return self.answer_create_agent(statement, database_name)
-        elif type(statement) is DropAgent:
+        elif statement_type is DropAgent:
             return self.answer_drop_agent(statement, database_name)
-        elif type(statement) is UpdateAgent:
+        elif statement_type is UpdateAgent:
             return self.answer_update_agent(statement, database_name)
-        elif type(statement) is Evaluate:
+        elif statement_type is Evaluate:
             statement.data = parse_sql(statement.query_str)
             return self.answer_evaluate_metric(statement, database_name)
         else:
             logger.warning(f"Unknown SQL statement: {sql}")
             raise NotSupportedYet(f"Unknown SQL statement: {sql}")
 
+    def exec_service_function(self, statement: Select, database_name: str) -> Optional[ExecuteAnswer]:
+        """
+        If input query is a single line select without FROM
+          and has function in targets that matches with one of the mindsdb service functions:
+          - execute this function and return response
+        Otherwise, return None to allow to continue execution query outside
+        """
+
+        if statement.from_table is not None or len(statement.targets) != 1:
+            return
+
+        target = statement.targets[0]
+        if not isinstance(target, Function):
+            return
+
+        command = target.op.lower()
+        args = [arg.value for arg in target.args if isinstance(arg, Constant)]
+        if command == 'query_resume':
+            ret = SQLQuery(None, session=self.session, database=database_name, query_id=args[0])
+            return self.answer_select(ret)
+
+        elif command == 'query_cancel':
+            query_context_controller.cancel_query(*args)
+            return ExecuteAnswer()
+
     def answer_create_trigger(self, statement, database_name):
         triggers_controller = TriggersController()
 
@@ -785,8 +818,7 @@ class ExecuteCommands:
             raise Exception(
                 f'Nested query failed to execute with error: "{e}", please check and try again.'
             )
-        result = sqlquery.fetch('dataframe')
-        df = result["result"]
+        df = sqlquery.fetched_data.to_df()
         df.columns = [
             str(t.alias) if hasattr(t, "alias") else str(t.parts[-1])
             for t in statement.data.targets
@@ -1253,7 +1285,6 @@ class ExecuteCommands:
             project_name = parts[0]
 
         query_str = statement.query_str
-        query = parse_sql(query_str)
 
         if isinstance(statement.from_table, Identifier):
             query = Select(
@@ -1263,6 +1294,8 @@ class ExecuteCommands:
                 ),
             )
             query_str = str(query)
+        else:
+            query = parse_sql(query_str)
 
         if isinstance(query, Select):
             # check create view sql
@@ -1272,9 +1305,7 @@ class ExecuteCommands:
                 query_context_controller.IGNORE_CONTEXT
             )
             try:
-                sqlquery = SQLQuery(query, session=self.session, database=database_name)
-                if sqlquery.fetch()["success"] is not True:
-                    raise ExecutorException("Wrong view query")
+                SQLQuery(query, session=self.session, database=database_name)
             finally:
                 query_context_controller.release_context(
                     query_context_controller.IGNORE_CONTEXT
@@ -1920,9 +1951,8 @@ class ExecuteCommands:
         return ExecuteAnswer()
 
     def answer_select(self, query):
-        data = query.fetch()
-
-        return ExecuteAnswer(data=data["result"])
+        data = query.fetched_data
+        return ExecuteAnswer(data=data)
 
     def answer_update_model_version(self, model_version, database_name):
         if not isinstance(model_version, Identifier):

mindsdb/api/executor/data_types/answer.py

@@ -1,16 +1,13 @@
-from typing import List
+from dataclasses import dataclass
+from typing import List, Optional
+
 from mindsdb.api.executor.sql_query.result_set import ResultSet
 
 
+@dataclass(kw_only=True, slots=True)
 class ExecuteAnswer:
-    def __init__(
-        self,
-        data: ResultSet = None,
-        state_track: List[List] = None,
-        error_code: int = None,
-        error_message: str = None,
-    ):
-        self.data = data
-        self.state_track = state_track
-        self.error_code = error_code
-        self.error_message = error_message
+    data: Optional[ResultSet] = None
+    state_track: Optional[List[List]] = None
+    error_code: Optional[int] = None
+    error_message: Optional[str] = None
+    affected_rows: Optional[int] = None

mindsdb/api/executor/datahub/classes/response.py

@@ -0,0 +1,11 @@
+from dataclasses import dataclass, field
+from typing import Optional, List, Dict
+
+import pandas as pd
+
+
+@dataclass
+class DataHubResponse:
+    data_frame: pd.DataFrame = field(default_factory=pd.DataFrame)
+    columns: List[Dict] = field(default_factory=list)
+    affected_rows: Optional[int] = None

mindsdb/api/executor/datahub/datanodes/datanode.py

@@ -1,3 +1,6 @@
+from mindsdb.api.executor.datahub.classes.response import DataHubResponse
+
+
 class DataNode:
     type = 'meta'
 
@@ -10,11 +13,8 @@ class DataNode:
     def get_tables(self):
         pass
 
-    def has_table(self, tableName):
-        pass
-
     def get_table_columns(self, tableName, schema_name=None):
         pass
 
-    def query(self, query=None, native_query=None, session=None):
+    def query(self, query=None, native_query=None, session=None) -> DataHubResponse:
         return []

mindsdb/api/executor/datahub/datanodes/information_schema_datanode.py

@@ -9,7 +9,7 @@ from mindsdb.api.executor import exceptions as exc
 from mindsdb.api.executor.utilities.sql import query_df
 from mindsdb.api.executor.utilities.sql import get_query_tables
 from mindsdb.interfaces.database.projects import ProjectController
-
+from mindsdb.api.executor.datahub.classes.response import DataHubResponse
 from mindsdb.utilities import log
 
 from .system_tables import (
@@ -17,7 +17,7 @@ from .system_tables import (
     PluginsTable, EnginesTable, KeyColumnUsageTable, StatisticsTable,
     CharacterSetsTable, CollationsTable)
 from .mindsdb_tables import (
-    ModelsTable, DatabasesTable, MLEnginesTable, HandlersTable, JobsTable,
+    ModelsTable, DatabasesTable, MLEnginesTable, HandlersTable, JobsTable, QueriesTable,
     ChatbotsTable, KBTable, SkillsTable, AgentsTable, ViewsTable, TriggersTable)
 
 
@@ -32,7 +32,8 @@ class InformationSchemaDataNode(DataNode):
         PluginsTable, EnginesTable, KeyColumnUsageTable, StatisticsTable,
         CharacterSetsTable, CollationsTable,
         ModelsTable, DatabasesTable, MLEnginesTable, HandlersTable, JobsTable,
-        ChatbotsTable, KBTable, SkillsTable, AgentsTable, ViewsTable, TriggersTable
+        ChatbotsTable, KBTable, SkillsTable, AgentsTable, ViewsTable, TriggersTable,
+        QueriesTable
     ]
 
     def __init__(self, session):
@@ -110,12 +111,6 @@ class InformationSchemaDataNode(DataNode):
 
         return None
 
-    def has_table(self, tableName):
-        tn = tableName.upper()
-        if tn in self.tables:
-            return True
-        return False
-
     def get_table_columns(self, tableName, schema_name=None):
         tn = tableName.upper()
         if tn in self.tables:
@@ -143,7 +138,7 @@ class InformationSchemaDataNode(DataNode):
             if table.visible
         }
 
-    def query(self, query: ASTNode, session=None):
+    def query(self, query: ASTNode, session=None) -> DataHubResponse:
         query_tables = [x[1] for x in get_query_tables(query)]
 
         if len(query_tables) != 1:
@@ -166,7 +161,11 @@ class InformationSchemaDataNode(DataNode):
 
         columns_info = [{"name": k, "type": v} for k, v in data.dtypes.items()]
 
-        return data, columns_info
+        return DataHubResponse(
+            data_frame=data,
+            columns=columns_info,
+            affected_rows=0
+        )
 
     def _get_empty_table(self, table):
         columns = table.columns

mindsdb/api/executor/datahub/datanodes/integration_datanode.py

@@ -10,16 +10,19 @@ from sqlalchemy.types import (
     Integer, Float, Text
 )
 
+from mindsdb_sql_parser.ast.base import ASTNode
 from mindsdb_sql_parser.ast import Insert, Identifier, CreateTable, TableColumn, DropTables
 
+from mindsdb.api.executor.datahub.classes.response import DataHubResponse
 from mindsdb.api.executor.datahub.datanodes.datanode import DataNode
-from mindsdb.api.executor.data_types.response_type import RESPONSE_TYPE
 from mindsdb.api.executor.datahub.classes.tables_row import TablesRow
+from mindsdb.api.executor.data_types.response_type import RESPONSE_TYPE
 from mindsdb.api.executor.sql_query.result_set import ResultSet
 from mindsdb.integrations.utilities.utils import get_class_name
 from mindsdb.metrics import metrics
 from mindsdb.utilities import log
 from mindsdb.utilities.profiler import profiler
+from mindsdb.integrations.libs.response import HandlerResponse
 
 logger = log.getLogger(__name__)
 
@@ -52,9 +55,6 @@ class IntegrationDataNode(DataNode):
         else:
             raise Exception(f"Can't get tables: {response.error_message}")
 
-    def has_table(self, tableName):
-        return True
-
     def get_table_columns(self, table_name: str, schema_name: Optional[str] = None):
         if 'schema_name' in inspect.signature(self.integration_handler.get_columns).parameters:
             response = self.integration_handler.get_columns(table_name, schema_name)
@@ -107,7 +107,7 @@ class IntegrationDataNode(DataNode):
             raise Exception(result.error_message)
 
     def create_table(self, table_name: Identifier, result_set: ResultSet = None, columns=None,
-                     is_replace=False, is_create=False):
+                     is_replace=False, is_create=False) -> DataHubResponse:
         # is_create - create table
         # is_replace - drop table if exists
         # is_create==False and is_replace==False: just insert
@@ -164,14 +164,14 @@ class IntegrationDataNode(DataNode):
 
         if result_set is None:
             # it is just a 'create table'
-            return
+            return DataHubResponse()
 
         # native insert
         if hasattr(self.integration_handler, 'insert'):
             df = result_set.to_df()
 
-            self.integration_handler.insert(table_name.parts[-1], df)
-            return
+            result: HandlerResponse = self.integration_handler.insert(table_name.parts[-1], df)
+            return DataHubResponse(affected_rows=result.affected_rows)
 
         insert_columns = [Identifier(parts=[x.alias]) for x in result_set.columns]
 
@@ -195,7 +195,7 @@ class IntegrationDataNode(DataNode):
 
         if len(values) == 0:
             # not need to insert
-            return
+            return DataHubResponse()
 
         insert_ast = Insert(
             table=table_name,
@@ -213,7 +213,9 @@ class IntegrationDataNode(DataNode):
         if result.type == RESPONSE_TYPE.ERROR:
             raise Exception(result.error_message)
 
-    def _query(self, query):
+        return DataHubResponse(affected_rows=result.affected_rows)
+
+    def _query(self, query) -> HandlerResponse:
         time_before_query = time.perf_counter()
         result = self.integration_handler.query(query)
         elapsed_seconds = time.perf_counter() - time_before_query
@@ -229,7 +231,7 @@ class IntegrationDataNode(DataNode):
         response_size_with_labels.observe(num_rows)
         return result
 
-    def _native_query(self, native_query):
+    def _native_query(self, native_query) -> HandlerResponse:
         time_before_query = time.perf_counter()
         result = self.integration_handler.native_query(native_query)
         elapsed_seconds = time.perf_counter() - time_before_query
@@ -246,13 +248,13 @@ class IntegrationDataNode(DataNode):
         return result
 
     @profiler.profile()
-    def query(self, query=None, native_query=None, session=None):
+    def query(self, query: Optional[ASTNode] = None, native_query: Optional[str] = None, session=None) -> DataHubResponse:
         try:
             if query is not None:
-                result = self._query(query)
+                result: HandlerResponse = self._query(query)
             else:
                 # try to fetch native query
-                result = self._native_query(native_query)
+                result: HandlerResponse = self._native_query(native_query)
         except Exception as e:
             msg = str(e).strip()
             if msg == '':
@@ -263,7 +265,7 @@ class IntegrationDataNode(DataNode):
         if result.type == RESPONSE_TYPE.ERROR:
             raise Exception(f'Error in {self.integration_name}: {result.error_message}')
         if result.type == RESPONSE_TYPE.OK:
-            return pd.DataFrame(), []
+            return DataHubResponse(affected_rows=result.affected_rows)
 
         df = result.data_frame
         # region clearing df from NaN values
@@ -286,4 +288,8 @@ class IntegrationDataNode(DataNode):
             for k, v in df.dtypes.items()
         ]
 
-        return df, columns_info
+        return DataHubResponse(
+            data_frame=df,
+            columns=columns_info,
+            affected_rows=result.affected_rows
+        )

mindsdb/api/executor/datahub/datanodes/mindsdb_tables.py

@@ -9,6 +9,7 @@ from mindsdb.interfaces.jobs.jobs_controller import JobsController
 from mindsdb.interfaces.skills.skills_controller import SkillsController
 from mindsdb.interfaces.database.views import ViewController
 from mindsdb.interfaces.database.projects import ProjectController
+from mindsdb.interfaces.query_context.context_controller import query_context_controller
 
 from mindsdb.api.executor.datahub.datanodes.system_tables import Table
 
@@ -326,7 +327,8 @@ class ChatbotsTable(MdbTable):
 
 class KBTable(MdbTable):
     name = 'KNOWLEDGE_BASES'
-    columns = ["NAME", "PROJECT", "MODEL", "STORAGE", "PARAMS"]
+    columns = ["NAME", "PROJECT", "MODEL", "STORAGE", "PARAMS",
+               "INSERT_STARTED_AT", "INSERT_FINISHED_AT", "PROCESSED_ROWS", "ERROR", "QUERY_ID"]
 
     @classmethod
     def get_data(cls, query: ASTNode = None, inf_schema=None, **kwargs):
@@ -336,17 +338,36 @@ class KBTable(MdbTable):
         controller = KnowledgeBaseController(inf_schema.session)
         kb_list = controller.list(project_name)
 
+        # shouldn't be a lot of queries, we can fetch them all
+        queries_data = {
+            item['id']: item
+            for item in query_context_controller.list_queries()
+        }
+
         data = []
 
         for kb in kb_list:
             vector_database_name = kb['vector_database'] or ''
 
+            query_item = {}
+            query_id = kb['query_id']
+            if query_id is not None:
+                if query_id in queries_data:
+                    query_item = queries_data.get(query_id)
+                else:
+                    query_id = None
+
             data.append((
                 kb['name'],
                 kb['project_name'],
                 kb['embedding_model'],
                 vector_database_name + '.' + kb['vector_database_table'],
                 to_json(kb['params']),
+                query_item.get('started_at'),
+                query_item.get('finished_at'),
+                query_item.get('processed_rows'),
+                query_item.get('error'),
+                query_id,
             ))
 
         return pd.DataFrame(data, columns=cls.columns)
@@ -426,3 +447,24 @@ class ViewsTable(MdbTable):
         data = [[row[k] for k in columns_lower] for row in data]
 
         return pd.DataFrame(data, columns=cls.columns)
+
+
+class QueriesTable(MdbTable):
+    name = 'QUERIES'
+    columns = ["ID", "STARTED_AT", "FINISHED_AT", "PROCESSED_ROWS", "ERROR", "SQL", "PARAMETERS", "CONTEXT", "UPDATED_AT"]
+
+    @classmethod
+    def get_data(cls, **kwargs):
+        """
+        Returns all queries in progres or recently completed
+        Only queries marked as is_resumable by planner are stored in this table
+        :param kwargs:
+        :return:
+        """
+
+        data = query_context_controller.list_queries()
+        columns_lower = [col.lower() for col in cls.columns]
+
+        data = [[row[k] for k in columns_lower] for row in data]
+
+        return pd.DataFrame(data, columns=cls.columns)