MindsDB 25.9.1.2__py3-none-any.whl → 25.9.3rc1__py3-none-any.whl

mindsdb/interfaces/tasks/task_monitor.py

@@ -16,7 +16,6 @@ logger = log.getLogger(__name__)
 
 
 class TaskMonitor:
-
     MONITOR_INTERVAL_SECONDS = 2
     LOCK_EXPIRED_SECONDS = MONITOR_INTERVAL_SECONDS * 30
 
@@ -39,15 +38,14 @@ class TaskMonitor:
                 self.stop_all_tasks()
                 return
 
-            except Exception as e:
-                logger.error(e)
+            except Exception:
+                logger.exception("Error in TaskMonitor.start")
                 db.session.rollback()
 
             if stop_event is not None and stop_event.is_set():
                 return
 
     def stop_all_tasks(self):
-
         active_tasks = list(self._active_tasks.keys())
         for task_id in active_tasks:
             self.stop_task(task_id)
@@ -65,7 +63,6 @@ class TaskMonitor:
         # Check active tasks
         active_tasks = list(self._active_tasks.items())
         for task_id, task in active_tasks:
-
             if task_id not in allowed_tasks:
                 # old task
                 self.stop_task(task_id)
@@ -96,9 +93,7 @@ class TaskMonitor:
             task.run_by = run_by
             task.alive_time = db_date
 
-        elif db_date - task.alive_time > dt.timedelta(
-            seconds=self.LOCK_EXPIRED_SECONDS
-        ):
+        elif db_date - task.alive_time > dt.timedelta(seconds=self.LOCK_EXPIRED_SECONDS):
             # lock expired
             task.run_by = run_by
             task.alive_time = db_date
@@ -145,7 +140,6 @@ class TaskMonitor:
 
 
 def start(verbose=False):
-
     monitor = TaskMonitor()
     monitor.start()
 

mindsdb/interfaces/tasks/task_thread.py

@@ -12,7 +12,6 @@ logger = log.getLogger(__name__)
 
 
 class TaskThread(threading.Thread):
-
     def __init__(self, task_id):
         threading.Thread.__init__(self)
         self.task_id = task_id
@@ -34,28 +33,27 @@ class TaskThread(threading.Thread):
         self.object_type = task_record.object_type
         self.object_id = task_record.object_id
 
-        logger.info(f'Task starting: {self.object_type}.{self.object_id}')
+        logger.info(f"Task starting: {self.object_type}.{self.object_id}")
         try:
-            if self.object_type == 'trigger':
-
+            if self.object_type == "trigger":
                 trigger = TriggerTask(self.task_id, self.object_id)
                 trigger.run(self._stop_event)
 
-            elif self.object_type == 'chatbot':
+            elif self.object_type == "chatbot":
                 bot = ChatBotTask(self.task_id, self.object_id)
                 bot.run(self._stop_event)
 
-            elif self.object_type == 'query':
+            elif self.object_type == "query":
                 query = QueryTask(self.task_id, self.object_id)
                 query.run(self._stop_event)
 
         except Exception:
-            logger.error(traceback.format_exc())
-            task_record.last_error = str(traceback.format_exc())
+            logger.exception("Error during task processing:")
+            task_record.last_error = traceback.format_exc()
 
         db.session.commit()
 
     def stop(self):
-        logger.info(f'Task stopping: {self.object_type}.{self.object_id}')
+        logger.info(f"Task stopping: {self.object_type}.{self.object_id}")
 
         self._stop_event.set()

mindsdb/interfaces/triggers/trigger_task.py

@@ -44,19 +44,19 @@ class TriggerTask(BaseTask):
 
         # subscribe
         database = session.integration_controller.get_by_id(trigger.database_id)
-        data_handler = session.integration_controller.get_data_handler(database['name'])
+        data_handler = session.integration_controller.get_data_handler(database["name"])
 
         columns = trigger.columns
         if columns is not None:
-            if columns == '':
+            if columns == "":
                 columns = None
             else:
-                columns = columns.split('|')
+                columns = columns.split("|")
 
         data_handler.subscribe(stop_event, self._callback, trigger.table_name, columns=columns)
 
     def _callback(self, row, key=None):
-        logger.debug(f'trigger call: {row}, {key}')
+        logger.debug(f"trigger call: {row}, {key}")
 
         # set up environment
         ctx.load(self._ctx_dump)
@@ -64,21 +64,14 @@ class TriggerTask(BaseTask):
         try:
             if key is not None:
                 row.update(key)
-            table = [
-                row
-            ]
+            table = [row]
 
             # inject data to query
             query = copy.deepcopy(self.query)
 
             def find_table(node, is_table, **kwargs):
-
                 if is_table:
-                    if (
-                            isinstance(node, Identifier)
-                            and len(node.parts) == 1
-                            and node.parts[0] == 'TABLE_DELTA'
-                    ):
+                    if isinstance(node, Identifier) and len(node.parts) == 1 and node.parts[0] == "TABLE_DELTA":
                         # replace with data
                         return Data(table, alias=node.alias)
 
@@ -90,6 +83,7 @@ class TriggerTask(BaseTask):
                 self.set_error(ret.error_message)
 
         except Exception:
+            logger.exception("Error during trigger call processing")
             self.set_error(str(traceback.format_exc()))
 
         db.session.commit()

mindsdb/interfaces/triggers/triggers_controller.py

@@ -11,27 +11,28 @@ from mindsdb.api.executor.controllers.session_controller import SessionControlle
 
 
 class TriggersController:
-    OBJECT_TYPE = 'trigger'
+    OBJECT_TYPE = "trigger"
 
     def add(self, name, project_name, table, query_str, columns=None):
         name = name.lower()
 
         if project_name is None:
-            project_name = config.get('default_project')
+            project_name = config.get("default_project")
         project_controller = ProjectController()
         project = project_controller.get(name=project_name)
 
         from mindsdb.api.executor.controllers.session_controller import SessionController
+
         session = SessionController()
 
         # check exists
         trigger = self.get_trigger_record(name, project_name)
         if trigger is not None:
-            raise Exception(f'Trigger already exists: {name}')
+            raise Exception(f"Trigger already exists: {name}")
 
         # check table
         if len(table.parts) < 2:
-            raise Exception(f'Database or table not found: {table}')
+            raise Exception(f"Database or table not found: {table}")
 
         table_name = Identifier(parts=table.parts[1:]).to_string()
         db_name = table.parts[0]
@@ -39,39 +40,38 @@ class TriggersController:
         db_integration = session.integration_controller.get(db_name)
         db_handler = session.integration_controller.get_data_handler(db_name)
 
-        if not hasattr(db_handler, 'subscribe'):
-            raise Exception(f'Handler {db_integration["engine"]} does''t support subscription')
+        if not hasattr(db_handler, "subscribe"):
+            raise Exception(f"Handler {db_integration['engine']} doest support subscription")
 
         df = db_handler.get_tables().data_frame
-        column = 'table_name'
+        column = "table_name"
         if column not in df.columns:
             column = df.columns[0]
         tables = list(df[column])
 
         # check only if tables are visible
         if len(tables) > 0 and table_name not in tables:
-            raise Exception(f'Table {table_name} not found in {db_name}')
+            raise Exception(f"Table {table_name} not found in {db_name}")
 
         columns_str = None
         if columns is not None and len(columns) > 0:
             # join to string with delimiter
-            columns_str = '|'.join([col.parts[-1] for col in columns])
+            columns_str = "|".join([col.parts[-1] for col in columns])
 
         # check sql
         try:
             parse_sql(query_str)
         except ParsingException as e:
-            raise ParsingException(f'Unable to parse: {query_str}: {e}')
+            raise ParsingException(f"Unable to parse: {query_str}: {e}") from e
 
         # create job record
         record = db.Triggers(
             name=name,
             project_id=project.id,
-
-            database_id=db_integration['id'],
+            database_id=db_integration["id"],
             table_name=table_name,
             query_str=query_str,
-            columns=columns_str
+            columns=columns_str,
         )
         db.session.add(record)
         db.session.flush()
@@ -79,7 +79,6 @@ class TriggersController:
         task_record = db.Tasks(
             company_id=ctx.company_id,
             user_class=ctx.user_class,
-
             object_type=self.OBJECT_TYPE,
             object_id=record.id,
         )
@@ -110,34 +109,36 @@ class TriggersController:
         project_controller = ProjectController()
         project = project_controller.get(name=project_name)
 
-        query = db.session.query(
-            db.Triggers
-        ).join(
-            db.Tasks, db.Triggers.id == db.Tasks.object_id
-        ).filter(
-            db.Triggers.project_id == project.id,
-            db.Triggers.name == name,
-            db.Tasks.object_type == self.OBJECT_TYPE,
-            db.Tasks.company_id == ctx.company_id,
+        query = (
+            db.session.query(db.Triggers)
+            .join(db.Tasks, db.Triggers.id == db.Tasks.object_id)
+            .filter(
+                db.Triggers.project_id == project.id,
+                db.Triggers.name == name,
+                db.Tasks.object_type == self.OBJECT_TYPE,
+                db.Tasks.company_id == ctx.company_id,
+            )
         )
         return query.first()
 
     def get_list(self, project_name=None):
         session = SessionController()
 
-        query = db.session.query(
-            db.Tasks.object_id,
-            db.Triggers.project_id,
-            db.Triggers.name,
-            db.Triggers.database_id,
-            db.Triggers.table_name,
-            db.Triggers.query_str,
-            db.Tasks.last_error,
-        )\
-            .join(db.Triggers, db.Triggers.id == db.Tasks.object_id)\
+        query = (
+            db.session.query(
+                db.Tasks.object_id,
+                db.Triggers.project_id,
+                db.Triggers.name,
+                db.Triggers.database_id,
+                db.Triggers.table_name,
+                db.Triggers.query_str,
+                db.Tasks.last_error,
+            )
+            .join(db.Triggers, db.Triggers.id == db.Tasks.object_id)
             .filter(
                 db.Tasks.object_type == self.OBJECT_TYPE,
                 db.Tasks.company_id == ctx.company_id,
+            )
         )
 
         project_controller = ProjectController()
@@ -145,24 +146,20 @@ class TriggersController:
             project = project_controller.get(name=project_name)
             query = query.filter(db.Triggers.project_id == project.id)
 
-        database_names = {
-            i['id']: i['name']
-            for i in session.database_controller.get_list()
-        }
+        database_names = {i["id"]: i["name"] for i in session.database_controller.get_list()}
 
-        project_names = {
-            i.id: i.name
-            for i in project_controller.get_list()
-        }
+        project_names = {i.id: i.name for i in project_controller.get_list()}
         data = []
         for record in query:
-            data.append({
-                'id': record.object_id,
-                'project': project_names[record.project_id],
-                'name': record.name.lower(),
-                'database': database_names.get(record.database_id, '?'),
-                'table': record.table_name,
-                'query': record.query_str,
-                'last_error': record.last_error,
-            })
+            data.append(
+                {
+                    "id": record.object_id,
+                    "project": project_names[record.project_id],
+                    "name": record.name.lower(),
+                    "database": database_names.get(record.database_id, "?"),
+                    "table": record.table_name,
+                    "query": record.query_str,
+                    "last_error": record.last_error,
+                }
+            )
         return data

mindsdb/migrations/migrate.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 
-from alembic.command import upgrade, autogen    # noqa
+from alembic.command import upgrade, autogen  # noqa
 from alembic.config import Config
 from alembic.script import ScriptDirectory
 from alembic.script.revision import ResolutionError
@@ -15,20 +15,19 @@ logger = log.getLogger(__name__)
 
 # This is a migration that is like a 'base version'. Applying only this
 # migration to a fresh DB is equivalent to applying all previous migrations.
-current_checkpoint = '9f150e4f9a05'
+current_checkpoint = "9f150e4f9a05"
 
 
 def apply_checkpoint_migration(script) -> None:
-    """Apply the checkpoint migration to the database.
-    """
+    """Apply the checkpoint migration to the database."""
     with db.engine.begin() as connection:
         context = MigrationContext.configure(
             connection,
             opts={
-                'as_sql': False,
-                'starting_rev': None,  # ignore current version
-                'destination_rev': current_checkpoint,
-            }
+                "as_sql": False,
+                "starting_rev": None,  # ignore current version
+                "destination_rev": current_checkpoint,
+            },
         )
         revision = script.get_revision(current_checkpoint)
         if not revision:
@@ -41,7 +40,7 @@ def apply_checkpoint_migration(script) -> None:
 
 
 def get_current_revision() -> str | None:
-    """ Get the current revision of the database.
+    """Get the current revision of the database.
 
     Returns:
         str | None: The current revision of the database.
@@ -52,17 +51,18 @@ def get_current_revision() -> str | None:
 
 
 def migrate_to_head():
-    """ Trying to update database to head revision.
-        If alembic unable to recognize current revision (In case when database version is newer than backend)
-        then do nothing.
+    """Trying to update database to head revision.
+    If alembic unable to recognize current revision (In case when database version is newer than backend)
+    then do nothing.
     """
+    logger.debug("Applying database migrations")
 
-    config_file = Path(__file__).parent / 'alembic.ini'
+    config_file = Path(__file__).parent / "alembic.ini"
     config = Config(config_file)
 
     # mindsdb can runs not from project directory
-    script_location_abc = config_file.parent / config.get_main_option('script_location')
-    config.set_main_option('script_location', str(script_location_abc))
+    script_location_abc = config_file.parent / config.get_main_option("script_location")
+    config.set_main_option("script_location", str(script_location_abc))
 
     script = ScriptDirectory.from_config(config)
     cur_revision = get_current_revision()
@@ -81,7 +81,7 @@ def migrate_to_head():
         return
 
     logger.info("Migrations are available. Applying updates to the database.")
-    upgrade(config=config, revision='head')
+    upgrade(config=config, revision="head")
 
 
 if __name__ == "__main__":

mindsdb/utilities/api_status.py

@@ -0,0 +1,58 @@
+import os
+import json
+
+from mindsdb.utilities.config import config
+from mindsdb.utilities import log
+
+
+logger = log.getLogger(__name__)
+_api_status_file = None
+
+
+def _get_api_status_file():
+    global _api_status_file
+    if _api_status_file is None:
+        # Use a temporary file that can be shared across processes.
+        temp_dir = config["paths"]["tmp"]
+        _api_status_file = os.path.join(temp_dir, "mindsdb_api_status.json")
+        # Overwrite the file if it exists.
+        if os.path.exists(_api_status_file):
+            try:
+                os.remove(_api_status_file)
+            except OSError:
+                logger.exception(f"Error removing existing API status file: {_api_status_file}")
+
+    return _api_status_file
+
+
+def get_api_status():
+    """Get the current API status from the shared file."""
+    status_file = _get_api_status_file()
+    try:
+        if os.path.exists(status_file):
+            with open(status_file, "r") as f:
+                return json.load(f)
+    except (json.JSONDecodeError, IOError):
+        pass
+    return {}
+
+
+def set_api_status(api_name: str, status: bool):
+    """Set the status of an API in the shared file."""
+    status_file = _get_api_status_file()
+    current_status = get_api_status()
+    current_status[api_name] = status
+
+    # Write atomically to avoid race conditions.
+    temp_file = status_file + ".tmp"
+    try:
+        with open(temp_file, "w") as f:
+            json.dump(current_status, f)
+        os.replace(temp_file, status_file)
+    except IOError:
+        # Clean up temp file if it exists.
+        if os.path.exists(temp_file):
+            try:
+                os.remove(temp_file)
+            except OSError:
+                pass

mindsdb/utilities/config.py

@@ -170,6 +170,7 @@ class Config:
                     "restart_on_failure": True,
                     "max_restart_count": 1,
                     "max_restart_interval_seconds": 60,
+                    "a2wsgi": {"workers": 10, "send_queue_size": 10},
                 },
                 "mysql": {
                     "host": api_host,
@@ -300,6 +301,54 @@ class Config:
             self._env_config["default_reranking_model"] = {
                 "api_key": os.environ["MINDSDB_DEFAULT_RERANKING_MODEL_API_KEY"]
             }
+
+        # Reranker configuration from environment variables
+        reranker_config = {}
+        if os.environ.get("MINDSDB_RERANKER_N", "") != "":
+            try:
+                reranker_config["n"] = int(os.environ["MINDSDB_RERANKER_N"])
+            except ValueError:
+                raise ValueError(f"MINDSDB_RERANKER_N must be an integer, got: {os.environ['MINDSDB_RERANKER_N']}")
+
+        if os.environ.get("MINDSDB_RERANKER_LOGPROBS", "") != "":
+            logprobs_value = os.environ["MINDSDB_RERANKER_LOGPROBS"].lower()
+            if logprobs_value in ("true", "1", "yes", "y"):
+                reranker_config["logprobs"] = True
+            elif logprobs_value in ("false", "0", "no", "n"):
+                reranker_config["logprobs"] = False
+            else:
+                raise ValueError(
+                    f"MINDSDB_RERANKER_LOGPROBS must be a boolean value, got: {os.environ['MINDSDB_RERANKER_LOGPROBS']}"
+                )
+
+        if os.environ.get("MINDSDB_RERANKER_TOP_LOGPROBS", "") != "":
+            try:
+                reranker_config["top_logprobs"] = int(os.environ["MINDSDB_RERANKER_TOP_LOGPROBS"])
+            except ValueError:
+                raise ValueError(
+                    f"MINDSDB_RERANKER_TOP_LOGPROBS must be an integer, got: {os.environ['MINDSDB_RERANKER_TOP_LOGPROBS']}"
+                )
+
+        if os.environ.get("MINDSDB_RERANKER_MAX_TOKENS", "") != "":
+            try:
+                reranker_config["max_tokens"] = int(os.environ["MINDSDB_RERANKER_MAX_TOKENS"])
+            except ValueError:
+                raise ValueError(
+                    f"MINDSDB_RERANKER_MAX_TOKENS must be an integer, got: {os.environ['MINDSDB_RERANKER_MAX_TOKENS']}"
+                )
+
+        if os.environ.get("MINDSDB_RERANKER_VALID_CLASS_TOKENS", "") != "":
+            try:
+                reranker_config["valid_class_tokens"] = os.environ["MINDSDB_RERANKER_VALID_CLASS_TOKENS"].split(",")
+            except ValueError:
+                raise ValueError(
+                    f"MINDSDB_RERANKER_VALID_CLASS_TOKENS must be a comma-separated list of strings, got: {os.environ['MINDSDB_RERANKER_VALID_CLASS_TOKENS']}"
+                )
+
+        if reranker_config:
+            if "default_reranking_model" not in self._env_config:
+                self._env_config["default_reranking_model"] = {}
+            self._env_config["default_reranking_model"].update(reranker_config)
         if os.environ.get("MINDSDB_DATA_CATALOG_ENABLED", "").lower() in ("1", "true"):
             self._env_config["data_catalog"] = {"enabled": True}
 

mindsdb/utilities/exception.py

@@ -1,7 +1,14 @@
 from textwrap import indent
 
 
-class BaseEntityException(Exception):
+from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import ERR
+
+
+class MindsDBError(Exception):
+    pass
+
+
+class BaseEntityException(MindsDBError):
     """Base exception for entitys errors
 
     Attributes:
@@ -35,6 +42,38 @@ class EntityNotExistsError(BaseEntityException):
         super().__init__(message, entity_name)
 
 
+class ParsingError(MindsDBError):
+    pass
+
+
+class QueryError(MindsDBError):
+    def __init__(
+        self,
+        db_name: str | None = None,
+        db_type: str | None = None,
+        db_error_msg: str | None = None,
+        failed_query: str | None = None,
+        is_external: bool = True,
+        is_acceptable: bool = False,
+    ) -> None:
+        self.mysql_error_code = ERR.ER_UNKNOWN_ERROR
+        self.db_name = db_name
+        self.db_type = db_type
+        self.db_error_msg = db_error_msg
+        self.failed_query = failed_query
+        self.is_external = is_external
+        self.is_acceptable = is_acceptable
+
+    def __str__(self) -> str:
+        return format_db_error_message(
+            db_name=self.db_name,
+            db_type=self.db_type,
+            db_error_msg=self.db_error_msg,
+            failed_query=self.failed_query,
+            is_external=self.is_external,
+        )
+
+
 def format_db_error_message(
     db_name: str | None = None,
     db_type: str | None = None,

mindsdb/utilities/fs.py

@@ -118,7 +118,6 @@ def clean_unlinked_process_marks() -> List[int]:
 
         except psutil.AccessDenied:
             logger.warning(f"access to {process_id} denied")
-
             continue
 
         except psutil.NoSuchProcess:

mindsdb/utilities/hooks/profiling.py

@@ -46,28 +46,31 @@ def send_profiling_results(profiling_data: dict):
             user=MINDSDB_PROFILING_DB_USER,
             password=MINDSDB_PROFILING_DB_PASSWORD,
             dbname="postgres",
-            connect_timeout=5
+            connect_timeout=5,
         )
     except Exception:
-        logger.error('cant get acceess to profiling database')
+        logger.warning("cant get acceess to profiling database")
         return
     cur = connection.cursor()
-    cur.execute("""
+    cur.execute(
+        """
         insert into profiling
             (data, query, time, hostname, environment, api, total_time, company_id, instance_id)
         values
             (%s, %s, %s, %s, %s, %s, %s, %s, %s)
-    """, (
-        json.dumps(profiling["tree"]),
-        profiling.get("query", "?"),
-        time_start_at,
-        profiling["hostname"],
-        profiling.get("environment", "?"),
-        profiling.get("api", "?"),
-        profiling["tree"]["value"],
-        profiling["company_id"],
-        profiling["instance_id"]
-    ))
+    """,
+        (
+            json.dumps(profiling["tree"]),
+            profiling.get("query", "?"),
+            time_start_at,
+            profiling["hostname"],
+            profiling.get("environment", "?"),
+            profiling.get("api", "?"),
+            profiling["tree"]["value"],
+            profiling["company_id"],
+            profiling["instance_id"],
+        ),
+    )
 
     connection.commit()
     cur.close()