MindsDB 25.4.5.0__py3-none-any.whl → 25.5.4.0__py3-none-any.whl

mindsdb/__about__.py

@@ -1,6 +1,6 @@
 __title__ = 'MindsDB'
 __package_name__ = 'mindsdb'
-__version__ = '25.4.5.0'
+__version__ = '25.5.4.0'
 __description__ = "MindsDB's AI SQL Server enables developers to build AI tools that need access to real-time data to perform their tasks"
 __email__ = "jorge@mindsdb.com"
 __author__ = 'MindsDB Inc'

mindsdb/__main__.py

@@ -1,3 +1,5 @@
+import gc
+gc.disable()
 import os
 import sys
 import time
@@ -12,7 +14,7 @@ from enum import Enum
 from dataclasses import dataclass, field
 from typing import Callable, Optional, Tuple, List
 
-from packaging import version
+from sqlalchemy import func
 from sqlalchemy.orm.attributes import flag_modified
 
 from mindsdb.utilities import log
@@ -22,17 +24,20 @@ logger.debug("Starting MindsDB...")
 
 from mindsdb.__about__ import __version__ as mindsdb_version
 from mindsdb.utilities.config import config
-from mindsdb.utilities.exception import EntityNotExistsError
 from mindsdb.utilities.starters import (
-    start_http, start_mysql, start_mongo, start_postgres, start_ml_task_queue, start_scheduler, start_tasks,
-    start_mcp, start_litellm
+    start_http,
+    start_mysql,
+    start_mongo,
+    start_postgres,
+    start_ml_task_queue,
+    start_scheduler,
+    start_tasks,
+    start_mcp,
+    start_litellm,
+    start_a2a,
 )
 from mindsdb.utilities.ps import is_pid_listen_port, get_child_pids
-from mindsdb.utilities.functions import get_versions_where_predictors_become_obsolete
-from mindsdb.interfaces.database.integrations import integration_controller
-from mindsdb.interfaces.database.projects import ProjectController
 import mindsdb.interfaces.storage.db as db
-from mindsdb.integrations.utilities.install import install_dependencies
 from mindsdb.utilities.fs import clean_process_marks, clean_unlinked_process_marks
 from mindsdb.utilities.context import context as ctx
 from mindsdb.utilities.auth import register_oauth_client, get_aws_meta_data
@@ -43,23 +48,26 @@ try:
 except Exception:
     import multiprocessing as mp
 try:
-    mp.set_start_method('spawn')
+    mp.set_start_method("spawn")
 except RuntimeError:
-    logger.info('Torch multiprocessing context already set, ignoring...')
+    logger.info("Torch multiprocessing context already set, ignoring...")
+
+gc.enable()
 
 _stop_event = threading.Event()
 
 
 class TrunkProcessEnum(Enum):
-    HTTP = 'http'
-    MYSQL = 'mysql'
-    MONGODB = 'mongodb'
-    POSTGRES = 'postgres'
-    JOBS = 'jobs'
-    TASKS = 'tasks'
-    ML_TASK_QUEUE = 'ml_task_queue'
-    MCP = 'mcp'
-    LITELLM = 'litellm'
+    HTTP = "http"
+    MYSQL = "mysql"
+    MONGODB = "mongodb"
+    POSTGRES = "postgres"
+    JOBS = "jobs"
+    TASKS = "tasks"
+    ML_TASK_QUEUE = "ml_task_queue"
+    MCP = "mcp"
+    LITELLM = "litellm"
+    A2A = "a2a"
 
     @classmethod
     def _missing_(cls, value):
@@ -97,7 +105,8 @@ class TrunkProcessData:
         self._restarts_time.append(current_time_seconds)
         if self.max_restart_interval_seconds > 0:
             self._restarts_time = [
-                x for x in self._restarts_time
+                x
+                for x in self._restarts_time
                 if x >= (current_time_seconds - self.max_restart_interval_seconds)
             ]
         if len(self._restarts_time) > self.max_restart_count:
@@ -114,12 +123,17 @@ class TrunkProcessData:
         """
         if config.is_cloud:
             return False
-        if sys.platform in ('linux', 'darwin'):
-            return self.restart_on_failure and self.process.exitcode == -signal.SIGKILL.value
+        if sys.platform in ("linux", "darwin"):
+            return (
+                self.restart_on_failure
+                and self.process.exitcode == -signal.SIGKILL.value
+            )
         else:
             if self.max_restart_count == 0:
                 # to prevent infinity restarts, max_restart_count should be > 0
-                logger.warning('In the current OS, it is not possible to use `max_restart_count=0`')
+                logger.warning(
+                    "In the current OS, it is not possible to use `max_restart_count=0`"
+                )
                 return False
             return self.restart_on_failure
 
@@ -160,22 +174,25 @@ def set_error_model_status_by_pids(unexisting_pids: List[int]):
         db.session.query(db.Predictor)
         .filter(
             db.Predictor.deleted_at.is_(None),
-            db.Predictor.status.not_in([
-                db.PREDICTOR_STATUS.COMPLETE,
-                db.PREDICTOR_STATUS.ERROR
-            ])
+            db.Predictor.status.not_in(
+                [db.PREDICTOR_STATUS.COMPLETE, db.PREDICTOR_STATUS.ERROR]
+            ),
         )
         .all()
     )
     for predictor_record in predictor_records:
-        predictor_process_id = (predictor_record.training_metadata or {}).get('process_id')
+        predictor_process_id = (predictor_record.training_metadata or {}).get(
+            "process_id"
+        )
         if predictor_process_id in unexisting_pids:
             predictor_record.status = db.PREDICTOR_STATUS.ERROR
             if isinstance(predictor_record.data, dict) is False:
                 predictor_record.data = {}
-            if 'error' not in predictor_record.data:
-                predictor_record.data['error'] = 'The training process was terminated for unknown reasons'
-                flag_modified(predictor_record, 'data')
+            if "error" not in predictor_record.data:
+                predictor_record.data["error"] = (
+                    "The training process was terminated for unknown reasons"
+                )
+                flag_modified(predictor_record, "data")
             db.session.commit()
 
 
@@ -187,10 +204,9 @@ def set_error_model_status_for_unfinished():
         db.session.query(db.Predictor)
         .filter(
             db.Predictor.deleted_at.is_(None),
-            db.Predictor.status.not_in([
-                db.PREDICTOR_STATUS.COMPLETE,
-                db.PREDICTOR_STATUS.ERROR
-            ])
+            db.Predictor.status.not_in(
+                [db.PREDICTOR_STATUS.COMPLETE, db.PREDICTOR_STATUS.ERROR]
+            ),
         )
         .all()
     )
@@ -198,34 +214,131 @@ def set_error_model_status_for_unfinished():
         predictor_record.status = db.PREDICTOR_STATUS.ERROR
         if isinstance(predictor_record.data, dict) is False:
             predictor_record.data = {}
-        if 'error' not in predictor_record.data:
-            predictor_record.data['error'] = 'Unknown error'
-            flag_modified(predictor_record, 'data')
+        if "error" not in predictor_record.data:
+            predictor_record.data["error"] = "Unknown error"
+            flag_modified(predictor_record, "data")
         db.session.commit()
 
 
 def do_clean_process_marks():
-    """delete unexisting 'process marks'
-    """
+    """delete unexisting 'process marks'"""
     while _stop_event.wait(timeout=5) is False:
         unexisting_pids = clean_unlinked_process_marks()
         if not config.is_cloud and len(unexisting_pids) > 0:
             set_error_model_status_by_pids(unexisting_pids)
 
 
-if __name__ == '__main__':
+def create_permanent_integrations():
+    """
+    Create permanent integrations, for now only the 'files' integration.
+    NOTE: this is intentional to avoid importing integration_controller
+    """
+    integration_name = "files"
+    existing = (
+        db.session.query(db.Integration)
+        .filter_by(name=integration_name, company_id=None)
+        .first()
+    )
+    if existing is None:
+        integration_record = db.Integration(
+            name=integration_name,
+            data={},
+            engine=integration_name,
+            company_id=None,
+        )
+        db.session.add(integration_record)
+        try:
+            db.session.commit()
+        except Exception as e:
+            logger.error(
+                f"Failed to commit permanent integration {integration_name}: {e}"
+            )
+            db.session.rollback()
+
+
+def validate_default_project() -> None:
+    """Handle 'default_project' config option.
+    Project with the name specified in 'default_project' must exists and be marked with
+    'is_default' metadata. If it is not possible, then terminate the process with error.
+    Note: this can be done using 'project_controller', but we want to save init time and used RAM.
+    """
+    new_default_project_name = config.get("default_project")
+    logger.debug(f"Checking if default project {new_default_project_name} exists")
+    filter_company_id = ctx.company_id if ctx.company_id is not None else 0
+
+    current_default_project: db.Project | None = db.Project.query.filter(
+        db.Project.company_id == filter_company_id,
+        db.Project.metadata_["is_default"].as_boolean() == True,  # noqa
+    ).first()
+
+    if current_default_project is None:
+        # Legacy: If the default project does not exist, mark the new one as default.
+        existing_project = db.Project.query.filter(
+            db.Project.company_id == filter_company_id,
+            func.lower(db.Project.name) == func.lower(new_default_project_name),
+        ).first()
+        if existing_project is None:
+            logger.critical(
+                f"A project with the name '{new_default_project_name}' does not exist"
+            )
+            sys.exit(1)
+
+        existing_project.metadata_ = {"is_default": True}
+        flag_modified(existing_project, "metadata_")
+        db.session.commit()
+    elif current_default_project.name != new_default_project_name:
+        # If the default project exists, but the name is different, update the name.
+        existing_project = db.Project.query.filter(
+            db.Project.company_id == filter_company_id,
+            func.lower(db.Project.name) == func.lower(new_default_project_name),
+        ).first()
+        if existing_project is not None:
+            logger.critical(
+                f"A project with the name '{new_default_project_name}' already exists"
+            )
+            sys.exit(1)
+        current_default_project.name = new_default_project_name
+        db.session.commit()
+
+
+def start_process(trunc_process_data: TrunkProcessData) -> None:
+    """Start a process.
+
+    Args:
+        trunc_process_data (TrunkProcessData): The data of the process to start.
+    """
+    mp_ctx = mp.get_context("spawn")
+    logger.info(f"{trunc_process_data.name} API: starting...")
+    try:
+        trunc_process_data.process = mp_ctx.Process(
+            target=trunc_process_data.entrypoint,
+            args=trunc_process_data.args,
+            name=trunc_process_data.name,
+        )
+        trunc_process_data.process.start()
+    except Exception as e:
+        logger.error(
+            f"Failed to start {trunc_process_data.name} API with exception {e}\n{traceback.format_exc()}"
+        )
+        close_api_gracefully(trunc_processes_struct)
+        raise e
+
+
+if __name__ == "__main__":
+    mp.freeze_support()
     # warn if less than 1Gb of free RAM
     if psutil.virtual_memory().available < (1 << 30):
         logger.warning(
-            'The system is running low on memory. '
-            + 'This may impact the stability and performance of the program.'
+            "The system is running low on memory. "
+            + "This may impact the stability and performance of the program."
         )
 
     ctx.set_default()
 
     # ---- CHECK SYSTEM ----
     if not (sys.version_info[0] >= 3 and sys.version_info[1] >= 10):
-        print("""
+        print(
+            """
      MindsDB requires Python >= 3.10 to run
 
      Once you have supported Python version installed you can start mindsdb as follows:
@@ -241,11 +354,12 @@ if __name__ == '__main__':
      python3 -m mindsdb
 
      More instructions in https://docs.mindsdb.com
-         """)
+         """
+        )
         exit(1)
 
     if config.cmd_args.version:
-        print(f'MindsDB {mindsdb_version}')
+        print(f"MindsDB {mindsdb_version}")
         sys.exit(0)
 
     config.raise_warnings(logger=logger)
@@ -254,23 +368,23 @@ if __name__ == '__main__':
     if os.environ.get("FLASK_SECRET_KEY") is None:
         os.environ["FLASK_SECRET_KEY"] = secrets.token_hex(32)
 
-    if os.environ.get('ARROW_DEFAULT_MEMORY_POOL') is None:
+    if os.environ.get("ARROW_DEFAULT_MEMORY_POOL") is None:
         try:
             """It seems like snowflake handler have memory issue that related to pyarrow. Memory usage keep growing with
             requests. This is related to 'memory pool' that is 'mimalloc' by default: it is fastest but use a lot of ram
             """
             import pyarrow as pa
+
             try:
                 pa.jemalloc_memory_pool()
-                os.environ['ARROW_DEFAULT_MEMORY_POOL'] = 'jemalloc'
+                os.environ["ARROW_DEFAULT_MEMORY_POOL"] = "jemalloc"
             except NotImplementedError:
                 pa.system_memory_pool()
-                os.environ['ARROW_DEFAULT_MEMORY_POOL'] = 'system'
+                os.environ["ARROW_DEFAULT_MEMORY_POOL"] = "system"
         except Exception:
             pass
 
     db.init()
-    mp.freeze_support()
 
     environment = config["environment"]
     if environment == "aws_marketplace":
@@ -281,87 +395,18 @@ if __name__ == '__main__':
     elif environment != "local":
         try:
             aws_meta_data = get_aws_meta_data()
-            config.update({
-                'aws_meta_data': aws_meta_data
-            })
+            config.update({"aws_meta_data": aws_meta_data})
         except Exception:
             pass
 
-    is_cloud = config.is_cloud
-
-    if not is_cloud:
-        logger.debug("Applying database migrations")
-        try:
-            from mindsdb.migrations import migrate
-            migrate.migrate_to_head()
-        except Exception as e:
-            logger.error(f"Error! Something went wrong during DB migrations: {e}")
-
-        logger.debug(f"Checking if default project {config.get('default_project')} exists")
-        project_controller = ProjectController()
-
-        try:
-            current_default_project = project_controller.get(is_default=True)
-        except EntityNotExistsError:
-            # In previous versions, the default project could be deleted. This is no longer possible.
-            current_default_project = None
-
-        if current_default_project:
-            if current_default_project.record.name != config.get('default_project'):
-                try:
-                    project_controller.get(name=config.get('default_project'))
-                    log.critical(f"A project with the name '{config.get('default_project')}' already exists")
-                    sys.exit(1)
-                except EntityNotExistsError:
-                    pass
-                project_controller.update(current_default_project.record.id, new_name=config.get('default_project'))
-
-        # Legacy: If the default project does not exist, mark the new one as default.
-        else:
-            try:
-                project_controller.get(name=config.get('default_project'))
-            except EntityNotExistsError:
-                log.critical(
-                    f"A project with the name '{config.get('default_project')}' does not exist"
-                )
-                raise
-
-            project_controller.update(
-                name=config.get('default_project'),
-                new_metadata={
-                    "is_default": True
-                }
-            )
-
-    apis = os.getenv('MINDSDB_APIS') or config.cmd_args.api
+    apis = os.getenv("MINDSDB_APIS") or config.cmd_args.api
 
     if apis is None:  # If "--api" option is not specified, start the default APIs
         api_arr = [TrunkProcessEnum.HTTP, TrunkProcessEnum.MYSQL]
     elif apis == "":  # If "--api=" (blank) is specified, don't start any APIs
         api_arr = []
     else:  # The user has provided a list of APIs to start
-        api_arr = [TrunkProcessEnum(name) for name in apis.split(',')]
-
-    if config.cmd_args.install_handlers is not None:
-        handlers_list = [s.strip() for s in config.cmd_args.install_handlers.split(",")]
-        # import_meta = handler_meta.get('import', {})
-        for handler_name, handler_meta in integration_controller.get_handlers_import_status().items():
-            if handler_name not in handlers_list:
-                continue
-            import_meta = handler_meta.get("import", {})
-            if import_meta.get("success") is True:
-                logger.info(f"{'{0: <18}'.format(handler_name)} - already installed")
-                continue
-            result = install_dependencies(import_meta.get("dependencies", []))
-            if result.get("success") is True:
-                logger.info(
-                    f"{'{0: <18}'.format(handler_name)} - successfully installed"
-                )
-            else:
-                logger.info(
-                    f"{'{0: <18}'.format(handler_name)} - error during dependencies installation: {result.get('error_message', 'unknown error')}"
-                )
-        sys.exit(0)
+        api_arr = [TrunkProcessEnum(name) for name in apis.split(",")]
 
     logger.info(f"Version: {mindsdb_version}")
     logger.info(f"Configuration file: {config.config_path or 'absent'}")
@@ -370,50 +415,32 @@ if __name__ == '__main__':
     logger.debug(f"System config: {config.auto_config}")
     logger.debug(f"Env config: {config.env_config}")
 
+    is_cloud = config.is_cloud
     unexisting_pids = clean_unlinked_process_marks()
     if not is_cloud:
+        logger.debug("Applying database migrations")
+        try:
+            from mindsdb.migrations import migrate
+
+            migrate.migrate_to_head()
+        except Exception as e:
+            logger.error(f"Error! Something went wrong during DB migrations: {e}")
+
+        validate_default_project()
+
         if len(unexisting_pids) > 0:
             set_error_model_status_by_pids(unexisting_pids)
         set_error_model_status_for_unfinished()
-
-        integration_controller.create_permanent_integrations()
-
-        # region Mark old predictors as outdated
-        is_modified = False
-        predictor_records = (
-            db.session.query(db.Predictor)
-            .filter(db.Predictor.deleted_at.is_(None))
-            .all()
-        )
-        if len(predictor_records) > 0:
-            (
-                sucess,
-                compatible_versions,
-            ) = get_versions_where_predictors_become_obsolete()
-            if sucess is True:
-                compatible_versions = [version.parse(x) for x in compatible_versions]
-                mindsdb_version_parsed = version.parse(mindsdb_version)
-                compatible_versions = [x for x in compatible_versions if x <= mindsdb_version_parsed]
-                if len(compatible_versions) > 0:
-                    last_compatible_version = compatible_versions[-1]
-                    for predictor_record in predictor_records:
-                        if (
-                            isinstance(predictor_record.mindsdb_version, str)
-                            and version.parse(predictor_record.mindsdb_version) < last_compatible_version
-                        ):
-                            predictor_record.update_status = "available"
-                            is_modified = True
-        if is_modified is True:
-            db.session.commit()
-        # endregion
+        create_permanent_integrations()
 
     clean_process_marks()
 
     # Get config values for APIs
-    http_api_config = config.get('api', {}).get('http', {})
-    mysql_api_config = config.get('api', {}).get('mysql', {})
-    mcp_api_config = config.get('api', {}).get('mcp', {})
-    litellm_api_config = config.get('api', {}).get('litellm', {})
+    http_api_config = config.get("api", {}).get("http", {})
+    mysql_api_config = config.get("api", {}).get("mysql", {})
+    mcp_api_config = config.get("api", {}).get("mcp", {})
+    litellm_api_config = config.get("api", {}).get("litellm", {})
+    a2a_api_config = config.get("a2a", {})
     trunc_processes_struct = {
         TrunkProcessEnum.HTTP: TrunkProcessData(
             name=TrunkProcessEnum.HTTP.value,
@@ -469,6 +496,7 @@ if __name__ == '__main__':
             entrypoint=start_mcp,
             port=mcp_api_config.get('port', 47337),
             args=(config.cmd_args.verbose,),
+            need_to_run=mcp_api_config.get('need_to_run', False),
             restart_on_failure=mcp_api_config.get('restart_on_failure', False),
             max_restart_count=mcp_api_config.get('max_restart_count', TrunkProcessData.max_restart_count),
             max_restart_interval_seconds=mcp_api_config.get(
@@ -485,6 +513,20 @@ if __name__ == '__main__':
             max_restart_interval_seconds=litellm_api_config.get(
                 'max_restart_interval_seconds', TrunkProcessData.max_restart_interval_seconds
             )
+        ),
+        TrunkProcessEnum.A2A: TrunkProcessData(
+            name=TrunkProcessEnum.A2A.value,
+            entrypoint=start_a2a,
+            port=a2a_api_config.get('port', 8001),
+            args=(config.cmd_args.verbose,),
+            need_to_run=a2a_api_config.get('enabled', False),
+            restart_on_failure=a2a_api_config.get('restart_on_failure', True),
+            max_restart_count=a2a_api_config.get(
+                'max_restart_count', TrunkProcessData.max_restart_count
+            ),
+            max_restart_interval_seconds=a2a_api_config.get(
+                'max_restart_interval_seconds', TrunkProcessData.max_restart_interval_seconds
+            )
         )
     }
 
@@ -494,35 +536,20 @@ if __name__ == '__main__':
         else:
             logger.error(f"ERROR: {api_enum} API is not a valid api in config")
 
-    if config['jobs']['disable'] is False:
+    if config["jobs"]["disable"] is False:
         trunc_processes_struct[TrunkProcessEnum.JOBS].need_to_run = True
 
-    if config['tasks']['disable'] is False:
+    if config["tasks"]["disable"] is False:
         trunc_processes_struct[TrunkProcessEnum.TASKS].need_to_run = True
 
     if config.cmd_args.ml_task_queue_consumer is True:
         trunc_processes_struct[TrunkProcessEnum.ML_TASK_QUEUE].need_to_run = True
 
-    def start_process(trunc_process_data):
-        # TODO this 'ctx' is eclipsing 'context' class imported as 'ctx'
-        ctx = mp.get_context("spawn")
-        logger.info(f"{trunc_process_data.name} API: starting...")
-        try:
-            trunc_process_data.process = ctx.Process(
-                target=trunc_process_data.entrypoint,
-                args=trunc_process_data.args,
-                name=trunc_process_data.name
-            )
-            trunc_process_data.process.start()
-        except Exception as e:
-            logger.error(
-                f"Failed to start {trunc_process_data.name} API with exception {e}\n{traceback.format_exc()}"
-            )
-            close_api_gracefully(trunc_processes_struct)
-            raise e
-
     for trunc_process_data in trunc_processes_struct.values():
-        if trunc_process_data.started is True or trunc_process_data.need_to_run is False:
+        if (
+            trunc_process_data.started is True
+            or trunc_process_data.need_to_run is False
+        ):
             continue
         start_process(trunc_process_data)
 
@@ -542,10 +569,11 @@ if __name__ == '__main__':
             wait_api_start(
                 trunc_process_data.name,
                 trunc_process_data.process.pid,
-                trunc_process_data.port
+                trunc_process_data.port,
             )
             for trunc_process_data in trunc_processes_struct.values()
-            if trunc_process_data.port is not None and trunc_process_data.need_to_run is True
+            if trunc_process_data.port is not None
+            and trunc_process_data.need_to_run is True
         ]
         for future in asyncio.as_completed(futures):
             api_name, port, started = await future
@@ -568,7 +596,9 @@ if __name__ == '__main__':
             finally:
                 if trunc_process_data.should_restart:
                     if trunc_process_data.request_restart_attempt():
-                        logger.warning(f"{trunc_process_data.name} API: stopped unexpectedly, restarting")
+                        logger.warning(
+                            f"{trunc_process_data.name} API: stopped unexpectedly, restarting"
+                        )
                         trunc_process_data.process = None
                         if trunc_process_data.name == TrunkProcessEnum.HTTP.value:
                             # do not open GUI on HTTP API restart
@@ -577,7 +607,7 @@ if __name__ == '__main__':
                         api_name, port, started = await wait_api_start(
                             trunc_process_data.name,
                             trunc_process_data.process.pid,
-                            trunc_process_data.port
+                            trunc_process_data.port,
                         )
                         if started:
                             logger.info(f"{api_name} API: started on {port}")
@@ -587,7 +617,7 @@ if __name__ == '__main__':
                         finish = True
                         logger.error(
                             f'The "{trunc_process_data.name}" process could not restart after failure. '
-                            'There will be no further attempts to restart.'
+                            "There will be no further attempts to restart."
                         )
                 else:
                     finish = True
@@ -600,13 +630,13 @@ if __name__ == '__main__':
                 for trunc_process_data in trunc_processes_struct.values()
                 if trunc_process_data.need_to_run is True
             ],
-            return_exceptions=False
+            return_exceptions=False,
         )
 
     ioloop = asyncio.new_event_loop()
     ioloop.run_until_complete(wait_apis_start())
 
-    threading.Thread(target=do_clean_process_marks, name='clean_process_marks').start()
+    threading.Thread(target=do_clean_process_marks, name="clean_process_marks").start()
 
     ioloop.run_until_complete(gather_apis())
     ioloop.close()