MindsDB 25.9.2.0a1__py3-none-any.whl → 25.10.0rc1__py3-none-any.whl

mindsdb/migrations/migrate.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 
-from alembic.command import upgrade, autogen    # noqa
+from alembic.command import upgrade, autogen  # noqa
 from alembic.config import Config
 from alembic.script import ScriptDirectory
 from alembic.script.revision import ResolutionError
@@ -15,20 +15,19 @@ logger = log.getLogger(__name__)
 
 # This is a migration that is like a 'base version'. Applying only this
 # migration to a fresh DB is equivalent to applying all previous migrations.
-current_checkpoint = '9f150e4f9a05'
+current_checkpoint = "9f150e4f9a05"
 
 
 def apply_checkpoint_migration(script) -> None:
-    """Apply the checkpoint migration to the database.
-    """
+    """Apply the checkpoint migration to the database."""
     with db.engine.begin() as connection:
         context = MigrationContext.configure(
             connection,
             opts={
-                'as_sql': False,
-                'starting_rev': None,  # ignore current version
-                'destination_rev': current_checkpoint,
-            }
+                "as_sql": False,
+                "starting_rev": None,  # ignore current version
+                "destination_rev": current_checkpoint,
+            },
         )
         revision = script.get_revision(current_checkpoint)
         if not revision:
@@ -41,7 +40,7 @@ def apply_checkpoint_migration(script) -> None:
 
 
 def get_current_revision() -> str | None:
-    """ Get the current revision of the database.
+    """Get the current revision of the database.
 
     Returns:
         str | None: The current revision of the database.
@@ -52,17 +51,18 @@ def get_current_revision() -> str | None:
 
 
 def migrate_to_head():
-    """ Trying to update database to head revision.
-        If alembic unable to recognize current revision (In case when database version is newer than backend)
-        then do nothing.
+    """Trying to update database to head revision.
+    If alembic unable to recognize current revision (In case when database version is newer than backend)
+    then do nothing.
     """
+    logger.debug("Applying database migrations")
 
-    config_file = Path(__file__).parent / 'alembic.ini'
+    config_file = Path(__file__).parent / "alembic.ini"
     config = Config(config_file)
 
     # mindsdb can runs not from project directory
-    script_location_abc = config_file.parent / config.get_main_option('script_location')
-    config.set_main_option('script_location', str(script_location_abc))
+    script_location_abc = config_file.parent / config.get_main_option("script_location")
+    config.set_main_option("script_location", str(script_location_abc))
 
     script = ScriptDirectory.from_config(config)
     cur_revision = get_current_revision()
@@ -81,7 +81,7 @@ def migrate_to_head():
         return
 
     logger.info("Migrations are available. Applying updates to the database.")
-    upgrade(config=config, revision='head')
+    upgrade(config=config, revision="head")
 
 
 if __name__ == "__main__":

mindsdb/utilities/api_status.py

@@ -0,0 +1,58 @@
+import os
+import json
+
+from mindsdb.utilities.config import config
+from mindsdb.utilities import log
+
+
+logger = log.getLogger(__name__)
+_api_status_file = None
+
+
+def _get_api_status_file():
+    global _api_status_file
+    if _api_status_file is None:
+        # Use a temporary file that can be shared across processes.
+        temp_dir = config["paths"]["tmp"]
+        _api_status_file = os.path.join(temp_dir, "mindsdb_api_status.json")
+        # Overwrite the file if it exists.
+        if os.path.exists(_api_status_file):
+            try:
+                os.remove(_api_status_file)
+            except OSError:
+                logger.exception(f"Error removing existing API status file: {_api_status_file}")
+
+    return _api_status_file
+
+
+def get_api_status():
+    """Get the current API status from the shared file."""
+    status_file = _get_api_status_file()
+    try:
+        if os.path.exists(status_file):
+            with open(status_file, "r") as f:
+                return json.load(f)
+    except (json.JSONDecodeError, IOError):
+        pass
+    return {}
+
+
+def set_api_status(api_name: str, status: bool):
+    """Set the status of an API in the shared file."""
+    status_file = _get_api_status_file()
+    current_status = get_api_status()
+    current_status[api_name] = status
+
+    # Write atomically to avoid race conditions.
+    temp_file = status_file + ".tmp"
+    try:
+        with open(temp_file, "w") as f:
+            json.dump(current_status, f)
+        os.replace(temp_file, status_file)
+    except IOError:
+        # Clean up temp file if it exists.
+        if os.path.exists(temp_file):
+            try:
+                os.remove(temp_file)
+            except OSError:
+                pass

mindsdb/utilities/config.py

@@ -170,6 +170,7 @@ class Config:
                     "restart_on_failure": True,
                     "max_restart_count": 1,
                     "max_restart_interval_seconds": 60,
+                    "a2wsgi": {"workers": 10, "send_queue_size": 10},
                 },
                 "mysql": {
                     "host": api_host,
@@ -180,7 +181,6 @@ class Config:
                     "max_restart_count": 1,
                     "max_restart_interval_seconds": 60,
                 },
-                "postgres": {"host": api_host, "port": "55432", "database": "mindsdb"},
                 "litellm": {
                     "host": "0.0.0.0",  # API server binds to all interfaces by default
                     "port": "8000",
@@ -233,7 +233,17 @@ class Config:
 
         # region storage root path
         if os.environ.get("MINDSDB_STORAGE_DIR", "") != "":
-            self._env_config["paths"] = {"root": Path(os.environ["MINDSDB_STORAGE_DIR"])}
+            storage_root_path = Path(os.environ["MINDSDB_STORAGE_DIR"])
+            self._env_config["paths"] = {
+                "root": storage_root_path,
+                "content": storage_root_path / "content",
+                "storage": storage_root_path / "storage",
+                "static": storage_root_path / "static",
+                "tmp": storage_root_path / "tmp",
+                "log": storage_root_path / "log",
+                "cache": storage_root_path / "cache",
+                "locks": storage_root_path / "locks",
+            }
         # endregion
 
         # region vars: permanent storage disabled?
@@ -300,6 +310,54 @@ class Config:
             self._env_config["default_reranking_model"] = {
                 "api_key": os.environ["MINDSDB_DEFAULT_RERANKING_MODEL_API_KEY"]
             }
+
+        # Reranker configuration from environment variables
+        reranker_config = {}
+        if os.environ.get("MINDSDB_RERANKER_N", "") != "":
+            try:
+                reranker_config["n"] = int(os.environ["MINDSDB_RERANKER_N"])
+            except ValueError:
+                raise ValueError(f"MINDSDB_RERANKER_N must be an integer, got: {os.environ['MINDSDB_RERANKER_N']}")
+
+        if os.environ.get("MINDSDB_RERANKER_LOGPROBS", "") != "":
+            logprobs_value = os.environ["MINDSDB_RERANKER_LOGPROBS"].lower()
+            if logprobs_value in ("true", "1", "yes", "y"):
+                reranker_config["logprobs"] = True
+            elif logprobs_value in ("false", "0", "no", "n"):
+                reranker_config["logprobs"] = False
+            else:
+                raise ValueError(
+                    f"MINDSDB_RERANKER_LOGPROBS must be a boolean value, got: {os.environ['MINDSDB_RERANKER_LOGPROBS']}"
+                )
+
+        if os.environ.get("MINDSDB_RERANKER_TOP_LOGPROBS", "") != "":
+            try:
+                reranker_config["top_logprobs"] = int(os.environ["MINDSDB_RERANKER_TOP_LOGPROBS"])
+            except ValueError:
+                raise ValueError(
+                    f"MINDSDB_RERANKER_TOP_LOGPROBS must be an integer, got: {os.environ['MINDSDB_RERANKER_TOP_LOGPROBS']}"
+                )
+
+        if os.environ.get("MINDSDB_RERANKER_MAX_TOKENS", "") != "":
+            try:
+                reranker_config["max_tokens"] = int(os.environ["MINDSDB_RERANKER_MAX_TOKENS"])
+            except ValueError:
+                raise ValueError(
+                    f"MINDSDB_RERANKER_MAX_TOKENS must be an integer, got: {os.environ['MINDSDB_RERANKER_MAX_TOKENS']}"
+                )
+
+        if os.environ.get("MINDSDB_RERANKER_VALID_CLASS_TOKENS", "") != "":
+            try:
+                reranker_config["valid_class_tokens"] = os.environ["MINDSDB_RERANKER_VALID_CLASS_TOKENS"].split(",")
+            except ValueError:
+                raise ValueError(
+                    f"MINDSDB_RERANKER_VALID_CLASS_TOKENS must be a comma-separated list of strings, got: {os.environ['MINDSDB_RERANKER_VALID_CLASS_TOKENS']}"
+                )
+
+        if reranker_config:
+            if "default_reranking_model" not in self._env_config:
+                self._env_config["default_reranking_model"] = {}
+            self._env_config["default_reranking_model"].update(reranker_config)
         if os.environ.get("MINDSDB_DATA_CATALOG_ENABLED", "").lower() in ("1", "true"):
             self._env_config["data_catalog"] = {"enabled": True}
 
@@ -307,6 +365,14 @@ class Config:
             self._env_config["gui"]["open_on_start"] = False
             self._env_config["gui"]["autoupdate"] = False
 
+        mindsdb_gui_autoupdate = os.environ.get("MINDSDB_GUI_AUTOUPDATE", "").lower()
+        if mindsdb_gui_autoupdate in ("0", "false"):
+            self._env_config["gui"]["autoupdate"] = False
+        elif mindsdb_gui_autoupdate in ("1", "true"):
+            self._env_config["gui"]["autoupdate"] = True
+        elif mindsdb_gui_autoupdate != "":
+            raise ValueError(f"Wrong value of env var MINDSDB_GUI_AUTOUPDATE={mindsdb_gui_autoupdate}")
+
     def fetch_auto_config(self) -> bool:
         """Load dict readed from config.auto.json to `auto_config`.
         Do it only if `auto_config` was not loaded before or config.auto.json been changed.

mindsdb/utilities/exception.py

@@ -1,7 +1,14 @@
 from textwrap import indent
 
 
-class BaseEntityException(Exception):
+from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import ERR
+
+
+class MindsDBError(Exception):
+    pass
+
+
+class BaseEntityException(MindsDBError):
     """Base exception for entitys errors
 
     Attributes:
@@ -35,6 +42,38 @@ class EntityNotExistsError(BaseEntityException):
         super().__init__(message, entity_name)
 
 
+class ParsingError(MindsDBError):
+    pass
+
+
+class QueryError(MindsDBError):
+    def __init__(
+        self,
+        db_name: str | None = None,
+        db_type: str | None = None,
+        db_error_msg: str | None = None,
+        failed_query: str | None = None,
+        is_external: bool = True,
+        is_expected: bool = False,
+    ) -> None:
+        self.mysql_error_code = ERR.ER_UNKNOWN_ERROR
+        self.db_name = db_name
+        self.db_type = db_type
+        self.db_error_msg = db_error_msg
+        self.failed_query = failed_query
+        self.is_external = is_external
+        self.is_expected = is_expected
+
+    def __str__(self) -> str:
+        return format_db_error_message(
+            db_name=self.db_name,
+            db_type=self.db_type,
+            db_error_msg=self.db_error_msg,
+            failed_query=self.failed_query,
+            is_external=self.is_external,
+        )
+
+
 def format_db_error_message(
     db_name: str | None = None,
     db_type: str | None = None,

mindsdb/utilities/fs.py

@@ -118,7 +118,6 @@ def clean_unlinked_process_marks() -> List[int]:
 
         except psutil.AccessDenied:
             logger.warning(f"access to {process_id} denied")
-
             continue
 
         except psutil.NoSuchProcess:

mindsdb/utilities/hooks/profiling.py

@@ -46,28 +46,31 @@ def send_profiling_results(profiling_data: dict):
             user=MINDSDB_PROFILING_DB_USER,
             password=MINDSDB_PROFILING_DB_PASSWORD,
             dbname="postgres",
-            connect_timeout=5
+            connect_timeout=5,
         )
     except Exception:
-        logger.error('cant get acceess to profiling database')
+        logger.warning("cant get acceess to profiling database")
         return
     cur = connection.cursor()
-    cur.execute("""
+    cur.execute(
+        """
         insert into profiling
             (data, query, time, hostname, environment, api, total_time, company_id, instance_id)
         values
             (%s, %s, %s, %s, %s, %s, %s, %s, %s)
-    """, (
-        json.dumps(profiling["tree"]),
-        profiling.get("query", "?"),
-        time_start_at,
-        profiling["hostname"],
-        profiling.get("environment", "?"),
-        profiling.get("api", "?"),
-        profiling["tree"]["value"],
-        profiling["company_id"],
-        profiling["instance_id"]
-    ))
+    """,
+        (
+            json.dumps(profiling["tree"]),
+            profiling.get("query", "?"),
+            time_start_at,
+            profiling["hostname"],
+            profiling.get("environment", "?"),
+            profiling.get("api", "?"),
+            profiling["tree"]["value"],
+            profiling["company_id"],
+            profiling["instance_id"],
+        ),
+    )
 
     connection.commit()
     cur.close()

mindsdb/utilities/json_encoder.py

@@ -1,13 +1,12 @@
 from datetime import datetime, date, timedelta
 from decimal import Decimal
-import numpy as np
 import pandas as pd
-import json
-
+import numpy as np
+import orjson
 from flask.json.provider import DefaultJSONProvider
 
 
-class CustomJSONEncoder(json.JSONEncoder):
+class CustomJSONEncoder:
     def default(self, obj):
         if isinstance(obj, timedelta):
             return str(obj)
@@ -15,12 +14,10 @@ class CustomJSONEncoder(json.JSONEncoder):
             return obj.strftime("%Y-%m-%d %H:%M:%S.%f")
         if isinstance(obj, date):
             return obj.strftime("%Y-%m-%d")
+        if isinstance(obj, Decimal):
+            return float(obj)
         if isinstance(obj, np.bool_):
             return bool(obj)
-        if isinstance(obj, np.int8) or isinstance(obj, np.int16) or isinstance(obj, np.int32) or isinstance(obj, np.int64):
-            return int(obj)
-        if isinstance(obj, np.float16) or isinstance(obj, np.float32) or isinstance(obj, np.float64) or isinstance(obj, Decimal):
-            return float(obj)
         if isinstance(obj, np.ndarray):
             return obj.tolist()
         if pd.isnull(obj):
@@ -29,5 +26,22 @@ class CustomJSONEncoder(json.JSONEncoder):
         return str(obj)
 
 
-class CustomJSONProvider(CustomJSONEncoder, DefaultJSONProvider):
-    ...
+class ORJSONProvider(DefaultJSONProvider):
+    """
+    Use orjson to serialize data instead of flask json provider.
+    """
+
+    def dumps(self, obj, **kwargs):
+        return orjson.dumps(
+            obj,
+            option=(
+                orjson.OPT_SERIALIZE_NUMPY
+                | orjson.OPT_NON_STR_KEYS
+                # keep this for using CustomJSON encoder
+                | orjson.OPT_PASSTHROUGH_DATETIME
+            ),
+            default=CustomJSONEncoder().default,
+        ).decode("utf-8")
+
+    def loads(self, s, **kwargs):
+        return orjson.loads(s)

mindsdb/utilities/langfuse.py

@@ -44,17 +44,19 @@ class LangfuseClientWrapper:
     Langfuse client wrapper. Defines Langfuse client configuration and initializes Langfuse client.
     """
 
-    def __init__(self,
-                 public_key: str = LANGFUSE_PUBLIC_KEY,
-                 secret_key: str = LANGFUSE_SECRET_KEY,
-                 host: str = LANGFUSE_HOST,
-                 environment: str = LANGFUSE_ENVIRONMENT,
-                 release: str = LANGFUSE_RELEASE,
-                 debug: bool = LANGFUSE_DEBUG,
-                 timeout: int = LANGFUSE_TIMEOUT,
-                 sample_rate: float = LANGFUSE_SAMPLE_RATE,
-                 disable: bool = LANGFUSE_DISABLED,
-                 force_run: bool = LANGFUSE_FORCE_RUN) -> None:
+    def __init__(
+        self,
+        public_key: str = LANGFUSE_PUBLIC_KEY,
+        secret_key: str = LANGFUSE_SECRET_KEY,
+        host: str = LANGFUSE_HOST,
+        environment: str = LANGFUSE_ENVIRONMENT,
+        release: str = LANGFUSE_RELEASE,
+        debug: bool = LANGFUSE_DEBUG,
+        timeout: int = LANGFUSE_TIMEOUT,
+        sample_rate: float = LANGFUSE_SAMPLE_RATE,
+        disable: bool = LANGFUSE_DISABLED,
+        force_run: bool = LANGFUSE_FORCE_RUN,
+    ) -> None:
         """
         Initialize Langfuse client.
 
@@ -112,16 +114,18 @@ class LangfuseClientWrapper:
             release=release,
             debug=debug,
             timeout=timeout,
-            sample_rate=sample_rate
+            sample_rate=sample_rate,
         )
 
-    def setup_trace(self,
-                    name: str,
-                    input: typing.Optional[typing.Any] = None,
-                    tags: typing.Optional[typing.List] = None,
-                    metadata: typing.Optional[typing.Dict] = None,
-                    user_id: str = None,
-                    session_id: str = None) -> None:
+    def setup_trace(
+        self,
+        name: str,
+        input: typing.Optional[typing.Any] = None,
+        tags: typing.Optional[typing.List] = None,
+        metadata: typing.Optional[typing.Dict] = None,
+        user_id: str = None,
+        session_id: str = None,
+    ) -> None:
         """
         Setup trace. If Langfuse is disabled, nothing will be done.
         Args:
@@ -142,15 +146,10 @@ class LangfuseClientWrapper:
 
         try:
             self.trace = self.client.trace(
-                name=name,
-                input=input,
-                metadata=self.metadata,
-                tags=self.tags,
-                user_id=user_id,
-                session_id=session_id
+                name=name, input=input, metadata=self.metadata, tags=self.tags, user_id=user_id, session_id=session_id
             )
-        except Exception as e:
-            logger.error(f'Something went wrong while processing Langfuse trace {self.trace.id}: {str(e)}')
+        except Exception:
+            logger.exception(f"Something went wrong while processing Langfuse trace {self.trace.id}:")
 
         logger.info(f"Langfuse trace configured with ID: {self.trace.id}")
 
@@ -169,9 +168,7 @@ class LangfuseClientWrapper:
 
         return self.trace.id
 
-    def start_span(self,
-                   name: str,
-                   input: typing.Optional[typing.Any] = None) -> typing.Optional['StatefulSpanClient']:
+    def start_span(self, name: str, input: typing.Optional[typing.Any] = None) -> typing.Optional["StatefulSpanClient"]:
         """
         Create span. If Langfuse is disabled, nothing will be done.
 
@@ -186,8 +183,7 @@ class LangfuseClientWrapper:
 
         return self.trace.span(name=name, input=input)
 
-    def end_span_stream(self,
-                        span: typing.Optional['StatefulSpanClient'] = None) -> None:
+    def end_span_stream(self, span: typing.Optional["StatefulSpanClient"] = None) -> None:
         """
         End span. If Langfuse is disabled, nothing will happen.
         Args:
@@ -201,9 +197,9 @@ class LangfuseClientWrapper:
         span.end()
         self.trace.update()
 
-    def end_span(self,
-                 span: typing.Optional['StatefulSpanClient'] = None,
-                 output: typing.Optional[typing.Any] = None) -> None:
+    def end_span(
+        self, span: typing.Optional["StatefulSpanClient"] = None, output: typing.Optional[typing.Any] = None
+    ) -> None:
         """
         End trace. If Langfuse is disabled, nothing will be done.
 
@@ -228,13 +224,12 @@ class LangfuseClientWrapper:
         try:
             # Ensure all batched traces are sent before fetching.
             self.client.flush()
-            metadata['tool_usage'] = self._get_tool_usage()
+            metadata["tool_usage"] = self._get_tool_usage()
             self.trace.update(metadata=metadata)
+        except Exception:
+            logger.exception(f"Something went wrong while processing Langfuse trace {self.trace.id}:")
 
-        except Exception as e:
-            logger.error(f'Something went wrong while processing Langfuse trace {self.trace.id}: {str(e)}')
-
-    def get_langchain_handler(self) -> typing.Optional['CallbackHandler']:
+    def get_langchain_handler(self) -> typing.Optional["CallbackHandler"]:
         """
         Get Langchain handler. If Langfuse is disabled, returns None.
         """
@@ -275,14 +270,14 @@ class LangfuseClientWrapper:
             fetched_trace = self.client.get_trace(self.trace.id)
             steps = [s.name for s in fetched_trace.observations]
             for step in steps:
-                if 'AgentAction' in step:
-                    tool_name = step.split('-')[1]
+                if "AgentAction" in step:
+                    tool_name = step.split("-")[1]
                     if tool_name not in tool_usage:
                         tool_usage[tool_name] = 0
                     tool_usage[tool_name] += 1
         except TraceNotFoundError:
-            logger.warning(f'Langfuse trace {self.trace.id} not found')
-        except Exception as e:
-            logger.error(f'Something went wrong while processing Langfuse trace {self.trace.id}: {str(e)}')
+            logger.warning(f"Langfuse trace {self.trace.id} not found")
+        except Exception:
+            logger.exception(f"Something went wrong while processing Langfuse trace {self.trace.id}:")
 
         return tool_usage