MindsDB 25.9.3rc1__py3-none-any.whl → 25.10.0__py3-none-any.whl

mindsdb/interfaces/knowledge_base/llm_client.py

@@ -1,11 +1,23 @@
-import copy
 import os
 from typing import List
 
 from openai import OpenAI, AzureOpenAI
 
 from mindsdb.integrations.utilities.handler_utils import get_api_key
-from mindsdb.utilities.config import config
+
+try:
+    from mindsdb.integrations.handlers.openai_handler.helpers import retry_with_exponential_backoff
+except ImportError:
+
+    def retry_with_exponential_backoff(func):
+        """
+        An empty decorator
+        """
+
+        def wrapper(*args, **kwargs):
+            return func(*args, **kwargs)
+
+        return wrapper
 
 
 class LLMClient:
@@ -14,12 +26,8 @@ class LLMClient:
     It chooses openai client or litellm handler depending on the config
     """
 
-    def __init__(self, llm_params: dict = None):
-        params = copy.deepcopy(config.get("default_llm", {}))
-
-        if llm_params:
-            params.update(llm_params)
-
+    def __init__(self, params: dict = None, session=None):
+        self._session = session
         self.params = params
 
         self.provider = params.get("provider", "openai")
@@ -27,11 +35,13 @@ class LLMClient:
         if "api_key" not in params:
             params["api_key"] = get_api_key(self.provider, params, strict=False)
 
+        self.engine = "openai"
+
         if self.provider == "azure_openai":
             azure_api_key = params.get("api_key") or os.getenv("AZURE_OPENAI_API_KEY")
             azure_api_endpoint = params.get("base_url") or os.environ.get("AZURE_OPENAI_ENDPOINT")
             azure_api_version = params.get("api_version") or os.environ.get("AZURE_OPENAI_API_VERSION")
-            self._llm_client = AzureOpenAI(
+            self.client = AzureOpenAI(
                 api_key=azure_api_key, azure_endpoint=azure_api_endpoint, api_version=azure_api_version, max_retries=2
             )
         elif self.provider == "openai":
@@ -41,34 +51,58 @@ class LLMClient:
             if base_url:
                 kwargs["base_url"] = base_url
             self.client = OpenAI(**kwargs)
-
+        elif self.provider == "ollama":
+            kwargs = params.copy()
+            kwargs.pop("model_name")
+            kwargs.pop("provider", None)
+            if kwargs["api_key"] is None:
+                kwargs["api_key"] = "n/a"
+            self.client = OpenAI(**kwargs)
         else:
             # try to use litellm
-            from mindsdb.api.executor.controllers.session_controller import SessionController
+            if self._session is None:
+                from mindsdb.api.executor.controllers.session_controller import SessionController
 
-            session = SessionController()
-            module = session.integration_controller.get_handler_module("litellm")
+                self._session = SessionController()
+            module = self._session.integration_controller.get_handler_module("litellm")
 
             if module is None or module.Handler is None:
                 raise ValueError(f'Unable to use "{self.provider}" provider. Litellm handler is not installed')
 
             self.client = module.Handler
+            self.engine = "litellm"
+
+    @retry_with_exponential_backoff()
+    def embeddings(self, messages: List[str]):
+        params = self.params
+        if self.engine == "openai":
+            response = self.client.embeddings.create(
+                model=params["model_name"],
+                input=messages,
+            )
+            return [item.embedding for item in response.data]
+        else:
+            kwargs = params.copy()
+            model = kwargs.pop("model_name")
+            kwargs.pop("provider", None)
+
+            return self.client.embeddings(self.provider, model=model, messages=messages, args=kwargs)
 
-    def completion(self, messages: List[dict], json_output: bool = False) -> str:
+    def completion(self, messages: List[dict], json_output: bool = False) -> List[str]:
         """
         Call LLM completion and get response
         """
         params = self.params
         params["json_output"] = json_output
-        if self.provider in ("azure_openai", "openai"):
+        if self.engine == "openai":
             response = self.client.chat.completions.create(
                 model=params["model_name"],
                 messages=messages,
             )
-            return response.choices[0].message.content
+            return [item.message.content for item in response.choices]
         else:
             kwargs = params.copy()
             model = kwargs.pop("model_name")
             kwargs.pop("provider", None)
             response = self.client.completion(self.provider, model=model, messages=messages, args=kwargs)
-            return response.choices[0].message.content
+            return [item.message.content for item in response.choices]

mindsdb/interfaces/model/model_controller.py

@@ -246,8 +246,8 @@ class ModelController:
 
     def prepare_create_statement(self, statement, database_controller):
         # extract data from Create model or Retrain statement and prepare it for using in crate and retrain functions
-        project_name = statement.name.parts[0].lower()
-        model_name = statement.name.parts[1].lower()
+        project_name = statement.name.parts[0]
+        model_name = statement.name.parts[1]
 
         sql_task = None
         if statement.task is not None:
@@ -294,11 +294,11 @@ class ModelController:
     def create_model(self, statement, ml_handler):
         params = self.prepare_create_statement(statement, ml_handler.database_controller)
 
-        existing_projects_meta = ml_handler.database_controller.get_dict(filter_type="project")
+        existing_projects_meta = ml_handler.database_controller.get_dict(filter_type="project", lowercase=False)
         if params["project_name"] not in existing_projects_meta:
             raise EntityNotExistsError("Project does not exist", params["project_name"])
 
-        project = ml_handler.database_controller.get_project(name=params["project_name"])
+        project = ml_handler.database_controller.get_project(name=params["project_name"], strict_case=True)
         project_tables = project.get_tables()
         if params["model_name"] in project_tables:
             raise EntityExistsError("Model already exists", f"{params['project_name']}.{params['model_name']}")

mindsdb/interfaces/skills/custom/text2sql/mindsdb_sql_toolkit.py

@@ -177,29 +177,23 @@ class MindsDBSQLToolkit(SQLDatabaseToolkit):
 
                 Query Types and Examples:
                 1. Basic semantic search:
-                   kb_query_tool("SELECT * FROM kb_name WHERE content = 'your search query';")
+                   kb_query_tool("SELECT * FROM kb_name WHERE chunk_content = 'your search query';")
 
                 2. Metadata filtering:
                    kb_query_tool("SELECT * FROM kb_name WHERE metadata_field = 'value';")
 
                 3. Combined search:
-                   kb_query_tool("SELECT * FROM kb_name WHERE content = 'query' AND metadata_field = 'value';")
+                   kb_query_tool("SELECT * FROM kb_name WHERE chunk_content = 'query' AND metadata_field = 'value';")
 
                 4. Setting relevance threshold:
-                   kb_query_tool("SELECT * FROM kb_name WHERE content = 'query' AND relevance_threshold = 0.7;")
+                   kb_query_tool("SELECT * FROM kb_name WHERE chunk_content = 'query' AND relevance_threshold = 0.7;")
 
                 5. Limiting results:
-                   kb_query_tool("SELECT * FROM kb_name WHERE content = 'query' LIMIT 5;")
+                   kb_query_tool("SELECT * FROM kb_name WHERE chunk_content = 'query' LIMIT 5;")
 
                 6. Getting sample data:
                    kb_query_tool("SELECT * FROM kb_name LIMIT 3;")
 
-                7. Don't use LIKE operator on content filter ie semantic search:
-                SELECT * FROM `test_kb` WHERE content LIKE '%population of New York%' $STOP$
-
-                Like is not supported, use the following instead:
-                SELECT * FROM `test_kb` WHERE content = 'population of New York'
-
                 Result Format:
                 - Results include: id, chunk_id, chunk_content, metadata, distance, and relevance columns
                 - The metadata column contains a JSON object with all metadata fields

mindsdb/interfaces/skills/skills_controller.py

@@ -100,10 +100,7 @@ class SkillsController:
             project_name = default_project
         project = self.project_controller.get(name=project_name)
 
-        if not name.islower():
-            raise ValueError(f"The name must be in lower case: {name}")
-
-        skill = self.get_skill(name, project_name)
+        skill = self.get_skill(name, project_name, strict_case=True)
 
         if skill is not None:
             raise ValueError(f"Skill with name already exists: {name}")

mindsdb/interfaces/storage/db.py

@@ -1,4 +1,5 @@
 import json
+import orjson
 import datetime
 from typing import Dict, List, Optional
 
@@ -47,10 +48,20 @@ def init(connection_str: str = None):
     global Base, session, engine
     if connection_str is None:
         connection_str = config["storage_db"]
+    # Use orjson with our CustomJSONEncoder.default for JSON serialization
+    _default_json = CustomJSONEncoder().default
+
+    def _json_serializer(value):
+        return orjson.dumps(
+            value,
+            default=_default_json,
+            option=orjson.OPT_SERIALIZE_NUMPY | orjson.OPT_PASSTHROUGH_DATETIME,
+        ).decode("utf-8")
+
     base_args = {
         "pool_size": 30,
         "max_overflow": 200,
-        "json_serializer": CustomJSONEncoder().encode,
+        "json_serializer": _json_serializer,
     }
     engine = create_engine(connection_str, echo=False, **base_args)
     session = scoped_session(sessionmaker(bind=engine, autoflush=True))
@@ -534,11 +545,10 @@ class KnowledgeBase(Base):
         reranking_model = params.pop("reranking_model", None)
 
         if not with_secrets:
-            if embedding_model and "api_key" in embedding_model:
-                embedding_model["api_key"] = "******"
-
-            if reranking_model and "api_key" in reranking_model:
-                reranking_model["api_key"] = "******"
+            for key in ("api_key", "private_key"):
+                for el in (embedding_model, reranking_model):
+                    if el and key in el:
+                        el[key] = "******"
 
         return {
             "id": self.id,

mindsdb/interfaces/triggers/triggers_controller.py

@@ -14,8 +14,6 @@ class TriggersController:
     OBJECT_TYPE = "trigger"
 
     def add(self, name, project_name, table, query_str, columns=None):
-        name = name.lower()
-
         if project_name is None:
             project_name = config.get("default_project")
         project_controller = ProjectController()
@@ -155,7 +153,7 @@ class TriggersController:
                 {
                     "id": record.object_id,
                     "project": project_names[record.project_id],
-                    "name": record.name.lower(),
+                    "name": record.name,
                     "database": database_names.get(record.database_id, "?"),
                     "table": record.table_name,
                     "query": record.query_str,

mindsdb/utilities/config.py

@@ -181,7 +181,6 @@ class Config:
                     "max_restart_count": 1,
                     "max_restart_interval_seconds": 60,
                 },
-                "postgres": {"host": api_host, "port": "55432", "database": "mindsdb"},
                 "litellm": {
                     "host": "0.0.0.0",  # API server binds to all interfaces by default
                     "port": "8000",
@@ -234,7 +233,17 @@ class Config:
 
         # region storage root path
         if os.environ.get("MINDSDB_STORAGE_DIR", "") != "":
-            self._env_config["paths"] = {"root": Path(os.environ["MINDSDB_STORAGE_DIR"])}
+            storage_root_path = Path(os.environ["MINDSDB_STORAGE_DIR"])
+            self._env_config["paths"] = {
+                "root": storage_root_path,
+                "content": storage_root_path / "content",
+                "storage": storage_root_path / "storage",
+                "static": storage_root_path / "static",
+                "tmp": storage_root_path / "tmp",
+                "log": storage_root_path / "log",
+                "cache": storage_root_path / "cache",
+                "locks": storage_root_path / "locks",
+            }
         # endregion
 
         # region vars: permanent storage disabled?
@@ -356,6 +365,14 @@ class Config:
             self._env_config["gui"]["open_on_start"] = False
             self._env_config["gui"]["autoupdate"] = False
 
+        mindsdb_gui_autoupdate = os.environ.get("MINDSDB_GUI_AUTOUPDATE", "").lower()
+        if mindsdb_gui_autoupdate in ("0", "false"):
+            self._env_config["gui"]["autoupdate"] = False
+        elif mindsdb_gui_autoupdate in ("1", "true"):
+            self._env_config["gui"]["autoupdate"] = True
+        elif mindsdb_gui_autoupdate != "":
+            raise ValueError(f"Wrong value of env var MINDSDB_GUI_AUTOUPDATE={mindsdb_gui_autoupdate}")
+
     def fetch_auto_config(self) -> bool:
         """Load dict readed from config.auto.json to `auto_config`.
         Do it only if `auto_config` was not loaded before or config.auto.json been changed.

mindsdb/utilities/exception.py

@@ -54,7 +54,7 @@ class QueryError(MindsDBError):
         db_error_msg: str | None = None,
         failed_query: str | None = None,
         is_external: bool = True,
-        is_acceptable: bool = False,
+        is_expected: bool = False,
     ) -> None:
         self.mysql_error_code = ERR.ER_UNKNOWN_ERROR
         self.db_name = db_name
@@ -62,7 +62,7 @@ class QueryError(MindsDBError):
         self.db_error_msg = db_error_msg
         self.failed_query = failed_query
         self.is_external = is_external
-        self.is_acceptable = is_acceptable
+        self.is_expected = is_expected
 
     def __str__(self) -> str:
         return format_db_error_message(

mindsdb/utilities/json_encoder.py

@@ -1,13 +1,12 @@
 from datetime import datetime, date, timedelta
 from decimal import Decimal
-import numpy as np
 import pandas as pd
-import json
-
+import numpy as np
+import orjson
 from flask.json.provider import DefaultJSONProvider
 
 
-class CustomJSONEncoder(json.JSONEncoder):
+class CustomJSONEncoder:
     def default(self, obj):
         if isinstance(obj, timedelta):
             return str(obj)
@@ -15,12 +14,10 @@ class CustomJSONEncoder(json.JSONEncoder):
             return obj.strftime("%Y-%m-%d %H:%M:%S.%f")
         if isinstance(obj, date):
             return obj.strftime("%Y-%m-%d")
+        if isinstance(obj, Decimal):
+            return float(obj)
         if isinstance(obj, np.bool_):
             return bool(obj)
-        if isinstance(obj, np.int8) or isinstance(obj, np.int16) or isinstance(obj, np.int32) or isinstance(obj, np.int64):
-            return int(obj)
-        if isinstance(obj, np.float16) or isinstance(obj, np.float32) or isinstance(obj, np.float64) or isinstance(obj, Decimal):
-            return float(obj)
         if isinstance(obj, np.ndarray):
             return obj.tolist()
         if pd.isnull(obj):
@@ -29,5 +26,22 @@ class CustomJSONEncoder(json.JSONEncoder):
         return str(obj)
 
 
-class CustomJSONProvider(CustomJSONEncoder, DefaultJSONProvider):
-    ...
+class ORJSONProvider(DefaultJSONProvider):
+    """
+    Use orjson to serialize data instead of flask json provider.
+    """
+
+    def dumps(self, obj, **kwargs):
+        return orjson.dumps(
+            obj,
+            option=(
+                orjson.OPT_SERIALIZE_NUMPY
+                | orjson.OPT_NON_STR_KEYS
+                # keep this for using CustomJSON encoder
+                | orjson.OPT_PASSTHROUGH_DATETIME
+            ),
+            default=CustomJSONEncoder().default,
+        ).decode("utf-8")
+
+    def loads(self, s, **kwargs):
+        return orjson.loads(s)

mindsdb/utilities/render/sqlalchemy_render.py

@@ -10,6 +10,7 @@ from sqlalchemy.dialects import mysql, postgresql, sqlite, mssql, oracle
 from sqlalchemy.schema import CreateTable, DropTable
 from sqlalchemy.sql import ColumnElement
 from sqlalchemy.sql import functions as sa_fnc
+from sqlalchemy.engine.interfaces import Dialect
 
 from mindsdb_sql_parser import ast
 
@@ -80,27 +81,27 @@ def get_is_quoted(identifier: ast.Identifier):
     return quoted
 
 
-class SqlalchemyRender:
-    def __init__(self, dialect_name):
-        dialects = {
-            "mysql": mysql,
-            "postgresql": postgresql,
-            "postgres": postgresql,
-            "sqlite": sqlite,
-            "mssql": mssql,
-            "oracle": oracle,
-            "Snowflake": oracle,
-        }
+dialects = {
+    "mysql": mysql,
+    "postgresql": postgresql,
+    "postgres": postgresql,
+    "sqlite": sqlite,
+    "mssql": mssql,
+    "oracle": oracle,
+}
+
 
+class SqlalchemyRender:
+    def __init__(self, dialect_name: str | Dialect):
         if isinstance(dialect_name, str):
             dialect = dialects[dialect_name].dialect
         else:
             dialect = dialect_name
 
         # override dialect's preparer
-        if hasattr(dialect, "preparer"):
+        if hasattr(dialect, "preparer") and dialect.preparer.__name__ != "MDBPreparer":
 
-            class Preparer(dialect.preparer):
+            class MDBPreparer(dialect.preparer):
                 def _requires_quotes(self, value: str) -> bool:
                     # check force-quote flag
                     if isinstance(value, AttributedStr):
@@ -116,7 +117,7 @@ class SqlalchemyRender:
                         # or (lc_value != value)
                     )
 
-            dialect.preparer = Preparer
+            dialect.preparer = MDBPreparer
 
         # remove double percent signs
         # https://docs.sqlalchemy.org/en/14/faq/sqlexpressions.html#why-are-percent-signs-being-doubled-up-when-stringifying-sql-statements

mindsdb/utilities/starters.py

@@ -18,16 +18,6 @@ def start_mysql(*args, **kwargs):
     start(*args, **kwargs)
 
 
-def start_postgres(*args, **kwargs):
-    from mindsdb.utilities.log import initialize_logging
-
-    initialize_logging("postgres")
-
-    from mindsdb.api.postgres.start import start
-
-    start(*args, **kwargs)
-
-
 def start_tasks(*args, **kwargs):
     from mindsdb.utilities.log import initialize_logging