MindsDB 25.9.2.0a1__py3-none-any.whl → 25.10.0rc1__py3-none-any.whl

mindsdb/interfaces/database/data_handlers_cache.py

@@ -0,0 +1,190 @@
+import sys
+import time
+import threading
+from dataclasses import dataclass, field
+from collections import defaultdict
+
+from mindsdb.integrations.libs.base import DatabaseHandler
+from mindsdb.utilities.context import context as ctx
+from mindsdb.utilities import log
+
+logger = log.getLogger(__name__)
+
+
+@dataclass(kw_only=True, slots=True)
+class HandlersCacheRecord:
+    """Record for a handler in the cache
+
+    Args:
+        handler (DatabaseHandler): handler instance
+        expired_at (float): time when the handler will be expired
+    """
+
+    handler: DatabaseHandler
+    expired_at: float
+    connect_attempt_done: threading.Event = field(default_factory=threading.Event)
+
+    @property
+    def expired(self) -> bool:
+        """check if the handler is expired
+
+        Returns:
+            bool: True if the handler is expired, False otherwise
+        """
+        return self.expired_at < time.time()
+
+    @property
+    def has_references(self) -> bool:
+        """check if the handler has references
+
+        Returns:
+            bool: True if the handler has references, False otherwise
+        """
+        return sys.getrefcount(self.handler) > 2
+
+    def connect(self) -> None:
+        """connect to the handler"""
+        try:
+            if not self.handler.is_connected:
+                self.handler.connect()
+        except Exception:
+            logger.warning(f"Error connecting to handler: {self.handler.name}", exc_info=True)
+        finally:
+            self.connect_attempt_done.set()
+
+
+class HandlersCache:
+    """Cache for data handlers that keep connections opened during ttl time from handler last use"""
+
+    def __init__(self, ttl: int = 60, clean_timeout: float = 3):
+        """init cache
+
+        Args:
+            ttl (int): time to live (in seconds) for record in cache
+            clean_timeout (float): interval between cleanups of expired handlers
+        """
+        self.ttl: int = ttl
+        self._clean_timeout: int = clean_timeout
+        self.handlers: dict[str, list[HandlersCacheRecord]] = defaultdict(list)
+        self._lock = threading.RLock()
+        self._stop_event = threading.Event()
+        self.cleaner_thread = None
+
+    def __del__(self):
+        self._stop_clean()
+
+    def _start_clean(self) -> None:
+        """start worker that close connections after ttl expired"""
+        if isinstance(self.cleaner_thread, threading.Thread) and self.cleaner_thread.is_alive():
+            return
+        with self._lock:
+            self._stop_event.clear()
+            self.cleaner_thread = threading.Thread(target=self._clean, name="HandlersCache.clean")
+            self.cleaner_thread.daemon = True
+            self.cleaner_thread.start()
+
+    def _stop_clean(self) -> None:
+        """stop clean worker"""
+        self._stop_event.set()
+
+    def set(self, handler: DatabaseHandler):
+        """add (or replace) handler in cache
+
+        NOTE: If the handler is not thread-safe, then use a lock when making connection. Otherwise, make connection in
+        the same thread without using a lock to speed up parallel queries. (They don't need to wait for a connection in
+        another thread.)
+
+        Args:
+            handler (DatabaseHandler)
+        """
+        thread_safe = getattr(handler, "thread_safe", True)
+        with self._lock:
+            try:
+                # If the handler is defined to be thread safe, set 0 as the last element of the key, otherwise set the thrad ID.
+                key = (
+                    handler.name,
+                    ctx.company_id,
+                    0 if thread_safe else threading.get_native_id(),
+                )
+                record = HandlersCacheRecord(handler=handler, expired_at=time.time() + self.ttl)
+                self.handlers[key].append(record)
+            except Exception:
+                logger.warning("Error setting data handler cache record:", exc_info=True)
+                return
+        self._start_clean()
+        record.connect()
+
+    def _get_cache_records(self, name: str) -> tuple[list[HandlersCacheRecord] | None, str]:
+        """get cache records by name
+
+        Args:
+            name (str): handler name
+
+        Returns:
+            tuple[list[HandlersCacheRecord] | None, str]: cache records and key of the handler in cache
+        """
+        # If the handler is not thread safe, the thread ID will be assigned to the last element of the key.
+        key = (name, ctx.company_id, 0)
+        if key not in self.handlers:
+            key = (name, ctx.company_id, threading.get_native_id())
+        return self.handlers.get(key, []), key
+
+    def get(self, name: str) -> DatabaseHandler | None:
+        """get handler from cache by name
+
+        Args:
+            name (str): handler name
+
+        Returns:
+            DatabaseHandler
+        """
+        with self._lock:
+            records, _ = self._get_cache_records(name)
+            for record in records:
+                if record.expired is False and record.has_references is False:
+                    record.expired_at = time.time() + self.ttl
+                    if record.connect_attempt_done.wait(timeout=10) is False:
+                        logger.warning(f"Handler's connection attempt has not finished in 10s: {record.handler.name}")
+                    return record.handler
+            return None
+
+    def delete(self, name: str) -> None:
+        """delete handler from cache
+
+        Args:
+            name (str): handler name
+        """
+        with self._lock:
+            records, key = self._get_cache_records(name)
+            if len(records) > 0:
+                del self.handlers[key]
+            for record in records:
+                try:
+                    record.handler.disconnect()
+                except Exception:
+                    logger.debug("Error disconnecting data handler:", exc_info=True)
+
+            if len(self.handlers) == 0:
+                self._stop_clean()
+
+    def _clean(self) -> None:
+        """worker that delete from cache handlers that was not in use for ttl"""
+        while self._stop_event.wait(timeout=self._clean_timeout) is False:
+            with self._lock:
+                for key in list(self.handlers.keys()):
+                    active_handlers_list = []
+                    for record in self.handlers[key]:
+                        if record.expired and record.has_references is False:
+                            try:
+                                record.handler.disconnect()
+                            except Exception:
+                                logger.debug("Error disconnecting data handler:", exc_info=True)
+                        else:
+                            active_handlers_list.append(record)
+                    if len(active_handlers_list) > 0:
+                        self.handlers[key] = active_handlers_list
+                    else:
+                        del self.handlers[key]
+
+                if len(self.handlers) == 0:
+                    self._stop_event.set()

mindsdb/interfaces/database/database.py

@@ -32,10 +32,10 @@ class DatabaseController:
         Returns:
             None
         """
-        databases = self.get_dict()
-        if name.lower() not in databases:
+        databases = self.get_dict(lowercase=False)
+        if name not in databases:
             raise EntityNotExistsError("Database does not exists", name)
-        db_type = databases[name.lower()]["type"]
+        db_type = databases[name]["type"]
         if db_type == "project":
             project = self.get_project(name, strict_case)
             project.delete()

mindsdb/interfaces/database/integrations.py

@@ -1,5 +1,4 @@
 import os
-import sys
 import base64
 import shutil
 import ast
@@ -33,116 +32,11 @@ from mindsdb.integrations.libs.ml_exec_base import BaseMLEngineExec
 from mindsdb.integrations.libs.base import BaseHandler
 import mindsdb.utilities.profiler as profiler
 from mindsdb.interfaces.data_catalog.data_catalog_loader import DataCatalogLoader
+from mindsdb.interfaces.database.data_handlers_cache import HandlersCache
 
 logger = log.getLogger(__name__)
 
 
-class HandlersCache:
-    """Cache for data handlers that keep connections opened during ttl time from handler last use"""
-
-    def __init__(self, ttl: int = 60):
-        """init cache
-
-        Args:
-            ttl (int): time to live (in seconds) for record in cache
-        """
-        self.ttl = ttl
-        self.handlers = {}
-        self._lock = threading.RLock()
-        self._stop_event = threading.Event()
-        self.cleaner_thread = None
-
-    def __del__(self):
-        self._stop_clean()
-
-    def _start_clean(self) -> None:
-        """start worker that close connections after ttl expired"""
-        if isinstance(self.cleaner_thread, threading.Thread) and self.cleaner_thread.is_alive():
-            return
-        self._stop_event.clear()
-        self.cleaner_thread = threading.Thread(target=self._clean, name="HandlersCache.clean")
-        self.cleaner_thread.daemon = True
-        self.cleaner_thread.start()
-
-    def _stop_clean(self) -> None:
-        """stop clean worker"""
-        self._stop_event.set()
-
-    def set(self, handler: DatabaseHandler):
-        """add (or replace) handler in cache
-
-        Args:
-            handler (DatabaseHandler)
-        """
-        with self._lock:
-            try:
-                # If the handler is defined to be thread safe, set 0 as the last element of the key, otherwise set the thrad ID.
-                key = (
-                    handler.name,
-                    ctx.company_id,
-                    0 if getattr(handler, "thread_safe", False) else threading.get_native_id(),
-                )
-                handler.connect()
-                self.handlers[key] = {"handler": handler, "expired_at": time.time() + self.ttl}
-            except Exception:
-                pass
-            self._start_clean()
-
-    def get(self, name: str) -> Optional[DatabaseHandler]:
-        """get handler from cache by name
-
-        Args:
-            name (str): handler name
-
-        Returns:
-            DatabaseHandler
-        """
-        with self._lock:
-            # If the handler is not thread safe, the thread ID will be assigned to the last element of the key.
-            key = (name, ctx.company_id, threading.get_native_id())
-            if key not in self.handlers:
-                # If the handler is thread safe, a 0 will be assigned to the last element of the key.
-                key = (name, ctx.company_id, 0)
-            if key not in self.handlers or self.handlers[key]["expired_at"] < time.time():
-                return None
-            self.handlers[key]["expired_at"] = time.time() + self.ttl
-            return self.handlers[key]["handler"]
-
-    def delete(self, name: str) -> None:
-        """delete handler from cache
-
-        Args:
-            name (str): handler name
-        """
-        with self._lock:
-            key = (name, ctx.company_id, threading.get_native_id())
-            if key in self.handlers:
-                try:
-                    self.handlers[key].disconnect()
-                except Exception:
-                    pass
-                del self.handlers[key]
-            if len(self.handlers) == 0:
-                self._stop_clean()
-
-    def _clean(self) -> None:
-        """worker that delete from cache handlers that was not in use for ttl"""
-        while self._stop_event.wait(timeout=3) is False:
-            with self._lock:
-                for key in list(self.handlers.keys()):
-                    if (
-                        self.handlers[key]["expired_at"] < time.time()
-                        and sys.getrefcount(self.handlers[key]) == 2  # returned ref count is always 1 higher
-                    ):
-                        try:
-                            self.handlers[key].disconnect()
-                        except Exception:
-                            pass
-                        del self.handlers[key]
-                if len(self.handlers) == 0:
-                    self._stop_event.set()
-
-
 class IntegrationController:
     @staticmethod
     def _is_not_empty_str(s):
@@ -172,9 +66,6 @@ class IntegrationController:
         )
         handler_meta = self.get_handler_meta(engine)
 
-        if not name.islower():
-            raise ValueError(f"The name must be in lower case: {name}")
-
         accept_connection_args = handler_meta.get("connection_args")
         logger.debug("%s: accept_connection_args - %s", self.__class__.__name__, accept_connection_args)
 
@@ -866,6 +757,8 @@ class IntegrationController:
 
     def import_handler(self, handler_name: str, base_import: str = None):
         with self._import_lock:
+            time_before_import = time.perf_counter()
+            logger.debug(f"Importing handler '{handler_name}'")
             handler_meta = self.handlers_import_status[handler_name]
             handler_dir = handler_meta["path"]
 
@@ -877,9 +770,13 @@ class IntegrationController:
                 handler_module = importlib.import_module(f"{base_import}{handler_folder_name}")
                 self.handler_modules[handler_name] = handler_module
                 handler_meta = self._get_handler_meta(handler_name)
+                logger.debug(
+                    f"Handler '{handler_name}' imported successfully in {(time.perf_counter() - time_before_import):.3f} seconds"
+                )
             except Exception as e:
                 handler_meta["import"]["success"] = False
                 handler_meta["import"]["error_message"] = str(e)
+                logger.debug(f"Failed to import handler '{handler_name}': {e}")
 
             self.handlers_import_status[handler_meta["name"]] = handler_meta
             return handler_meta

mindsdb/interfaces/database/projects.py

@@ -38,20 +38,16 @@ class Project:
         return p
 
     def create(self, name: str):
-        name = name.lower()
-
         company_id = ctx.company_id if ctx.company_id is not None else 0
 
         existing_record = db.Integration.query.filter(
-            sa.func.lower(db.Integration.name) == name, db.Integration.company_id == ctx.company_id
+            db.Integration.name == name, db.Integration.company_id == ctx.company_id
         ).first()
         if existing_record is not None:
             raise EntityExistsError("Database exists with this name ", name)
 
         existing_record = db.Project.query.filter(
-            (sa.func.lower(db.Project.name) == name)
-            & (db.Project.company_id == company_id)
-            & (db.Project.deleted_at == sa.null())
+            (db.Project.name == name) & (db.Project.company_id == company_id) & (db.Project.deleted_at == sa.null())
         ).first()
         if existing_record is not None:
             raise EntityExistsError("Project already exists", name)

mindsdb/interfaces/database/views.py

@@ -8,7 +8,6 @@ from mindsdb.interfaces.model.functions import get_project_record, get_project_r
 
 class ViewController:
     def add(self, name, query, project_name):
-        name = name.lower()
         from mindsdb.interfaces.database.database import DatabaseController
 
         database_controller = DatabaseController()
@@ -20,9 +19,7 @@ class ViewController:
         project_id = project_databases_dict[project_name]["id"]
         view_record = (
             db.session.query(db.View.id)
-            .filter(
-                func.lower(db.View.name) == name, db.View.company_id == ctx.company_id, db.View.project_id == project_id
-            )
+            .filter(db.View.name == name, db.View.company_id == ctx.company_id, db.View.project_id == project_id)
             .first()
         )
         if view_record is not None:

mindsdb/interfaces/file/file_controller.py

@@ -107,8 +107,8 @@ class FileController:
             self.fs_store.put(store_file_path, base_dir=self.dir)
             db.session.commit()
 
-        except Exception as e:
-            logger.error(e)
+        except Exception:
+            logger.exception("An error occurred while saving the file:")
             if file_dir is not None:
                 shutil.rmtree(file_dir)
             raise
@@ -151,7 +151,7 @@ class FileController:
     def delete_file(self, name):
         file_record = db.session.query(db.File).filter_by(company_id=ctx.company_id, name=name).first()
         if file_record is None:
-            return None
+            raise FileNotFoundError(f"File '{name}' does not exists")
         file_id = file_record.id
         db.session.delete(file_record)
         db.session.commit()
@@ -161,7 +161,7 @@ class FileController:
     def get_file_path(self, name):
         file_record = db.session.query(db.File).filter_by(company_id=ctx.company_id, name=name).first()
         if file_record is None:
-            raise Exception(f"File '{name}' does not exists")
+            raise FileNotFoundError(f"File '{name}' does not exists")
         file_dir = f"file_{ctx.company_id}_{file_record.id}"
         self.fs_store.get(file_dir, base_dir=self.dir)
         return str(Path(self.dir).joinpath(file_dir).joinpath(Path(file_record.source_file_path).name))
@@ -176,7 +176,7 @@ class FileController:
         """
         file_record = db.session.query(db.File).filter_by(company_id=ctx.company_id, name=name).first()
         if file_record is None:
-            raise Exception(f"File '{name}' does not exists")
+            raise FileNotFoundError(f"File '{name}' does not exists")
 
         file_dir = f"file_{ctx.company_id}_{file_record.id}"
         self.fs_store.get(file_dir, base_dir=self.dir)
@@ -217,7 +217,7 @@ class FileController:
 
         file_record = db.session.query(db.File).filter_by(company_id=ctx.company_id, name=name).first()
         if file_record is None:
-            raise Exception(f"File '{name}' does not exists")
+            raise FileNotFoundError(f"File '{name}' does not exists")
 
         file_dir = f"file_{ctx.company_id}_{file_record.id}"
         self.fs_store.get(file_dir, base_dir=self.dir)

mindsdb/interfaces/functions/controller.py

@@ -140,7 +140,7 @@ class FunctionController(BYOMFunctionsController):
 
             llm = create_chat_model(chat_model_params)
         except Exception as e:
-            raise RuntimeError(f"Unable to use LLM function, check ENV variables: {e}")
+            raise RuntimeError(f"Unable to use LLM function, check ENV variables: {e}") from e
 
         def callback(question):
             resp = llm([HumanMessage(question)])

mindsdb/interfaces/functions/to_markdown.py

@@ -69,8 +69,8 @@ class ToMarkdown:
                 ext = os.path.splitext(parsed_url.path)[1]
                 if ext:
                     return ext
-            except requests.RequestException:
-                raise RuntimeError(f"Unable to retrieve file extension from URL: {file_path_or_url}")
+            except requests.RequestException as e:
+                raise RuntimeError(f"Unable to retrieve file extension from URL: {file_path_or_url}") from e
         else:
             return os.path.splitext(file_path_or_url)[1]
 

mindsdb/interfaces/jobs/jobs_controller.py

@@ -128,8 +128,6 @@ class JobsController:
             at the moment supports: 'every <number> <dimension>' or 'every <dimension>'
         :return: name of created job
         """
-        if not name.islower():
-            raise ValueError(f"The name must be in lower case: {name}")
 
         project_controller = ProjectController()
         project = project_controller.get(name=project_name)
@@ -152,7 +150,7 @@ class JobsController:
 
                 parse_sql(sql)
             except ParsingException as e:
-                raise ParsingException(f"Unable to parse: {sql}: {e}")
+                raise ParsingException(f"Unable to parse: {sql}: {e}") from e
 
         if if_query is not None:
             for sql in split_sql(if_query):
@@ -162,7 +160,7 @@ class JobsController:
 
                     parse_sql(sql)
                 except ParsingException as e:
-                    raise ParsingException(f"Unable to parse: {sql}: {e}")
+                    raise ParsingException(f"Unable to parse: {sql}: {e}") from e
 
         # plan next run
         next_run_at = start_at
@@ -174,8 +172,6 @@ class JobsController:
             # no schedule for job end_at is meaningless
             end_at = None
 
-        name = name.lower()
-
         # create job record
         record = db.Jobs(
             company_id=ctx.company_id,
@@ -494,7 +490,7 @@ class JobsExecutor:
 
                     data = ret.data
                 except Exception as e:
-                    logger.error(e)
+                    logger.exception("Error to execute job`s condition query")
                     error = str(e)
                     break
 
@@ -518,7 +514,7 @@ class JobsExecutor:
                         error = ret.error_message
                         break
                 except Exception as e:
-                    logger.error(e)
+                    logger.exception("Error to execute job`s query")
                     error = str(e)
                     break
 
@@ -526,7 +522,7 @@ class JobsExecutor:
             self.update_task_schedule(record)
         except Exception as e:
             db.session.rollback()
-            logger.error(f"Error to update schedule: {e}")
+            logger.exception("Error to update schedule:")
             error += f"Error to update schedule: {e}"
 
             # stop scheduling

mindsdb/interfaces/jobs/scheduler.py

@@ -14,7 +14,6 @@ logger = log.getLogger(__name__)
 
 
 def execute_async(q_in, q_out):
-
     while True:
         task = q_in.get()
 
@@ -44,7 +43,7 @@ class Scheduler:
         self.q_in = queue.Queue()
         self.q_out = queue.Queue()
         self.work_thread = threading.Thread(
-            target=execute_async, args=(self.q_in, self.q_out), name='Scheduler.execute_async'
+            target=execute_async, args=(self.q_in, self.q_out), name="Scheduler.execute_async"
         )
         self.work_thread.start()
 
@@ -58,14 +57,13 @@ class Scheduler:
         check_interval = self.config.get("jobs", {}).get("check_interval", 30)
 
         while True:
-
             logger.debug("Scheduler check timetable")
             try:
                 self.check_timetable()
             except (SystemExit, KeyboardInterrupt):
                 raise
-            except Exception as e:
-                logger.error(e)
+            except Exception:
+                logger.exception("Error in 'scheduler_monitor'")
 
             # different instances should start in not the same time
 
@@ -83,7 +81,6 @@ class Scheduler:
         db.session.remove()
 
     def execute_task(self, record_id, exec_method):
-
         executor = JobsExecutor()
         if exec_method == "local":
             history_id = executor.lock_record(record_id)
@@ -117,7 +114,6 @@ class Scheduler:
             raise NotImplementedError()
 
     def start(self):
-
         config = Config()
         db.init()
         self.config = config
@@ -127,13 +123,11 @@ class Scheduler:
         try:
             self.scheduler_monitor()
         except (KeyboardInterrupt, SystemExit):
-
             self.stop_thread()
             pass
 
 
 def start(verbose=False):
-    logger.info("Jobs API is starting..")
     scheduler = Scheduler()
 
     scheduler.start()