MindsDB 25.9.3rc1__py3-none-any.whl → 25.10.0__py3-none-any.whl

mindsdb/interfaces/data_catalog/data_catalog_loader.py

@@ -19,9 +19,7 @@ class DataCatalogLoader(BaseDataCatalog):
         if not self.is_data_catalog_supported():
             return
 
-        loaded_table_names = self._get_loaded_table_names()
-
-        tables = self._load_table_metadata(loaded_table_names)
+        tables = self._load_table_metadata()
 
         if tables:
             columns = self._load_column_metadata(tables)
@@ -34,7 +32,7 @@ class DataCatalogLoader(BaseDataCatalog):
 
         self.logger.info(f"Metadata loading completed for {self.database_name}.")
 
-    def _get_loaded_table_names(self) -> List[str]:
+    def _get_loaded_table_names(self, in_table_names) -> List[str]:
         """
         Retrieve the names of tables that are already present in the data catalog for the current integration.
         If table_names are provided, only those tables will be checked.
@@ -43,8 +41,8 @@ class DataCatalogLoader(BaseDataCatalog):
             List[str]: Names of tables already loaded in the data catalog.
         """
         query = db.session.query(db.MetaTables).filter_by(integration_id=self.integration_id)
-        if self.table_names:
-            query = query.filter(db.MetaTables.name.in_(self.table_names))
+        if in_table_names:
+            query = query.filter(db.MetaTables.name.in_(in_table_names))
 
         tables = query.all()
         table_names = [table.name for table in tables]
@@ -54,7 +52,7 @@ class DataCatalogLoader(BaseDataCatalog):
 
         return table_names
 
-    def _load_table_metadata(self, loaded_table_names: List[str] = None) -> List[Union[db.MetaTables, None]]:
+    def _load_table_metadata(self) -> List[Union[db.MetaTables, None]]:
         """
         Load the table metadata from the handler.
         """
@@ -75,6 +73,7 @@ class DataCatalogLoader(BaseDataCatalog):
         df.columns = df.columns.str.lower()
 
         # Filter out tables that are already loaded in the data catalog
+        loaded_table_names = self._get_loaded_table_names(list(df["table_name"]))
         if loaded_table_names:
             df = df[~df["table_name"].isin(loaded_table_names)]
 

mindsdb/interfaces/data_catalog/data_catalog_reader.py

@@ -1,3 +1,5 @@
+import sqlalchemy as sa
+
 from mindsdb.interfaces.data_catalog.base_data_catalog import BaseDataCatalog
 from mindsdb.interfaces.storage import db
 
@@ -13,8 +15,11 @@ class DataCatalogReader(BaseDataCatalog):
         """
         tables = self._read_metadata()
         if not tables:
-            self.logger.warning(f"No metadata found for database '{self.database_name}'")
-            return f"No metadata found for database '{self.database_name}'"
+            # try case independent
+            tables = self._read_metadata(strict_case=False)
+            if not tables:
+                self.logger.warning(f"No metadata found for database '{self.database_name}'")
+                return f"No metadata found for database '{self.database_name}'"
 
         metadata_str = "Data Catalog: \n"
         if hasattr(self.data_handler, "meta_get_handler_info"):
@@ -42,7 +47,7 @@ class DataCatalogReader(BaseDataCatalog):
         """
         return self.data_handler.meta_get_handler_info()
 
-    def _read_metadata(self) -> list:
+    def _read_metadata(self, strict_case=True) -> list:
         """
         Read the metadata from the data catalog and return it in a structured format.
         """
@@ -52,6 +57,12 @@ class DataCatalogReader(BaseDataCatalog):
         query = db.session.query(db.MetaTables).filter_by(integration_id=self.integration_id)
         if self.table_names:
             cleaned_table_names = [name.strip("`").split(".")[-1] for name in self.table_names]
-            query = query.filter(db.MetaTables.name.in_(cleaned_table_names))
+
+            if strict_case:
+                query = query.filter(db.MetaTables.name.in_(cleaned_table_names))
+            else:
+                cleaned_table_names = [name.lower() for name in cleaned_table_names]
+                query = query.filter(sa.func.lower(db.MetaTables.name).in_(cleaned_table_names))
         tables = query.all()
+
         return tables

mindsdb/interfaces/database/data_handlers_cache.py

@@ -0,0 +1,190 @@
+import sys
+import time
+import threading
+from dataclasses import dataclass, field
+from collections import defaultdict
+
+from mindsdb.integrations.libs.base import DatabaseHandler
+from mindsdb.utilities.context import context as ctx
+from mindsdb.utilities import log
+
+logger = log.getLogger(__name__)
+
+
+@dataclass(kw_only=True, slots=True)
+class HandlersCacheRecord:
+    """Record for a handler in the cache
+
+    Args:
+        handler (DatabaseHandler): handler instance
+        expired_at (float): time when the handler will be expired
+    """
+
+    handler: DatabaseHandler
+    expired_at: float
+    connect_attempt_done: threading.Event = field(default_factory=threading.Event)
+
+    @property
+    def expired(self) -> bool:
+        """check if the handler is expired
+
+        Returns:
+            bool: True if the handler is expired, False otherwise
+        """
+        return self.expired_at < time.time()
+
+    @property
+    def has_references(self) -> bool:
+        """check if the handler has references
+
+        Returns:
+            bool: True if the handler has references, False otherwise
+        """
+        return sys.getrefcount(self.handler) > 2
+
+    def connect(self) -> None:
+        """connect to the handler"""
+        try:
+            if not self.handler.is_connected:
+                self.handler.connect()
+        except Exception:
+            logger.warning(f"Error connecting to handler: {self.handler.name}", exc_info=True)
+        finally:
+            self.connect_attempt_done.set()
+
+
+class HandlersCache:
+    """Cache for data handlers that keep connections opened during ttl time from handler last use"""
+
+    def __init__(self, ttl: int = 60, clean_timeout: float = 3):
+        """init cache
+
+        Args:
+            ttl (int): time to live (in seconds) for record in cache
+            clean_timeout (float): interval between cleanups of expired handlers
+        """
+        self.ttl: int = ttl
+        self._clean_timeout: int = clean_timeout
+        self.handlers: dict[str, list[HandlersCacheRecord]] = defaultdict(list)
+        self._lock = threading.RLock()
+        self._stop_event = threading.Event()
+        self.cleaner_thread = None
+
+    def __del__(self):
+        self._stop_clean()
+
+    def _start_clean(self) -> None:
+        """start worker that close connections after ttl expired"""
+        if isinstance(self.cleaner_thread, threading.Thread) and self.cleaner_thread.is_alive():
+            return
+        with self._lock:
+            self._stop_event.clear()
+            self.cleaner_thread = threading.Thread(target=self._clean, name="HandlersCache.clean")
+            self.cleaner_thread.daemon = True
+            self.cleaner_thread.start()
+
+    def _stop_clean(self) -> None:
+        """stop clean worker"""
+        self._stop_event.set()
+
+    def set(self, handler: DatabaseHandler):
+        """add (or replace) handler in cache
+
+        NOTE: If the handler is not thread-safe, then use a lock when making connection. Otherwise, make connection in
+        the same thread without using a lock to speed up parallel queries. (They don't need to wait for a connection in
+        another thread.)
+
+        Args:
+            handler (DatabaseHandler)
+        """
+        thread_safe = getattr(handler, "thread_safe", True)
+        with self._lock:
+            try:
+                # If the handler is defined to be thread safe, set 0 as the last element of the key, otherwise set the thrad ID.
+                key = (
+                    handler.name,
+                    ctx.company_id,
+                    0 if thread_safe else threading.get_native_id(),
+                )
+                record = HandlersCacheRecord(handler=handler, expired_at=time.time() + self.ttl)
+                self.handlers[key].append(record)
+            except Exception:
+                logger.warning("Error setting data handler cache record:", exc_info=True)
+                return
+        self._start_clean()
+        record.connect()
+
+    def _get_cache_records(self, name: str) -> tuple[list[HandlersCacheRecord] | None, str]:
+        """get cache records by name
+
+        Args:
+            name (str): handler name
+
+        Returns:
+            tuple[list[HandlersCacheRecord] | None, str]: cache records and key of the handler in cache
+        """
+        # If the handler is not thread safe, the thread ID will be assigned to the last element of the key.
+        key = (name, ctx.company_id, 0)
+        if key not in self.handlers:
+            key = (name, ctx.company_id, threading.get_native_id())
+        return self.handlers.get(key, []), key
+
+    def get(self, name: str) -> DatabaseHandler | None:
+        """get handler from cache by name
+
+        Args:
+            name (str): handler name
+
+        Returns:
+            DatabaseHandler
+        """
+        with self._lock:
+            records, _ = self._get_cache_records(name)
+            for record in records:
+                if record.expired is False and record.has_references is False:
+                    record.expired_at = time.time() + self.ttl
+                    if record.connect_attempt_done.wait(timeout=10) is False:
+                        logger.warning(f"Handler's connection attempt has not finished in 10s: {record.handler.name}")
+                    return record.handler
+            return None
+
+    def delete(self, name: str) -> None:
+        """delete handler from cache
+
+        Args:
+            name (str): handler name
+        """
+        with self._lock:
+            records, key = self._get_cache_records(name)
+            if len(records) > 0:
+                del self.handlers[key]
+            for record in records:
+                try:
+                    record.handler.disconnect()
+                except Exception:
+                    logger.debug("Error disconnecting data handler:", exc_info=True)
+
+            if len(self.handlers) == 0:
+                self._stop_clean()
+
+    def _clean(self) -> None:
+        """worker that delete from cache handlers that was not in use for ttl"""
+        while self._stop_event.wait(timeout=self._clean_timeout) is False:
+            with self._lock:
+                for key in list(self.handlers.keys()):
+                    active_handlers_list = []
+                    for record in self.handlers[key]:
+                        if record.expired and record.has_references is False:
+                            try:
+                                record.handler.disconnect()
+                            except Exception:
+                                logger.debug("Error disconnecting data handler:", exc_info=True)
+                        else:
+                            active_handlers_list.append(record)
+                    if len(active_handlers_list) > 0:
+                        self.handlers[key] = active_handlers_list
+                    else:
+                        del self.handlers[key]
+
+                if len(self.handlers) == 0:
+                    self._stop_event.set()

mindsdb/interfaces/database/database.py

@@ -32,10 +32,10 @@ class DatabaseController:
         Returns:
             None
         """
-        databases = self.get_dict()
-        if name.lower() not in databases:
+        databases = self.get_dict(lowercase=False)
+        if name not in databases:
             raise EntityNotExistsError("Database does not exists", name)
-        db_type = databases[name.lower()]["type"]
+        db_type = databases[name]["type"]
         if db_type == "project":
             project = self.get_project(name, strict_case)
             project.delete()

mindsdb/interfaces/database/integrations.py

@@ -1,5 +1,4 @@
 import os
-import sys
 import base64
 import shutil
 import ast
@@ -33,127 +32,11 @@ from mindsdb.integrations.libs.ml_exec_base import BaseMLEngineExec
 from mindsdb.integrations.libs.base import BaseHandler
 import mindsdb.utilities.profiler as profiler
 from mindsdb.interfaces.data_catalog.data_catalog_loader import DataCatalogLoader
+from mindsdb.interfaces.database.data_handlers_cache import HandlersCache
 
 logger = log.getLogger(__name__)
 
 
-class HandlersCache:
-    """Cache for data handlers that keep connections opened during ttl time from handler last use"""
-
-    def __init__(self, ttl: int = 60):
-        """init cache
-
-        Args:
-            ttl (int): time to live (in seconds) for record in cache
-        """
-        self.ttl = ttl
-        self.handlers = {}
-        self._lock = threading.RLock()
-        self._stop_event = threading.Event()
-        self.cleaner_thread = None
-
-    def __del__(self):
-        self._stop_clean()
-
-    def _start_clean(self) -> None:
-        """start worker that close connections after ttl expired"""
-        if isinstance(self.cleaner_thread, threading.Thread) and self.cleaner_thread.is_alive():
-            return
-        self._stop_event.clear()
-        self.cleaner_thread = threading.Thread(target=self._clean, name="HandlersCache.clean")
-        self.cleaner_thread.daemon = True
-        self.cleaner_thread.start()
-
-    def _stop_clean(self) -> None:
-        """stop clean worker"""
-        self._stop_event.set()
-
-    def set(self, handler: DatabaseHandler):
-        """add (or replace) handler in cache
-
-        NOTE: If the handler is not thread-safe, then use a lock when making connection. Otherwise, make connection in
-        the same thread without using a lock to speed up parallel queries. (They don't need to wait for a connection in
-        another thread.)
-
-        Args:
-            handler (DatabaseHandler)
-        """
-        thread_safe = getattr(handler, "thread_safe", False)
-        with self._lock:
-            try:
-                # If the handler is defined to be thread safe, set 0 as the last element of the key, otherwise set the thrad ID.
-                key = (
-                    handler.name,
-                    ctx.company_id,
-                    0 if thread_safe else threading.get_native_id(),
-                )
-                self.handlers[key] = {"handler": handler, "expired_at": time.time() + self.ttl}
-                if thread_safe:
-                    handler.connect()
-            except Exception:
-                pass
-            self._start_clean()
-        try:
-            if not thread_safe:
-                handler.connect()
-        except Exception:
-            pass
-
-    def get(self, name: str) -> Optional[DatabaseHandler]:
-        """get handler from cache by name
-
-        Args:
-            name (str): handler name
-
-        Returns:
-            DatabaseHandler
-        """
-        with self._lock:
-            # If the handler is not thread safe, the thread ID will be assigned to the last element of the key.
-            key = (name, ctx.company_id, threading.get_native_id())
-            if key not in self.handlers:
-                # If the handler is thread safe, a 0 will be assigned to the last element of the key.
-                key = (name, ctx.company_id, 0)
-            if key not in self.handlers or self.handlers[key]["expired_at"] < time.time():
-                return None
-            self.handlers[key]["expired_at"] = time.time() + self.ttl
-            return self.handlers[key]["handler"]
-
-    def delete(self, name: str) -> None:
-        """delete handler from cache
-
-        Args:
-            name (str): handler name
-        """
-        with self._lock:
-            key = (name, ctx.company_id, threading.get_native_id())
-            if key in self.handlers:
-                try:
-                    self.handlers[key].disconnect()
-                except Exception:
-                    pass
-                del self.handlers[key]
-            if len(self.handlers) == 0:
-                self._stop_clean()
-
-    def _clean(self) -> None:
-        """worker that delete from cache handlers that was not in use for ttl"""
-        while self._stop_event.wait(timeout=3) is False:
-            with self._lock:
-                for key in list(self.handlers.keys()):
-                    if (
-                        self.handlers[key]["expired_at"] < time.time()
-                        and sys.getrefcount(self.handlers[key]) == 2  # returned ref count is always 1 higher
-                    ):
-                        try:
-                            self.handlers[key].disconnect()
-                        except Exception:
-                            pass
-                        del self.handlers[key]
-                if len(self.handlers) == 0:
-                    self._stop_event.set()
-
-
 class IntegrationController:
     @staticmethod
     def _is_not_empty_str(s):
@@ -183,9 +66,6 @@ class IntegrationController:
         )
         handler_meta = self.get_handler_meta(engine)
 
-        if not name.islower():
-            raise ValueError(f"The name must be in lower case: {name}")
-
         accept_connection_args = handler_meta.get("connection_args")
         logger.debug("%s: accept_connection_args - %s", self.__class__.__name__, accept_connection_args)
 

mindsdb/interfaces/database/projects.py

@@ -38,20 +38,16 @@ class Project:
         return p
 
     def create(self, name: str):
-        name = name.lower()
-
         company_id = ctx.company_id if ctx.company_id is not None else 0
 
         existing_record = db.Integration.query.filter(
-            sa.func.lower(db.Integration.name) == name, db.Integration.company_id == ctx.company_id
+            db.Integration.name == name, db.Integration.company_id == ctx.company_id
         ).first()
         if existing_record is not None:
             raise EntityExistsError("Database exists with this name ", name)
 
         existing_record = db.Project.query.filter(
-            (sa.func.lower(db.Project.name) == name)
-            & (db.Project.company_id == company_id)
-            & (db.Project.deleted_at == sa.null())
+            (db.Project.name == name) & (db.Project.company_id == company_id) & (db.Project.deleted_at == sa.null())
         ).first()
         if existing_record is not None:
             raise EntityExistsError("Project already exists", name)

mindsdb/interfaces/database/views.py

@@ -8,7 +8,6 @@ from mindsdb.interfaces.model.functions import get_project_record, get_project_r
 
 class ViewController:
     def add(self, name, query, project_name):
-        name = name.lower()
         from mindsdb.interfaces.database.database import DatabaseController
 
         database_controller = DatabaseController()
@@ -20,9 +19,7 @@ class ViewController:
         project_id = project_databases_dict[project_name]["id"]
         view_record = (
             db.session.query(db.View.id)
-            .filter(
-                func.lower(db.View.name) == name, db.View.company_id == ctx.company_id, db.View.project_id == project_id
-            )
+            .filter(db.View.name == name, db.View.company_id == ctx.company_id, db.View.project_id == project_id)
             .first()
         )
         if view_record is not None:

mindsdb/interfaces/jobs/jobs_controller.py

@@ -128,8 +128,6 @@ class JobsController:
             at the moment supports: 'every <number> <dimension>' or 'every <dimension>'
         :return: name of created job
         """
-        if not name.islower():
-            raise ValueError(f"The name must be in lower case: {name}")
 
         project_controller = ProjectController()
         project = project_controller.get(name=project_name)
@@ -174,8 +172,6 @@ class JobsController:
             # no schedule for job end_at is meaningless
             end_at = None
 
-        name = name.lower()
-
         # create job record
         record = db.Jobs(
             company_id=ctx.company_id,

mindsdb/interfaces/jobs/scheduler.py

@@ -128,7 +128,6 @@ class Scheduler:
 
 
 def start(verbose=False):
-    logger.info("Jobs API is starting..")
     scheduler = Scheduler()
 
     scheduler.start()