MindsDB 25.5.4.2__py3-none-any.whl → 25.6.3.0__py3-none-any.whl

mindsdb/utilities/config.py

@@ -217,6 +217,9 @@ class Config:
                 "project_name": "mindsdb",
                 "enabled": False,
             },
+            "data_catalog": {
+                "enabled": False,
+            },
         }
         # endregion
 
@@ -360,6 +363,8 @@ class Config:
             self._env_config["default_reranking_model"] = {
                 "api_key": os.environ["MINDSDB_DEFAULT_RERANKING_MODEL_API_KEY"]
             }
+        if os.environ.get("MINDSDB_DATA_CATALOG_ENABLED", "").lower() in ("1", "true"):
+            self._env_config["data_catalog"] = {"enabled": True}
 
         # region vars: a2a configuration
         a2a_config = {}
@@ -395,11 +400,17 @@ class Config:
             bool: True if config was loaded or updated
         """
 
-        if self.auto_config_mtime != self.auto_config_path.stat().st_mtime:
+        if (
+            self.auto_config_path.is_file()
+            and self.auto_config_path.read_text() != ""
+            and self.auto_config_mtime != self.auto_config_path.stat().st_mtime
+        ):
             try:
                 self._auto_config = json.loads(self.auto_config_path.read_text())
             except json.JSONDecodeError as e:
-                raise ValueError(f"The 'auto' configuration file ({self.auto_config_path}) contains invalid JSON: {e}")
+                raise ValueError(
+                    f"The 'auto' configuration file ({self.auto_config_path}) contains invalid JSON: {e}\nFile content: {self.auto_config_path.read_text()}"
+                )
             self.auto_config_mtime = self.auto_config_path.stat().st_mtime
             return True
         return False

mindsdb/utilities/log.py

@@ -12,11 +12,11 @@ class JsonFormatter(logging.Formatter):
     def format(self, record):
         record_message = super().format(record)
         log_record = {
-            'process_name': record.processName,
-            'name': record.name,
-            'message': record_message,
-            'level': record.levelname,
-            'time': record.created
+            "process_name": record.processName,
+            "name": record.name,
+            "message": record_message,
+            "level": record.levelname,
+            "time": record.created,
         }
         return json.dumps(log_record)
 
@@ -44,12 +44,12 @@ class ColorFormatter(logging.Formatter):
 
 
 def get_console_handler_config_level() -> int:
-    console_handler_config = app_config['logging']['handlers']['console']
+    console_handler_config = app_config["logging"]["handlers"]["console"]
     return getattr(logging, console_handler_config["level"])
 
 
 def get_file_handler_config_level() -> int:
-    file_handler_config = app_config['logging']['handlers']['file']
+    file_handler_config = app_config["logging"]["handlers"]["file"]
     return getattr(logging, file_handler_config["level"])
 
 
@@ -60,27 +60,34 @@ def get_mindsdb_log_level() -> int:
     return min(console_handler_config_level, file_handler_config_level)
 
 
-def configure_logging():
+def configure_logging(process_name: str = None):
     handlers_config = {}
-    console_handler_config = app_config['logging']['handlers']['console']
+    console_handler_config = app_config["logging"]["handlers"]["console"]
     console_handler_config_level = getattr(logging, console_handler_config["level"])
-    if console_handler_config['enabled'] is True:
-        handlers_config['console'] = {
+    if console_handler_config["enabled"] is True:
+        handlers_config["console"] = {
             "class": "logging.StreamHandler",
-            "formatter": console_handler_config.get('formatter', 'default'),
-            "level": console_handler_config_level
+            "formatter": console_handler_config.get("formatter", "default"),
+            "level": console_handler_config_level,
         }
 
-    file_handler_config = app_config['logging']['handlers']['file']
+    file_handler_config = app_config["logging"]["handlers"]["file"]
     file_handler_config_level = getattr(logging, file_handler_config["level"])
-    if file_handler_config['enabled'] is True:
-        handlers_config['file'] = {
+    if file_handler_config["enabled"] is True:
+        file_name = file_handler_config["filename"]
+        if process_name is not None:
+            if "." in file_name:
+                parts = file_name.rpartition(".")
+                file_name = f"{parts[0]}_{process_name}.{parts[2]}"
+            else:
+                file_name = f"{file_name}_{process_name}"
+        handlers_config["file"] = {
             "class": "logging.handlers.RotatingFileHandler",
             "formatter": "file",
             "level": file_handler_config_level,
-            "filename": app_config.paths["log"] / file_handler_config["filename"],
+            "filename": app_config.paths["log"] / file_name,
             "maxBytes": file_handler_config["maxBytes"],  # 0.5 Mb
-            "backupCount": file_handler_config["backupCount"]
+            "backupCount": file_handler_config["backupCount"],
         }
 
     mindsdb_log_level = get_mindsdb_log_level()
@@ -90,9 +97,7 @@ def configure_logging():
         formatters={
             "default": {"()": ColorFormatter},
             "json": {"()": JsonFormatter},
-            "file": {
-                "format": "%(asctime)s %(processName)15s %(levelname)-8s %(name)s: %(message)s"
-            }
+            "file": {"format": "%(asctime)s %(processName)15s %(levelname)-8s %(name)s: %(message)s"},
         },
         handlers=handlers_config,
         loggers={
@@ -115,6 +120,14 @@ def configure_logging():
     dictConfig(logging_config)
 
 
+def initialize_logging(process_name: str = None) -> None:
+    """Initialyze logging"""
+    global logging_initialized
+    if not logging_initialized:
+        configure_logging(process_name)
+        logging_initialized = True
+
+
 # I would prefer to leave code to use logging.getLogger(), but there are a lot of complicated situations
 # in MindsDB with processes being spawned that require logging to be configured again in a lot of cases.
 # Using a custom logger-getter like this lets us do that logic here, once.
@@ -122,9 +135,5 @@ def getLogger(name=None):
     """
     Get a new logger, configuring logging first if it hasn't been done yet.
     """
-    global logging_initialized
-    if not logging_initialized:
-        configure_logging()
-        logging_initialized = True
-
+    initialize_logging()
     return logging.getLogger(name)

mindsdb/utilities/ml_task_queue/task.py

@@ -8,14 +8,14 @@ from mindsdb.utilities.ml_task_queue.const import ML_TASK_STATUS
 
 
 class Task:
-    """ Abstraction for ML task. Should have interface similat to concurrent.futures.Future
+    """Abstraction for ML task. Should have interface similat to concurrent.futures.Future
 
-        Attributes:
-            db (Redis): database object
-            redis_key (RedisKey): redis keys associated with task
-            dataframe (DataFrame): task result
-            exception (Exception): task exeuton  runtime exception
-            _timeout (int): max time without status updating
+    Attributes:
+        db (Redis): database object
+        redis_key (RedisKey): redis keys associated with task
+        dataframe (DataFrame): task result
+        exception (Exception): task exeuton  runtime exception
+        _timeout (int): max time without status updating
     """
 
     def __init__(self, connection: redis.Redis, redis_key: RedisKey) -> None:
@@ -23,18 +23,17 @@ class Task:
         self.redis_key = redis_key
         self.dataframe = None
         self.exception = None
-        self._timeout = 30
+        self._timeout = 60
 
     def subscribe(self) -> ML_TASK_STATUS:
-        """ return tasks status untill it is not done or failed
-        """
+        """return tasks status untill it is not done or failed"""
         pubsub = self.db.pubsub()
         cache = self.db.cache()
         pubsub.subscribe(self.redis_key.status)
-        while (msg := pubsub.get_message(timeout=self._timeout)):
-            if msg['type'] not in pubsub.PUBLISH_MESSAGE_TYPES:
+        while msg := pubsub.get_message(timeout=self._timeout):
+            if msg["type"] not in pubsub.PUBLISH_MESSAGE_TYPES:
                 continue
-            ml_task_status = ML_TASK_STATUS(msg['data'])
+            ml_task_status = ML_TASK_STATUS(msg["data"])
             if ml_task_status == ML_TASK_STATUS.COMPLETE:
                 dataframe_bytes = cache.get(self.redis_key.dataframe)
                 if dataframe_bytes is not None:
@@ -51,8 +50,7 @@ class Task:
             yield ml_task_status
 
     def wait(self, status: ML_TASK_STATUS = ML_TASK_STATUS.COMPLETE) -> None:
-        """ block threasd untill task is not done or failed
-        """
+        """block threasd untill task is not done or failed"""
         for status in self.subscribe():
             if status in (ML_TASK_STATUS.WAITING, ML_TASK_STATUS.PROCESSING):
                 continue
@@ -60,23 +58,22 @@ class Task:
                 if self.exception is not None:
                     raise self.exception
                 else:
-                    raise Exception('Unknown error during ML task execution')
+                    raise Exception("Unknown error during ML task execution")
             if status == ML_TASK_STATUS.TIMEOUT:
                 raise Exception(f"Can't get answer in {self._timeout} seconds")
             if status == ML_TASK_STATUS.COMPLETE:
                 return
-            raise KeyError('Unknown task status')
+            raise KeyError("Unknown task status")
 
     def result(self) -> DataFrame:
-        """ wait task is done and return result
+        """wait task is done and return result
 
-            Returns:
-                DataFrame: task result
+        Returns:
+            DataFrame: task result
         """
         self.wait()
         return self.dataframe
 
     def add_done_callback(self, fn: Callable) -> None:
-        """ need for compatability with concurrent.futures.Future interface
-        """
+        """need for compatability with concurrent.futures.Future interface"""
         pass