MindsDB 25.9.2.0a1__py3-none-any.whl → 25.10.0__py3-none-any.whl

mindsdb/interfaces/storage/json.py

@@ -23,7 +23,7 @@ class JsonStorage:
                 resource_group=self.resource_group,
                 resource_id=self.resource_id,
                 company_id=ctx.company_id,
-                content=value
+                content=value,
             )
             db.session.add(record)
         else:
@@ -43,26 +43,27 @@ class JsonStorage:
         return self[key]
 
     def get_record(self, key):
-        record = db.session.query(db.JsonStorage).filter_by(
-            name=key,
-            resource_group=self.resource_group,
-            resource_id=self.resource_id,
-            company_id=ctx.company_id
-        ).first()
+        record = (
+            db.session.query(db.JsonStorage)
+            .filter_by(
+                name=key, resource_group=self.resource_group, resource_id=self.resource_id, company_id=ctx.company_id
+            )
+            .first()
+        )
         return record
 
     def get_all_records(self):
-        records = db.session.query(db.JsonStorage).filter_by(
-            resource_group=self.resource_group,
-            resource_id=self.resource_id,
-            company_id=ctx.company_id
-        ).all()
+        records = (
+            db.session.query(db.JsonStorage)
+            .filter_by(resource_group=self.resource_group, resource_id=self.resource_id, company_id=ctx.company_id)
+            .all()
+        )
         return records
 
     def __repr__(self):
         records = self.get_all_records()
         names = [x.name for x in records]
-        return f'json_storage({names})'
+        return f"json_storage({names})"
 
     def __len__(self):
         records = self.get_all_records()
@@ -76,7 +77,7 @@ class JsonStorage:
                 db.session.commit()
             except Exception:
                 db.session.rollback()
-                logger.error('cant delete record from JSON storage')
+                logger.exception("cant delete record from JSON storage:")
 
     def delete(self, key):
         del self[key]
@@ -89,13 +90,13 @@ class JsonStorage:
             db.session.commit()
         except Exception:
             db.session.rollback()
-            logger.error('cant delete records from JSON storage')
+            logger.exception("cant delete records from JSON storage:")
 
 
 class EncryptedJsonStorage(JsonStorage):
     def __init__(self, resource_group: str, resource_id: int):
         super().__init__(resource_group, resource_id)
-        self.secret_key = config.get('secret_key', 'dummy-key')
+        self.secret_key = config.get("secret_key", "dummy-key")
 
     def __setitem__(self, key: str, value: dict) -> None:
         if isinstance(value, dict) is False:
@@ -110,7 +111,7 @@ class EncryptedJsonStorage(JsonStorage):
                 resource_group=self.resource_group,
                 resource_id=self.resource_id,
                 company_id=ctx.company_id,
-                encrypted_content=encrypted_value
+                encrypted_content=encrypted_value,
             )
             db.session.add(record)
         else:
@@ -125,7 +126,7 @@ class EncryptedJsonStorage(JsonStorage):
                 resource_group=self.resource_group,
                 resource_id=self.resource_id,
                 company_id=ctx.company_id,
-                encrypted_content=encrypted_value
+                encrypted_content=encrypted_value,
             )
             db.session.add(record)
         else:

mindsdb/interfaces/tabs/tabs_controller.py

@@ -11,16 +11,13 @@ from mindsdb.interfaces.storage.fs import FileStorageFactory, RESOURCE_GROUP, Fi
 logger = log.getLogger(__name__)
 
 
-TABS_FILENAME = 'tabs'
+TABS_FILENAME = "tabs"
 
 
 def get_storage():
     # deprecated
 
-    storageFactory = FileStorageFactory(
-        resource_group=RESOURCE_GROUP.TAB,
-        sync=True
-    )
+    storageFactory = FileStorageFactory(resource_group=RESOURCE_GROUP.TAB, sync=True)
 
     # resource_id is useless for 'tabs'
     # use constant
@@ -35,10 +32,7 @@ class TabsController:
     """
 
     def __init__(self) -> None:
-        self.storage_factory = FileStorageFactory(
-            resource_group=RESOURCE_GROUP.TAB,
-            sync=True
-        )
+        self.storage_factory = FileStorageFactory(resource_group=RESOURCE_GROUP.TAB, sync=True)
 
     def _get_file_storage(self) -> FileStorage:
         """Get user's tabs file storage
@@ -69,9 +63,9 @@ class TabsController:
         """
         tabs = {}
         for child in self._get_file_storage().folder_path.iterdir():
-            if (child.is_file() and child.name.startswith('tab_')) is False:
+            if (child.is_file() and child.name.startswith("tab_")) is False:
                 continue
-            tab_id = child.name.replace('tab_', '')
+            tab_id = child.name.replace("tab_", "")
             if tab_id.isnumeric() is False:
                 continue
             tabs[int(tab_id)] = child
@@ -85,12 +79,11 @@ class TabsController:
         """
         all_tabs = self.get_all()
         for tab in all_tabs:
-            del tab['content']
+            del tab["content"]
         return all_tabs
 
     def _migrate_legacy(self) -> None:
-        """Convert old single-file tabs storage to multiple files
-        """
+        """Convert old single-file tabs storage to multiple files"""
         file_storage = self._get_file_storage()
         try:
             file_data = file_storage.file_get(TABS_FILENAME)
@@ -106,12 +99,12 @@ class TabsController:
             file_storage.delete()
             return
 
-        if isinstance(data, dict) is False or isinstance(data.get('tabs'), str) is False:
+        if isinstance(data, dict) is False or isinstance(data.get("tabs"), str) is False:
             file_storage.delete()
             return
 
         try:
-            tabs_list = json.loads(data['tabs'])
+            tabs_list = json.loads(data["tabs"])
         except Exception:
             file_storage.delete()
             return
@@ -123,12 +116,10 @@ class TabsController:
         for tab in tabs_list:
             tab_id = self._get_next_tab_id()
 
-            b_types = json.dumps({
-                'index': tab.get('index', 0),
-                'name': tab.get('name', 'undefined'),
-                'content': tab.get('value', '')
-            }).encode("utf-8")
-            file_storage.file_set(f'tab_{tab_id}', b_types)
+            b_types = json.dumps(
+                {"index": tab.get("index", 0), "name": tab.get("name", "undefined"), "content": tab.get("value", "")}
+            ).encode("utf-8")
+            file_storage.file_set(f"tab_{tab_id}", b_types)
 
         file_storage.delete(TABS_FILENAME)
 
@@ -149,12 +140,9 @@ class TabsController:
             except Exception as e:
                 logger.error(f"Can't read data of tab {ctx.company_id}/{tab_id}: {e}")
                 continue
-            tabs_list.append({
-                'id': tab_id,
-                **data
-            })
+            tabs_list.append({"id": tab_id, **data})
 
-        tabs_list.sort(key=lambda x: x['index'])
+        tabs_list.sort(key=lambda x: x["index"])
         return tabs_list
 
     def get(self, tab_id: int) -> Dict:
@@ -167,25 +155,22 @@ class TabsController:
             dict: tabs data
         """
         if isinstance(tab_id, int) is False:
-            raise ValueError('Tab id must be integer')
+            raise ValueError("Tab id must be integer")
 
         try:
-            raw_tab_data = self._get_file_storage().file_get(f'tab_{tab_id}')
-        except FileNotFoundError:
-            raise EntityNotExistsError(f'tab {tab_id}')
+            raw_tab_data = self._get_file_storage().file_get(f"tab_{tab_id}")
+        except FileNotFoundError as e:
+            raise EntityNotExistsError(f"tab {tab_id}") from e
 
         try:
             data = json.loads(raw_tab_data)
         except Exception as e:
             logger.error(f"Can't read data of tab {ctx.company_id}/{tab_id}: {e}")
-            raise Exception(f"Can't read data of tab: {e}")
+            raise Exception(f"Can't read data of tab: {e}") from e
 
-        return {
-            'id': tab_id,
-            **data
-        }
+        return {"id": tab_id, **data}
 
-    def add(self, index: int = None, name: str = 'undefined', content: str = '') -> Dict:
+    def add(self, index: int = None, name: str = "undefined", content: str = "") -> Dict:
         """Add new tab
 
         Args:
@@ -205,27 +190,23 @@ class TabsController:
             if len(all_tabs) == 0:
                 index = 0
             else:
-                index = max([x.get('index', 0) for x in all_tabs]) + 1
+                index = max([x.get("index", 0) for x in all_tabs]) + 1
 
-        data_bytes = json.dumps({
-            'index': index,
-            'name': name,
-            'content': content
-        }).encode("utf-8")
-        file_storage.file_set(f'tab_{tab_id}', data_bytes)
+        data_bytes = json.dumps({"index": index, "name": name, "content": content}).encode("utf-8")
+        file_storage.file_set(f"tab_{tab_id}", data_bytes)
 
         if reorder_required:
             all_tabs = self.get_all()
-            all_tabs.sort(key=lambda x: (x['index'], 0 if x['id'] == tab_id else 1))
+            all_tabs.sort(key=lambda x: (x["index"], 0 if x["id"] == tab_id else 1))
             file_storage.sync = False
             for tab_index, tab in enumerate(all_tabs):
-                tab['index'] = tab_index
-                data_bytes = json.dumps(tab).encode('utf-8')
-                file_storage.file_set(f'tab_{tab["id"]}', data_bytes)
+                tab["index"] = tab_index
+                data_bytes = json.dumps(tab).encode("utf-8")
+                file_storage.file_set(f"tab_{tab['id']}", data_bytes)
             file_storage.sync = True
             file_storage.push()
 
-        return {'id': tab_id, 'index': index, 'name': name}
+        return {"id": tab_id, "index": index, "name": name}
 
     def modify(self, tab_id: int, index: int = None, name: str = None, content: str = None) -> Dict:
         """Modify the tab
@@ -243,49 +224,45 @@ class TabsController:
         current_data = self.get(tab_id)
 
         # region modify index
-        if index is not None and current_data['index'] != index:
-            current_data['index'] = index
-            all_tabs = [x for x in self.get_all() if x['id'] != tab_id]
-            all_tabs.sort(key=lambda x: x['index'])
+        if index is not None and current_data["index"] != index:
+            current_data["index"] = index
+            all_tabs = [x for x in self.get_all() if x["id"] != tab_id]
+            all_tabs.sort(key=lambda x: x["index"])
             file_storage.sync = False
             for tab_index, tab in enumerate(all_tabs):
                 if tab_index < index:
-                    tab['index'] = tab_index
+                    tab["index"] = tab_index
                 else:
-                    tab['index'] = tab_index + 1
-                data_bytes = json.dumps(tab).encode('utf-8')
-                file_storage.file_set(f'tab_{tab["id"]}', data_bytes)
+                    tab["index"] = tab_index + 1
+                data_bytes = json.dumps(tab).encode("utf-8")
+                file_storage.file_set(f"tab_{tab['id']}", data_bytes)
             file_storage.sync = True
             file_storage.push()
         # endregion
 
         # region modify name
-        if name is not None and current_data['name'] != name:
-            current_data['name'] = name
+        if name is not None and current_data["name"] != name:
+            current_data["name"] = name
         # endregion
 
         # region modify content
-        if content is not None and current_data['content'] != content:
-            current_data['content'] = content
+        if content is not None and current_data["content"] != content:
+            current_data["content"] = content
         # endregion
 
-        data_bytes = json.dumps(current_data).encode('utf-8')
-        file_storage.file_set(f'tab_{tab_id}', data_bytes)
+        data_bytes = json.dumps(current_data).encode("utf-8")
+        file_storage.file_set(f"tab_{tab_id}", data_bytes)
 
-        return {
-            'id': current_data['id'],
-            'index': current_data['index'],
-            'name': current_data['name']
-        }
+        return {"id": current_data["id"], "index": current_data["index"], "name": current_data["name"]}
 
     def delete(self, tab_id: int):
         file_storage = self._get_file_storage()
         try:
-            file_storage.file_get(f'tab_{tab_id}')
-        except FileNotFoundError:
-            raise EntityNotExistsError(f'tab {tab_id}')
+            file_storage.file_get(f"tab_{tab_id}")
+        except FileNotFoundError as e:
+            raise EntityNotExistsError(f"tab {tab_id}") from e
 
-        file_storage.delete(f'tab_{tab_id}')
+        file_storage.delete(f"tab_{tab_id}")
 
 
 tabs_controller = TabsController()

mindsdb/interfaces/tasks/task_monitor.py

@@ -16,7 +16,6 @@ logger = log.getLogger(__name__)
 
 
 class TaskMonitor:
-
     MONITOR_INTERVAL_SECONDS = 2
     LOCK_EXPIRED_SECONDS = MONITOR_INTERVAL_SECONDS * 30
 
@@ -39,15 +38,14 @@ class TaskMonitor:
                 self.stop_all_tasks()
                 return
 
-            except Exception as e:
-                logger.error(e)
+            except Exception:
+                logger.exception("Error in TaskMonitor.start")
                 db.session.rollback()
 
             if stop_event is not None and stop_event.is_set():
                 return
 
     def stop_all_tasks(self):
-
         active_tasks = list(self._active_tasks.keys())
         for task_id in active_tasks:
             self.stop_task(task_id)
@@ -65,7 +63,6 @@ class TaskMonitor:
         # Check active tasks
         active_tasks = list(self._active_tasks.items())
         for task_id, task in active_tasks:
-
             if task_id not in allowed_tasks:
                 # old task
                 self.stop_task(task_id)
@@ -96,9 +93,7 @@ class TaskMonitor:
             task.run_by = run_by
             task.alive_time = db_date
 
-        elif db_date - task.alive_time > dt.timedelta(
-            seconds=self.LOCK_EXPIRED_SECONDS
-        ):
+        elif db_date - task.alive_time > dt.timedelta(seconds=self.LOCK_EXPIRED_SECONDS):
             # lock expired
             task.run_by = run_by
             task.alive_time = db_date
@@ -145,7 +140,6 @@ class TaskMonitor:
 
 
 def start(verbose=False):
-
     monitor = TaskMonitor()
     monitor.start()
 

mindsdb/interfaces/tasks/task_thread.py

@@ -12,7 +12,6 @@ logger = log.getLogger(__name__)
 
 
 class TaskThread(threading.Thread):
-
     def __init__(self, task_id):
         threading.Thread.__init__(self)
         self.task_id = task_id
@@ -34,28 +33,27 @@ class TaskThread(threading.Thread):
         self.object_type = task_record.object_type
         self.object_id = task_record.object_id
 
-        logger.info(f'Task starting: {self.object_type}.{self.object_id}')
+        logger.info(f"Task starting: {self.object_type}.{self.object_id}")
         try:
-            if self.object_type == 'trigger':
-
+            if self.object_type == "trigger":
                 trigger = TriggerTask(self.task_id, self.object_id)
                 trigger.run(self._stop_event)
 
-            elif self.object_type == 'chatbot':
+            elif self.object_type == "chatbot":
                 bot = ChatBotTask(self.task_id, self.object_id)
                 bot.run(self._stop_event)
 
-            elif self.object_type == 'query':
+            elif self.object_type == "query":
                 query = QueryTask(self.task_id, self.object_id)
                 query.run(self._stop_event)
 
         except Exception:
-            logger.error(traceback.format_exc())
-            task_record.last_error = str(traceback.format_exc())
+            logger.exception("Error during task processing:")
+            task_record.last_error = traceback.format_exc()
 
         db.session.commit()
 
     def stop(self):
-        logger.info(f'Task stopping: {self.object_type}.{self.object_id}')
+        logger.info(f"Task stopping: {self.object_type}.{self.object_id}")
 
         self._stop_event.set()

mindsdb/interfaces/triggers/trigger_task.py

@@ -44,19 +44,19 @@ class TriggerTask(BaseTask):
 
         # subscribe
         database = session.integration_controller.get_by_id(trigger.database_id)
-        data_handler = session.integration_controller.get_data_handler(database['name'])
+        data_handler = session.integration_controller.get_data_handler(database["name"])
 
         columns = trigger.columns
         if columns is not None:
-            if columns == '':
+            if columns == "":
                 columns = None
             else:
-                columns = columns.split('|')
+                columns = columns.split("|")
 
         data_handler.subscribe(stop_event, self._callback, trigger.table_name, columns=columns)
 
     def _callback(self, row, key=None):
-        logger.debug(f'trigger call: {row}, {key}')
+        logger.debug(f"trigger call: {row}, {key}")
 
         # set up environment
         ctx.load(self._ctx_dump)
@@ -64,21 +64,14 @@ class TriggerTask(BaseTask):
         try:
             if key is not None:
                 row.update(key)
-            table = [
-                row
-            ]
+            table = [row]
 
             # inject data to query
             query = copy.deepcopy(self.query)
 
             def find_table(node, is_table, **kwargs):
-
                 if is_table:
-                    if (
-                            isinstance(node, Identifier)
-                            and len(node.parts) == 1
-                            and node.parts[0] == 'TABLE_DELTA'
-                    ):
+                    if isinstance(node, Identifier) and len(node.parts) == 1 and node.parts[0] == "TABLE_DELTA":
                         # replace with data
                         return Data(table, alias=node.alias)
 
@@ -90,6 +83,7 @@ class TriggerTask(BaseTask):
                 self.set_error(ret.error_message)
 
         except Exception:
+            logger.exception("Error during trigger call processing")
             self.set_error(str(traceback.format_exc()))
 
         db.session.commit()

mindsdb/interfaces/triggers/triggers_controller.py

@@ -11,27 +11,26 @@ from mindsdb.api.executor.controllers.session_controller import SessionControlle
 
 
 class TriggersController:
-    OBJECT_TYPE = 'trigger'
+    OBJECT_TYPE = "trigger"
 
     def add(self, name, project_name, table, query_str, columns=None):
-        name = name.lower()
-
         if project_name is None:
-            project_name = config.get('default_project')
+            project_name = config.get("default_project")
         project_controller = ProjectController()
         project = project_controller.get(name=project_name)
 
         from mindsdb.api.executor.controllers.session_controller import SessionController
+
         session = SessionController()
 
         # check exists
         trigger = self.get_trigger_record(name, project_name)
         if trigger is not None:
-            raise Exception(f'Trigger already exists: {name}')
+            raise Exception(f"Trigger already exists: {name}")
 
         # check table
         if len(table.parts) < 2:
-            raise Exception(f'Database or table not found: {table}')
+            raise Exception(f"Database or table not found: {table}")
 
         table_name = Identifier(parts=table.parts[1:]).to_string()
         db_name = table.parts[0]
@@ -39,39 +38,38 @@ class TriggersController:
         db_integration = session.integration_controller.get(db_name)
         db_handler = session.integration_controller.get_data_handler(db_name)
 
-        if not hasattr(db_handler, 'subscribe'):
-            raise Exception(f'Handler {db_integration["engine"]} does''t support subscription')
+        if not hasattr(db_handler, "subscribe"):
+            raise Exception(f"Handler {db_integration['engine']} doest support subscription")
 
         df = db_handler.get_tables().data_frame
-        column = 'table_name'
+        column = "table_name"
         if column not in df.columns:
             column = df.columns[0]
         tables = list(df[column])
 
         # check only if tables are visible
         if len(tables) > 0 and table_name not in tables:
-            raise Exception(f'Table {table_name} not found in {db_name}')
+            raise Exception(f"Table {table_name} not found in {db_name}")
 
         columns_str = None
         if columns is not None and len(columns) > 0:
             # join to string with delimiter
-            columns_str = '|'.join([col.parts[-1] for col in columns])
+            columns_str = "|".join([col.parts[-1] for col in columns])
 
         # check sql
         try:
             parse_sql(query_str)
         except ParsingException as e:
-            raise ParsingException(f'Unable to parse: {query_str}: {e}')
+            raise ParsingException(f"Unable to parse: {query_str}: {e}") from e
 
         # create job record
         record = db.Triggers(
             name=name,
             project_id=project.id,
-
-            database_id=db_integration['id'],
+            database_id=db_integration["id"],
             table_name=table_name,
             query_str=query_str,
-            columns=columns_str
+            columns=columns_str,
         )
         db.session.add(record)
         db.session.flush()
@@ -79,7 +77,6 @@ class TriggersController:
         task_record = db.Tasks(
             company_id=ctx.company_id,
             user_class=ctx.user_class,
-
             object_type=self.OBJECT_TYPE,
             object_id=record.id,
         )
@@ -110,34 +107,36 @@ class TriggersController:
         project_controller = ProjectController()
         project = project_controller.get(name=project_name)
 
-        query = db.session.query(
-            db.Triggers
-        ).join(
-            db.Tasks, db.Triggers.id == db.Tasks.object_id
-        ).filter(
-            db.Triggers.project_id == project.id,
-            db.Triggers.name == name,
-            db.Tasks.object_type == self.OBJECT_TYPE,
-            db.Tasks.company_id == ctx.company_id,
+        query = (
+            db.session.query(db.Triggers)
+            .join(db.Tasks, db.Triggers.id == db.Tasks.object_id)
+            .filter(
+                db.Triggers.project_id == project.id,
+                db.Triggers.name == name,
+                db.Tasks.object_type == self.OBJECT_TYPE,
+                db.Tasks.company_id == ctx.company_id,
+            )
         )
         return query.first()
 
     def get_list(self, project_name=None):
         session = SessionController()
 
-        query = db.session.query(
-            db.Tasks.object_id,
-            db.Triggers.project_id,
-            db.Triggers.name,
-            db.Triggers.database_id,
-            db.Triggers.table_name,
-            db.Triggers.query_str,
-            db.Tasks.last_error,
-        )\
-            .join(db.Triggers, db.Triggers.id == db.Tasks.object_id)\
+        query = (
+            db.session.query(
+                db.Tasks.object_id,
+                db.Triggers.project_id,
+                db.Triggers.name,
+                db.Triggers.database_id,
+                db.Triggers.table_name,
+                db.Triggers.query_str,
+                db.Tasks.last_error,
+            )
+            .join(db.Triggers, db.Triggers.id == db.Tasks.object_id)
             .filter(
                 db.Tasks.object_type == self.OBJECT_TYPE,
                 db.Tasks.company_id == ctx.company_id,
+            )
         )
 
         project_controller = ProjectController()
@@ -145,24 +144,20 @@ class TriggersController:
             project = project_controller.get(name=project_name)
             query = query.filter(db.Triggers.project_id == project.id)
 
-        database_names = {
-            i['id']: i['name']
-            for i in session.database_controller.get_list()
-        }
+        database_names = {i["id"]: i["name"] for i in session.database_controller.get_list()}
 
-        project_names = {
-            i.id: i.name
-            for i in project_controller.get_list()
-        }
+        project_names = {i.id: i.name for i in project_controller.get_list()}
         data = []
         for record in query:
-            data.append({
-                'id': record.object_id,
-                'project': project_names[record.project_id],
-                'name': record.name.lower(),
-                'database': database_names.get(record.database_id, '?'),
-                'table': record.table_name,
-                'query': record.query_str,
-                'last_error': record.last_error,
-            })
+            data.append(
+                {
+                    "id": record.object_id,
+                    "project": project_names[record.project_id],
+                    "name": record.name,
+                    "database": database_names.get(record.database_id, "?"),
+                    "table": record.table_name,
+                    "query": record.query_str,
+                    "last_error": record.last_error,
+                }
+            )
         return data