MindsDB 25.9.1.2__py3-none-any.whl → 25.9.3rc1__py3-none-any.whl

mindsdb/interfaces/storage/json.py

@@ -23,7 +23,7 @@ class JsonStorage:
                 resource_group=self.resource_group,
                 resource_id=self.resource_id,
                 company_id=ctx.company_id,
-                content=value
+                content=value,
             )
             db.session.add(record)
         else:
@@ -43,26 +43,27 @@ class JsonStorage:
         return self[key]
 
     def get_record(self, key):
-        record = db.session.query(db.JsonStorage).filter_by(
-            name=key,
-            resource_group=self.resource_group,
-            resource_id=self.resource_id,
-            company_id=ctx.company_id
-        ).first()
+        record = (
+            db.session.query(db.JsonStorage)
+            .filter_by(
+                name=key, resource_group=self.resource_group, resource_id=self.resource_id, company_id=ctx.company_id
+            )
+            .first()
+        )
         return record
 
     def get_all_records(self):
-        records = db.session.query(db.JsonStorage).filter_by(
-            resource_group=self.resource_group,
-            resource_id=self.resource_id,
-            company_id=ctx.company_id
-        ).all()
+        records = (
+            db.session.query(db.JsonStorage)
+            .filter_by(resource_group=self.resource_group, resource_id=self.resource_id, company_id=ctx.company_id)
+            .all()
+        )
         return records
 
     def __repr__(self):
         records = self.get_all_records()
         names = [x.name for x in records]
-        return f'json_storage({names})'
+        return f"json_storage({names})"
 
     def __len__(self):
         records = self.get_all_records()
@@ -76,7 +77,7 @@ class JsonStorage:
                 db.session.commit()
             except Exception:
                 db.session.rollback()
-                logger.error('cant delete record from JSON storage')
+                logger.exception("cant delete record from JSON storage:")
 
     def delete(self, key):
         del self[key]
@@ -89,13 +90,13 @@ class JsonStorage:
             db.session.commit()
         except Exception:
             db.session.rollback()
-            logger.error('cant delete records from JSON storage')
+            logger.exception("cant delete records from JSON storage:")
 
 
 class EncryptedJsonStorage(JsonStorage):
     def __init__(self, resource_group: str, resource_id: int):
         super().__init__(resource_group, resource_id)
-        self.secret_key = config.get('secret_key', 'dummy-key')
+        self.secret_key = config.get("secret_key", "dummy-key")
 
     def __setitem__(self, key: str, value: dict) -> None:
         if isinstance(value, dict) is False:
@@ -110,7 +111,7 @@ class EncryptedJsonStorage(JsonStorage):
                 resource_group=self.resource_group,
                 resource_id=self.resource_id,
                 company_id=ctx.company_id,
-                encrypted_content=encrypted_value
+                encrypted_content=encrypted_value,
             )
             db.session.add(record)
         else:
@@ -125,7 +126,7 @@ class EncryptedJsonStorage(JsonStorage):
                 resource_group=self.resource_group,
                 resource_id=self.resource_id,
                 company_id=ctx.company_id,
-                encrypted_content=encrypted_value
+                encrypted_content=encrypted_value,
             )
             db.session.add(record)
         else:

mindsdb/interfaces/storage/model_fs.py

@@ -11,18 +11,16 @@ from .fs import RESOURCE_GROUP, FileStorageFactory, SERVICE_FILES_NAMES
 from .json import get_json_storage, get_encrypted_json_storage
 
 
-JSON_STORAGE_FILE = 'json_storage.json'
+JSON_STORAGE_FILE = "json_storage.json"
 
 
 class ModelStorage:
     """
     This class deals with all model-related storage requirements, from setting status to storing artifacts.
     """
+
     def __init__(self, predictor_id):
-        storageFactory = FileStorageFactory(
-            resource_group=RESOURCE_GROUP.PREDICTOR,
-            sync=True
-        )
+        storageFactory = FileStorageFactory(resource_group=RESOURCE_GROUP.PREDICTOR, sync=True)
         self.fileStorage = storageFactory(predictor_id)
         self.predictor_id = predictor_id
 
@@ -43,15 +41,12 @@ class ModelStorage:
         """
         model_record = db.Predictor.query.get(self.predictor_id)
         if check_exists is True and model_record is None:
-            raise KeyError('Model does not exists')
+            raise KeyError("Model does not exists")
         return model_record
 
     def get_info(self):
         rec = self._get_model_record(self.predictor_id)
-        return dict(status=rec.status,
-                    to_predict=rec.to_predict,
-                    data=rec.data,
-                    learn_args=rec.learn_args)
+        return dict(status=rec.status, to_predict=rec.to_predict, data=rec.data, learn_args=rec.learn_args)
 
     def status_set(self, status, status_info=None):
         rec = self._get_model_record(self.predictor_id)
@@ -95,67 +90,52 @@ class ModelStorage:
 
     def folder_get(self, name):
         # pull folder and return path
-        name = name.lower().replace(' ', '_')
-        name = re.sub(r'([^a-z^A-Z^_\d]+)', '_', name)
+        name = name.lower().replace(" ", "_")
+        name = re.sub(r"([^a-z^A-Z^_\d]+)", "_", name)
 
         self.fileStorage.pull_path(name)
         return str(self.fileStorage.get_path(name))
 
     def folder_sync(self, name):
         # sync abs path
-        name = name.lower().replace(' ', '_')
-        name = re.sub(r'([^a-z^A-Z^_\d]+)', '_', name)
+        name = name.lower().replace(" ", "_")
+        name = re.sub(r"([^a-z^A-Z^_\d]+)", "_", name)
 
         self.fileStorage.push_path(name)
 
-    def file_list(self):
-        ...
+    def file_list(self): ...
 
-    def file_del(self, name):
-        ...
+    def file_del(self, name): ...
 
     # jsons
 
     def json_set(self, name, data):
-        json_storage = get_json_storage(
-            resource_id=self.predictor_id,
-            resource_group=RESOURCE_GROUP.PREDICTOR
-        )
+        json_storage = get_json_storage(resource_id=self.predictor_id, resource_group=RESOURCE_GROUP.PREDICTOR)
         return json_storage.set(name, data)
 
     def encrypted_json_set(self, name: str, data: dict) -> None:
         json_storage = get_encrypted_json_storage(
-            resource_id=self.predictor_id,
-            resource_group=RESOURCE_GROUP.PREDICTOR
+            resource_id=self.predictor_id, resource_group=RESOURCE_GROUP.PREDICTOR
         )
         return json_storage.set(name, data)
 
     def json_get(self, name):
-        json_storage = get_json_storage(
-            resource_id=self.predictor_id,
-            resource_group=RESOURCE_GROUP.PREDICTOR
-        )
+        json_storage = get_json_storage(resource_id=self.predictor_id, resource_group=RESOURCE_GROUP.PREDICTOR)
         return json_storage.get(name)
 
     def encrypted_json_get(self, name: str) -> dict:
         json_storage = get_encrypted_json_storage(
-            resource_id=self.predictor_id,
-            resource_group=RESOURCE_GROUP.PREDICTOR
+            resource_id=self.predictor_id, resource_group=RESOURCE_GROUP.PREDICTOR
         )
         return json_storage.get(name)
 
-    def json_list(self):
-        ...
+    def json_list(self): ...
 
-    def json_del(self, name):
-        ...
+    def json_del(self, name): ...
 
     def delete(self):
         self.fileStorage.delete()
-        json_storage = get_json_storage(
-            resource_id=self.predictor_id,
-            resource_group=RESOURCE_GROUP.PREDICTOR
-        )
+        json_storage = get_json_storage(resource_id=self.predictor_id, resource_group=RESOURCE_GROUP.PREDICTOR)
         json_storage.clean()
 
 
@@ -164,29 +144,26 @@ class HandlerStorage:
     This class deals with all handler-related storage requirements, from storing metadata to synchronizing folders
     across instances.
     """
+
     def __init__(self, integration_id: int, root_dir: str = None, is_temporal=False):
         args = {}
         if root_dir is not None:
-            args['root_dir'] = root_dir
-        storageFactory = FileStorageFactory(
-            resource_group=RESOURCE_GROUP.INTEGRATION,
-            sync=False,
-            **args
-        )
+            args["root_dir"] = root_dir
+        storageFactory = FileStorageFactory(resource_group=RESOURCE_GROUP.INTEGRATION, sync=False, **args)
         self.fileStorage = storageFactory(integration_id)
         self.integration_id = integration_id
         self.is_temporal = is_temporal
         # do not sync with remote storage
 
     def __convert_name(self, name):
-        name = name.lower().replace(' ', '_')
-        return re.sub(r'([^a-z^A-Z^_\d]+)', '_', name)
+        name = name.lower().replace(" ", "_")
+        return re.sub(r"([^a-z^A-Z^_\d]+)", "_", name)
 
     def is_empty(self):
-        """ check if storage directory is empty
+        """check if storage directory is empty
 
-            Returns:
-                bool: true if dir is empty
+        Returns:
+            bool: true if dir is empty
         """
         for path in self.fileStorage.folder_path.iterdir():
             if path.is_file() and path.name in SERVICE_FILES_NAMES:
@@ -221,19 +198,17 @@ class HandlerStorage:
         if not self.is_temporal:
             self.fileStorage.push_path(name)
 
-    def file_list(self):
-        ...
+    def file_list(self): ...
 
-    def file_del(self, name):
-        ...
+    def file_del(self, name): ...
 
     # folder
 
     def folder_get(self, name):
-        ''' Copies folder from remote to local file system and returns its path
+        """Copies folder from remote to local file system and returns its path
 
         :param name: name of the folder
-        '''
+        """
         name = self.__convert_name(name)
 
         self.fileStorage.pull_path(name)
@@ -249,38 +224,28 @@ class HandlerStorage:
     # jsons
 
     def json_set(self, name, content):
-        json_storage = get_json_storage(
-            resource_id=self.integration_id,
-            resource_group=RESOURCE_GROUP.INTEGRATION
-        )
+        json_storage = get_json_storage(resource_id=self.integration_id, resource_group=RESOURCE_GROUP.INTEGRATION)
         return json_storage.set(name, content)
 
     def encrypted_json_set(self, name: str, content: dict) -> None:
         json_storage = get_encrypted_json_storage(
-            resource_id=self.integration_id,
-            resource_group=RESOURCE_GROUP.INTEGRATION
+            resource_id=self.integration_id, resource_group=RESOURCE_GROUP.INTEGRATION
         )
         return json_storage.set(name, content)
 
     def json_get(self, name):
-        json_storage = get_json_storage(
-            resource_id=self.integration_id,
-            resource_group=RESOURCE_GROUP.INTEGRATION
-        )
+        json_storage = get_json_storage(resource_id=self.integration_id, resource_group=RESOURCE_GROUP.INTEGRATION)
         return json_storage.get(name)
 
     def encrypted_json_get(self, name: str) -> dict:
         json_storage = get_encrypted_json_storage(
-            resource_id=self.integration_id,
-            resource_group=RESOURCE_GROUP.INTEGRATION
+            resource_id=self.integration_id, resource_group=RESOURCE_GROUP.INTEGRATION
         )
         return json_storage.get(name)
 
-    def json_list(self):
-        ...
+    def json_list(self): ...
 
-    def json_del(self, name):
-        ...
+    def json_del(self, name): ...
 
     def export_files(self) -> bytes:
         json_storage = self.export_json_storage()
@@ -288,11 +253,11 @@ class HandlerStorage:
         if self.is_empty() and not json_storage:
             return None
 
-        folder_path = self.folder_get('')
+        folder_path = self.folder_get("")
 
         zip_fd = io.BytesIO()
 
-        with zipfile.ZipFile(zip_fd, 'w', zipfile.ZIP_DEFLATED) as zipf:
+        with zipfile.ZipFile(zip_fd, "w", zipfile.ZIP_DEFLATED) as zipf:
             for root, dirs, files in os.walk(folder_path):
                 for file_name in files:
                     if file_name in SERVICE_FILES_NAMES:
@@ -309,14 +274,13 @@ class HandlerStorage:
         return zip_fd.read()
 
     def import_files(self, content: bytes):
-
-        folder_path = self.folder_get('')
+        folder_path = self.folder_get("")
 
         zip_fd = io.BytesIO()
         zip_fd.write(content)
         zip_fd.seek(0)
 
-        with zipfile.ZipFile(zip_fd, 'r') as zip_ref:
+        with zipfile.ZipFile(zip_fd, "r") as zip_ref:
             for name in zip_ref.namelist():
                 # If JSON storage file is in the zip file, import the content to the JSON storage.
                 # Thereafter, remove the file from the folder.
@@ -327,38 +291,36 @@ class HandlerStorage:
                 else:
                     zip_ref.extract(name, folder_path)
 
-        self.folder_sync('')
+        self.folder_sync("")
 
     def export_json_storage(self) -> list[dict]:
-        json_storage = get_json_storage(
-            resource_id=self.integration_id,
-            resource_group=RESOURCE_GROUP.INTEGRATION
-        )
+        json_storage = get_json_storage(resource_id=self.integration_id, resource_group=RESOURCE_GROUP.INTEGRATION)
 
         records = []
         for record in json_storage.get_all_records():
             record_dict = record.to_dict()
-            if record_dict.get('encrypted_content'):
-                record_dict['encrypted_content'] = record_dict['encrypted_content'].decode()
+            if record_dict.get("encrypted_content"):
+                record_dict["encrypted_content"] = record_dict["encrypted_content"].decode()
             records.append(record_dict)
 
         return records
 
     def import_json_storage(self, records: bytes) -> None:
-        json_storage = get_json_storage(
-            resource_id=self.integration_id,
-            resource_group=RESOURCE_GROUP.INTEGRATION
-        )
+        json_storage = get_json_storage(resource_id=self.integration_id, resource_group=RESOURCE_GROUP.INTEGRATION)
 
         encrypted_json_storage = get_encrypted_json_storage(
-            resource_id=self.integration_id,
-            resource_group=RESOURCE_GROUP.INTEGRATION
+            resource_id=self.integration_id, resource_group=RESOURCE_GROUP.INTEGRATION
         )
 
         records = json.loads(records.decode())
 
         for record in records:
-            if record['encrypted_content']:
-                encrypted_json_storage.set_str(record['name'], record['encrypted_content'])
+            if record["encrypted_content"]:
+                encrypted_json_storage.set_str(record["name"], record["encrypted_content"])
             else:
-                json_storage.set(record['name'], record['content'])
+                json_storage.set(record["name"], record["content"])
+
+    def delete(self):
+        self.fileStorage.delete()
+        json_storage = get_json_storage(resource_id=self.integration_id, resource_group=RESOURCE_GROUP.INTEGRATION)
+        json_storage.clean()

mindsdb/interfaces/tabs/tabs_controller.py

@@ -11,16 +11,13 @@ from mindsdb.interfaces.storage.fs import FileStorageFactory, RESOURCE_GROUP, Fi
 logger = log.getLogger(__name__)
 
 
-TABS_FILENAME = 'tabs'
+TABS_FILENAME = "tabs"
 
 
 def get_storage():
     # deprecated
 
-    storageFactory = FileStorageFactory(
-        resource_group=RESOURCE_GROUP.TAB,
-        sync=True
-    )
+    storageFactory = FileStorageFactory(resource_group=RESOURCE_GROUP.TAB, sync=True)
 
     # resource_id is useless for 'tabs'
     # use constant
@@ -35,10 +32,7 @@ class TabsController:
     """
 
     def __init__(self) -> None:
-        self.storage_factory = FileStorageFactory(
-            resource_group=RESOURCE_GROUP.TAB,
-            sync=True
-        )
+        self.storage_factory = FileStorageFactory(resource_group=RESOURCE_GROUP.TAB, sync=True)
 
     def _get_file_storage(self) -> FileStorage:
         """Get user's tabs file storage
@@ -69,9 +63,9 @@ class TabsController:
         """
         tabs = {}
         for child in self._get_file_storage().folder_path.iterdir():
-            if (child.is_file() and child.name.startswith('tab_')) is False:
+            if (child.is_file() and child.name.startswith("tab_")) is False:
                 continue
-            tab_id = child.name.replace('tab_', '')
+            tab_id = child.name.replace("tab_", "")
             if tab_id.isnumeric() is False:
                 continue
             tabs[int(tab_id)] = child
@@ -85,12 +79,11 @@ class TabsController:
         """
         all_tabs = self.get_all()
         for tab in all_tabs:
-            del tab['content']
+            del tab["content"]
         return all_tabs
 
     def _migrate_legacy(self) -> None:
-        """Convert old single-file tabs storage to multiple files
-        """
+        """Convert old single-file tabs storage to multiple files"""
         file_storage = self._get_file_storage()
         try:
             file_data = file_storage.file_get(TABS_FILENAME)
@@ -106,12 +99,12 @@ class TabsController:
             file_storage.delete()
             return
 
-        if isinstance(data, dict) is False or isinstance(data.get('tabs'), str) is False:
+        if isinstance(data, dict) is False or isinstance(data.get("tabs"), str) is False:
             file_storage.delete()
             return
 
         try:
-            tabs_list = json.loads(data['tabs'])
+            tabs_list = json.loads(data["tabs"])
         except Exception:
             file_storage.delete()
             return
@@ -123,12 +116,10 @@ class TabsController:
         for tab in tabs_list:
             tab_id = self._get_next_tab_id()
 
-            b_types = json.dumps({
-                'index': tab.get('index', 0),
-                'name': tab.get('name', 'undefined'),
-                'content': tab.get('value', '')
-            }).encode("utf-8")
-            file_storage.file_set(f'tab_{tab_id}', b_types)
+            b_types = json.dumps(
+                {"index": tab.get("index", 0), "name": tab.get("name", "undefined"), "content": tab.get("value", "")}
+            ).encode("utf-8")
+            file_storage.file_set(f"tab_{tab_id}", b_types)
 
         file_storage.delete(TABS_FILENAME)
 
@@ -149,12 +140,9 @@ class TabsController:
             except Exception as e:
                 logger.error(f"Can't read data of tab {ctx.company_id}/{tab_id}: {e}")
                 continue
-            tabs_list.append({
-                'id': tab_id,
-                **data
-            })
+            tabs_list.append({"id": tab_id, **data})
 
-        tabs_list.sort(key=lambda x: x['index'])
+        tabs_list.sort(key=lambda x: x["index"])
         return tabs_list
 
     def get(self, tab_id: int) -> Dict:
@@ -167,25 +155,22 @@ class TabsController:
             dict: tabs data
         """
         if isinstance(tab_id, int) is False:
-            raise ValueError('Tab id must be integer')
+            raise ValueError("Tab id must be integer")
 
         try:
-            raw_tab_data = self._get_file_storage().file_get(f'tab_{tab_id}')
-        except FileNotFoundError:
-            raise EntityNotExistsError(f'tab {tab_id}')
+            raw_tab_data = self._get_file_storage().file_get(f"tab_{tab_id}")
+        except FileNotFoundError as e:
+            raise EntityNotExistsError(f"tab {tab_id}") from e
 
         try:
             data = json.loads(raw_tab_data)
         except Exception as e:
             logger.error(f"Can't read data of tab {ctx.company_id}/{tab_id}: {e}")
-            raise Exception(f"Can't read data of tab: {e}")
+            raise Exception(f"Can't read data of tab: {e}") from e
 
-        return {
-            'id': tab_id,
-            **data
-        }
+        return {"id": tab_id, **data}
 
-    def add(self, index: int = None, name: str = 'undefined', content: str = '') -> Dict:
+    def add(self, index: int = None, name: str = "undefined", content: str = "") -> Dict:
         """Add new tab
 
         Args:
@@ -205,27 +190,23 @@ class TabsController:
             if len(all_tabs) == 0:
                 index = 0
             else:
-                index = max([x.get('index', 0) for x in all_tabs]) + 1
+                index = max([x.get("index", 0) for x in all_tabs]) + 1
 
-        data_bytes = json.dumps({
-            'index': index,
-            'name': name,
-            'content': content
-        }).encode("utf-8")
-        file_storage.file_set(f'tab_{tab_id}', data_bytes)
+        data_bytes = json.dumps({"index": index, "name": name, "content": content}).encode("utf-8")
+        file_storage.file_set(f"tab_{tab_id}", data_bytes)
 
         if reorder_required:
             all_tabs = self.get_all()
-            all_tabs.sort(key=lambda x: (x['index'], 0 if x['id'] == tab_id else 1))
+            all_tabs.sort(key=lambda x: (x["index"], 0 if x["id"] == tab_id else 1))
             file_storage.sync = False
             for tab_index, tab in enumerate(all_tabs):
-                tab['index'] = tab_index
-                data_bytes = json.dumps(tab).encode('utf-8')
-                file_storage.file_set(f'tab_{tab["id"]}', data_bytes)
+                tab["index"] = tab_index
+                data_bytes = json.dumps(tab).encode("utf-8")
+                file_storage.file_set(f"tab_{tab['id']}", data_bytes)
             file_storage.sync = True
             file_storage.push()
 
-        return {'id': tab_id, 'index': index, 'name': name}
+        return {"id": tab_id, "index": index, "name": name}
 
     def modify(self, tab_id: int, index: int = None, name: str = None, content: str = None) -> Dict:
         """Modify the tab
@@ -243,49 +224,45 @@ class TabsController:
         current_data = self.get(tab_id)
 
         # region modify index
-        if index is not None and current_data['index'] != index:
-            current_data['index'] = index
-            all_tabs = [x for x in self.get_all() if x['id'] != tab_id]
-            all_tabs.sort(key=lambda x: x['index'])
+        if index is not None and current_data["index"] != index:
+            current_data["index"] = index
+            all_tabs = [x for x in self.get_all() if x["id"] != tab_id]
+            all_tabs.sort(key=lambda x: x["index"])
             file_storage.sync = False
             for tab_index, tab in enumerate(all_tabs):
                 if tab_index < index:
-                    tab['index'] = tab_index
+                    tab["index"] = tab_index
                 else:
-                    tab['index'] = tab_index + 1
-                data_bytes = json.dumps(tab).encode('utf-8')
-                file_storage.file_set(f'tab_{tab["id"]}', data_bytes)
+                    tab["index"] = tab_index + 1
+                data_bytes = json.dumps(tab).encode("utf-8")
+                file_storage.file_set(f"tab_{tab['id']}", data_bytes)
             file_storage.sync = True
             file_storage.push()
         # endregion
 
         # region modify name
-        if name is not None and current_data['name'] != name:
-            current_data['name'] = name
+        if name is not None and current_data["name"] != name:
+            current_data["name"] = name
         # endregion
 
         # region modify content
-        if content is not None and current_data['content'] != content:
-            current_data['content'] = content
+        if content is not None and current_data["content"] != content:
+            current_data["content"] = content
         # endregion
 
-        data_bytes = json.dumps(current_data).encode('utf-8')
-        file_storage.file_set(f'tab_{tab_id}', data_bytes)
+        data_bytes = json.dumps(current_data).encode("utf-8")
+        file_storage.file_set(f"tab_{tab_id}", data_bytes)
 
-        return {
-            'id': current_data['id'],
-            'index': current_data['index'],
-            'name': current_data['name']
-        }
+        return {"id": current_data["id"], "index": current_data["index"], "name": current_data["name"]}
 
     def delete(self, tab_id: int):
         file_storage = self._get_file_storage()
         try:
-            file_storage.file_get(f'tab_{tab_id}')
-        except FileNotFoundError:
-            raise EntityNotExistsError(f'tab {tab_id}')
+            file_storage.file_get(f"tab_{tab_id}")
+        except FileNotFoundError as e:
+            raise EntityNotExistsError(f"tab {tab_id}") from e
 
-        file_storage.delete(f'tab_{tab_id}')
+        file_storage.delete(f"tab_{tab_id}")
 
 
 tabs_controller = TabsController()