MindsDB 25.2.4.0__py3-none-any.whl → 25.3.2.0__py3-none-any.whl

mindsdb/interfaces/storage/json.py

@@ -1,3 +1,5 @@
+from mindsdb.utilities.functions import decrypt_json, encrypt_json
+from mindsdb.utilities.config import config
 from mindsdb.interfaces.storage import db
 from mindsdb.interfaces.storage.fs import RESOURCE_GROUP
 from mindsdb.utilities.context import context as ctx
@@ -90,8 +92,65 @@ class JsonStorage:
             logger.error('cant delete records from JSON storage')
 
 
+class EncryptedJsonStorage(JsonStorage):
+    def __init__(self, resource_group: str, resource_id: int):
+        super().__init__(resource_group, resource_id)
+        self.secret_key = config.get('secret_key', 'dummy-key')
+
+    def __setitem__(self, key: str, value: dict) -> None:
+        if isinstance(value, dict) is False:
+            raise TypeError(f"got {type(value)} instead of dict")
+
+        encrypted_value = encrypt_json(value, self.secret_key)
+
+        existing_record = self.get_record(key)
+        if existing_record is None:
+            record = db.JsonStorage(
+                name=key,
+                resource_group=self.resource_group,
+                resource_id=self.resource_id,
+                company_id=ctx.company_id,
+                encrypted_content=encrypted_value
+            )
+            db.session.add(record)
+        else:
+            existing_record.encrypted_content = encrypted_value
+        db.session.commit()
+
+    def set_bytes(self, key: str, encrypted_value: bytes):
+        existing_record = self.get_record(key)
+        if existing_record is None:
+            record = db.JsonStorage(
+                name=key,
+                resource_group=self.resource_group,
+                resource_id=self.resource_id,
+                company_id=ctx.company_id,
+                encrypted_content=encrypted_value
+            )
+            db.session.add(record)
+        else:
+            existing_record.encrypted_content = encrypted_value
+        db.session.commit()
+
+    def set_str(self, key: str, encrypted_value: str):
+        self.set_bytes(key, encrypted_value.encode())
+
+    def __getitem__(self, key: str) -> dict:
+        record = self.get_record(key)
+        if record is None:
+            return None
+        return decrypt_json(record.encrypted_content, self.secret_key)
+
+
 def get_json_storage(resource_id: int, resource_group: str = RESOURCE_GROUP.PREDICTOR):
     return JsonStorage(
         resource_group=resource_group,
         resource_id=resource_id,
     )
+
+
+def get_encrypted_json_storage(resource_id: int, resource_group: str = RESOURCE_GROUP.PREDICTOR):
+    return EncryptedJsonStorage(
+        resource_group=resource_group,
+        resource_id=resource_id,
+    )

mindsdb/interfaces/storage/model_fs.py

@@ -1,5 +1,6 @@
 import os
 import re
+import json
 import io
 import zipfile
 from typing import Union
@@ -7,7 +8,10 @@ from typing import Union
 import mindsdb.interfaces.storage.db as db
 
 from .fs import RESOURCE_GROUP, FileStorageFactory, SERVICE_FILES_NAMES
-from .json import get_json_storage
+from .json import get_json_storage, get_encrypted_json_storage
+
+
+JSON_STORAGE_FILE = 'json_storage.json'
 
 
 class ModelStorage:
@@ -119,6 +123,13 @@ class ModelStorage:
         )
         return json_storage.set(name, data)
 
+    def encrypted_json_set(self, name: str, data: dict) -> None:
+        json_storage = get_encrypted_json_storage(
+            resource_id=self.predictor_id,
+            resource_group=RESOURCE_GROUP.PREDICTOR
+        )
+        return json_storage.set(name, data)
+
     def json_get(self, name):
         json_storage = get_json_storage(
             resource_id=self.predictor_id,
@@ -126,6 +137,13 @@ class ModelStorage:
         )
         return json_storage.get(name)
 
+    def encrypted_json_get(self, name: str) -> dict:
+        json_storage = get_encrypted_json_storage(
+            resource_id=self.predictor_id,
+            resource_group=RESOURCE_GROUP.PREDICTOR
+        )
+        return json_storage.get(name)
+
     def json_list(self):
         ...
 
@@ -237,6 +255,13 @@ class HandlerStorage:
         )
         return json_storage.set(name, content)
 
+    def encrypted_json_set(self, name: str, content: dict) -> None:
+        json_storage = get_encrypted_json_storage(
+            resource_id=self.integration_id,
+            resource_group=RESOURCE_GROUP.INTEGRATION
+        )
+        return json_storage.set(name, content)
+
     def json_get(self, name):
         json_storage = get_json_storage(
             resource_id=self.integration_id,
@@ -244,6 +269,13 @@ class HandlerStorage:
         )
         return json_storage.get(name)
 
+    def encrypted_json_get(self, name: str) -> dict:
+        json_storage = get_encrypted_json_storage(
+            resource_id=self.integration_id,
+            resource_group=RESOURCE_GROUP.INTEGRATION
+        )
+        return json_storage.get(name)
+
     def json_list(self):
         ...
 
@@ -251,8 +283,11 @@ class HandlerStorage:
         ...
 
     def export_files(self) -> bytes:
-        if self.is_empty():
+        json_storage = self.export_json_storage()
+
+        if self.is_empty() and not json_storage:
             return None
+
         folder_path = self.folder_get('')
 
         zip_fd = io.BytesIO()
@@ -265,6 +300,11 @@ class HandlerStorage:
                     abs_path = os.path.join(root, file_name)
                     zipf.write(abs_path, os.path.relpath(abs_path, folder_path))
 
+            # If JSON storage is not empty, add it to the zip file.
+            if json_storage:
+                json_str = json.dumps(json_storage)
+                zipf.writestr(JSON_STORAGE_FILE, json_str)
+
         zip_fd.seek(0)
         return zip_fd.read()
 
@@ -277,6 +317,48 @@ class HandlerStorage:
         zip_fd.seek(0)
 
         with zipfile.ZipFile(zip_fd, 'r') as zip_ref:
-            zip_ref.extractall(folder_path)
+            for name in zip_ref.namelist():
+                # If JSON storage file is in the zip file, import the content to the JSON storage.
+                # Thereafter, remove the file from the folder.
+                if name == JSON_STORAGE_FILE:
+                    json_storage = zip_ref.read(JSON_STORAGE_FILE)
+                    self.import_json_storage(json_storage)
+
+                else:
+                    zip_ref.extract(name, folder_path)
 
         self.folder_sync('')
+
+    def export_json_storage(self) -> list[dict]:
+        json_storage = get_json_storage(
+            resource_id=self.integration_id,
+            resource_group=RESOURCE_GROUP.INTEGRATION
+        )
+
+        records = []
+        for record in json_storage.get_all_records():
+            record_dict = record.to_dict()
+            if record_dict.get('encrypted_content'):
+                record_dict['encrypted_content'] = record_dict['encrypted_content'].decode()
+            records.append(record_dict)
+
+        return records
+
+    def import_json_storage(self, records: bytes) -> None:
+        json_storage = get_json_storage(
+            resource_id=self.integration_id,
+            resource_group=RESOURCE_GROUP.INTEGRATION
+        )
+
+        encrypted_json_storage = get_encrypted_json_storage(
+            resource_id=self.integration_id,
+            resource_group=RESOURCE_GROUP.INTEGRATION
+        )
+
+        records = json.loads(records.decode())
+
+        for record in records:
+            if record['encrypted_content']:
+                encrypted_json_storage.set_str(record['name'], record['encrypted_content'])
+            else:
+                json_storage.set(record['name'], record['content'])

mindsdb/interfaces/triggers/triggers_controller.py

@@ -5,6 +5,7 @@ from mindsdb_sql_parser import parse_sql, ParsingException
 from mindsdb.interfaces.storage import db
 from mindsdb.interfaces.database.projects import ProjectController
 from mindsdb.utilities.context import context as ctx
+from mindsdb.utilities.config import config
 
 from mindsdb.api.executor.controllers.session_controller import SessionController
 
@@ -16,7 +17,7 @@ class TriggersController:
         name = name.lower()
 
         if project_name is None:
-            project_name = 'mindsdb'
+            project_name = config.get('default_project')
         project_controller = ProjectController()
         project = project_controller.get(name=project_name)
 

mindsdb/migrations/versions/2022-10-14_43c52d23845a_projects.py

@@ -32,12 +32,26 @@ def upgrade():
         sa.UniqueConstraint('name', 'company_id', name='unique_project_name_company_id')
     )
 
+    project_table = sa.Table(
+        'project',
+        sa.MetaData(),
+        sa.Column('id', sa.Integer()),
+        sa.Column('name', sa.String()),
+        sa.Column('company_id', sa.Integer()),
+    )
+
     conn = op.get_bind()
     session = sa.orm.Session(bind=conn)
 
-    project_record = db.Project(name='mindsdb')
-    session.add(project_record)
-    session.commit()
+    conn.execute(
+        project_table.insert().values(
+            name='mindsdb'
+        )
+    )
+
+    project_record = conn.execute(
+        project_table.select().where(project_table.c.name == 'mindsdb')
+    ).fetchone()
 
     with op.batch_alter_table('predictor', schema=None) as batch_op:
         batch_op.add_column(sa.Column('project_id', sa.Integer()))

mindsdb/migrations/versions/2025-02-14_4521dafe89ab_added_encrypted_content_to_json_storage.py

@@ -0,0 +1,29 @@
+"""added_encrypted_content_to_json_storage
+
+Revision ID: 4521dafe89ab
+Revises: 6ab9903fc59a
+Create Date: 2025-02-14 12:05:13.102594
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import mindsdb.interfaces.storage.db  # noqa
+
+
+# revision identifiers, used by Alembic.
+revision = '4521dafe89ab'
+down_revision = '6ab9903fc59a'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table('json_storage', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('encrypted_content', sa.LargeBinary(), nullable=True))
+        batch_op.alter_column('resource_id', existing_type=sa.Integer(), type_=sa.BigInteger())
+
+
+def downgrade():
+    with op.batch_alter_table('json_storage', schema=None) as batch_op:
+        batch_op.drop_column('encrypted_content')
+        batch_op.alter_column('resource_id', existing_type=sa.BigInteger(), type_=sa.Integer())

mindsdb/migrations/versions/2025-02-19_11347c213b36_added_metadata_to_projects.py

@@ -0,0 +1,41 @@
+"""added_metadata_to_projects
+
+Revision ID: 11347c213b36
+Revises: 4521dafe89ab
+Create Date: 2025-02-19 18:46:24.014843
+
+"""
+from alembic import op
+from sqlalchemy.orm.attributes import flag_modified
+import sqlalchemy as sa
+
+import mindsdb.interfaces.storage.db as db
+from mindsdb.utilities.config import config
+
+
+# revision identifiers, used by Alembic.
+revision = '11347c213b36'
+down_revision = '4521dafe89ab'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table('project', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('metadata', sa.JSON(), nullable=True))
+
+    conn = op.get_bind()
+    session = sa.orm.Session(bind=conn)
+    session.commit()
+
+    project = session.query(db.Project).filter_by(name='mindsdb').first()
+    if project:
+        project.name = config.get('default_project')
+        project.metadata_ = {"is_default": True}
+        flag_modified(project, 'metadata_')
+        session.commit()
+
+
+def downgrade():
+    with op.batch_alter_table('project', schema=None) as batch_op:
+        batch_op.drop_column('metadata')

mindsdb/utilities/config.py

@@ -217,7 +217,8 @@ class Config:
             },
             "tasks": {
                 "disable": False
-            }
+            },
+            "default_project": "mindsdb"
         }
         # endregion
 
@@ -353,6 +354,9 @@ class Config:
         if os.environ.get('MINDSDB_DB_CON', '') != '':
             self._env_config['storage_db'] = os.environ['MINDSDB_DB_CON']
 
+        if os.environ.get('MINDSDB_DEFAULT_PROJECT', '') != '':
+            self._env_config['default_project'] = os.environ['MINDSDB_DEFAULT_PROJECT'].lower()
+
     def parse_cmd_args(self) -> None:
         """Collect cmd args to self._cmd_args (accessable as self.cmd_args)
         """
@@ -495,7 +499,7 @@ class Config:
 
         for env_name in ('MINDSDB_HTTP_SERVER_TYPE', 'MINDSDB_DEFAULT_SERVER'):
             env_value = os.environ.get(env_name, '')
-            if env_value.lower() not in ('waitress', 'flask', 'gunicorn'):
+            if env_value.lower() not in ('waitress', 'flask', 'gunicorn', ''):
                 logger.warning(
                     f"The value '{env_value}' of the environment variable {env_name} is not valid. "
                     "It must be one of the following: 'waitress', 'flask', or 'gunicorn'."

mindsdb/utilities/functions.py

@@ -1,6 +1,7 @@
 import os
 import base64
 import hashlib
+import json
 import datetime
 import textwrap
 from functools import wraps
@@ -195,3 +196,13 @@ def decrypt(encripted: bytes, key: str) -> bytes:
 
     cipher = Fernet(fernet_key)
     return cipher.decrypt(encripted)
+
+
+def encrypt_json(data: dict, key: str) -> bytes:
+    json_str = json.dumps(data)
+    return encrypt(json_str.encode(), key)
+
+
+def decrypt_json(encrypted_data: bytes, key: str) -> dict:
+    decrypted = decrypt(encrypted_data, key)
+    return json.loads(decrypted)