MindsDB 25.2.2.1__py3-none-any.whl → 25.2.3.0__py3-none-any.whl

mindsdb/interfaces/file/file_controller.py

@@ -3,12 +3,17 @@ import os
 import shutil
 from pathlib import Path
 
-from mindsdb.integrations.handlers.file_handler import Handler as FileHandler
+import pandas as pd
+
 from mindsdb.interfaces.storage import db
 from mindsdb.interfaces.storage.fs import FsStore
 from mindsdb.utilities import log
 from mindsdb.utilities.config import Config
 from mindsdb.utilities.context import context as ctx
+from sqlalchemy.orm.attributes import flag_modified
+
+from mindsdb.integrations.utilities.files.file_reader import FileReader
+
 
 logger = log.getLogger(__name__)
 
@@ -82,31 +87,38 @@ class FileController:
 
         file_dir = None
         try:
-            df = FileHandler.handle_source(file_path)
-
-            ds_meta = {"row_count": len(df), "column_names": list(df.columns)}
+            pages_files, pages_index = self.get_file_pages(file_path)
 
+            metadata = {
+                'is_feather': True,
+                'pages': pages_index
+            }
+            df = pages_files[0]
             file_record = db.File(
                 name=name,
                 company_id=ctx.company_id,
                 source_file_path=file_name,
                 file_path="",
-                row_count=ds_meta["row_count"],
-                columns=ds_meta["column_names"],
+                row_count=len(df),
+                columns=list(df.columns),
+                metadata_=metadata
             )
             db.session.add(file_record)
-            db.session.commit()
+            db.session.flush()
+
             store_file_path = f"file_{ctx.company_id}_{file_record.id}"
             file_record.file_path = store_file_path
-            db.session.commit()
 
             file_dir = Path(self.dir).joinpath(store_file_path)
             file_dir.mkdir(parents=True, exist_ok=True)
-            source = file_dir.joinpath(file_name)
-            # NOTE may be delay between db record exists and file is really in folder
-            shutil.move(file_path, str(source))
+
+            self.store_pages_as_feather(file_dir, pages_files)
+            # store original file
+            shutil.move(file_path, str(file_dir.joinpath(file_name)))
 
             self.fs_store.put(store_file_path, base_dir=self.dir)
+            db.session.commit()
+
         except Exception as e:
             logger.error(e)
             if file_dir is not None:
@@ -115,6 +127,39 @@ class FileController:
 
         return file_record.id
 
+    def get_file_pages(self, source_path: str):
+        """
+        Reads file and extract pages from it
+        Returned structures:
+          - page_files: dict with content, {page_num: dataframe}
+          - pages_index: dict, link between page name and num: {page_name: page_num}
+        """
+        file_reader = FileReader(path=source_path)
+        tables = file_reader.get_contents()
+
+        pages_files = {}
+        pages_index = {}
+        if len(tables) == 1:
+            df = list(tables.values())[0]
+            pages_files[0] = df
+        else:
+            # file has several pages, create a new one with info
+            df = pd.DataFrame(tables.keys(), columns=["Tables"])
+            pages_files[0] = df
+            for i, page_name in enumerate(tables.keys(), 1):
+                pages_files[i] = tables[page_name]
+                pages_index[page_name] = i
+        return pages_files, pages_index
+
+    def store_pages_as_feather(self, dest_dir: Path, pages_files: dict):
+        """
+        Stores pages in file storage dir in feather format
+        """
+
+        for num, df in pages_files.items():
+            dest = dest_dir.joinpath(f'{num}.feather')
+            df.to_feather(str(dest))
+
     def delete_file(self, name):
         file_record = (
             db.session.query(db.File)
@@ -144,3 +189,87 @@ class FileController:
             .joinpath(file_dir)
             .joinpath(Path(file_record.source_file_path).name)
         )
+
+    def get_file_data(self, name: str, page_name: str = None) -> pd.DataFrame:
+        """
+        Returns file content as dataframe
+
+        :param name: name of file
+        :param page_name: page name, optional
+        :return: Page or file content
+        """
+        file_record = (
+            db.session.query(db.File)
+            .filter_by(company_id=ctx.company_id, name=name)
+            .first()
+        )
+        if file_record is None:
+            raise Exception(f"File '{name}' does not exists")
+
+        file_dir = f"file_{ctx.company_id}_{file_record.id}"
+        self.fs_store.get(file_dir, base_dir=self.dir)
+
+        metadata = file_record.metadata_ or {}
+        if metadata.get('is_feather') is not True:
+            # migrate file
+
+            file_path = (
+                Path(self.dir)
+                .joinpath(file_dir)
+                .joinpath(Path(file_record.source_file_path).name)
+            )
+
+            pages_files, pages_index = self.get_file_pages(str(file_path))
+
+            self.store_pages_as_feather(file_path.parent, pages_files)
+            metadata['is_feather'] = True
+            metadata['pages'] = pages_index
+
+            file_record.metadata_ = metadata
+            flag_modified(file_record, 'metadata_')
+            db.session.commit()
+
+        if page_name is None:
+            num = 0
+        else:
+            num = metadata.get('pages', {}).get(page_name)
+            if num is None:
+                raise KeyError(f'Page not found: {page_name}')
+
+        path = (
+            Path(self.dir)
+            .joinpath(file_dir)
+            .joinpath(f'{num}.feather')
+        )
+        return pd.read_feather(path)
+
+    def set_file_data(self, name: str, df: pd.DataFrame, page_name: str = None):
+        """
+        Save file content
+        :param name: name of file
+        :param df: content to store
+        :param page_name: name of page, optional
+        """
+
+        file_record = (
+            db.session.query(db.File)
+            .filter_by(company_id=ctx.company_id, name=name)
+            .first()
+        )
+        if file_record is None:
+            raise Exception(f"File '{name}' does not exists")
+
+        file_dir = f"file_{ctx.company_id}_{file_record.id}"
+        self.fs_store.get(file_dir, base_dir=self.dir)
+
+        num = 0
+        if page_name is not None and file_record.metadata_ is not None:
+            num = file_record.metadata_.get('pages', {}).get(page_name, 0)
+
+        path = (
+            Path(self.dir)
+            .joinpath(file_dir)
+            .joinpath(f'{num}.feather')
+        )
+        df.to_feather(path)
+        self.fs_store.put(file_dir, base_dir=self.dir)

mindsdb/interfaces/storage/db.py

@@ -258,6 +258,7 @@ class File(Base):
     row_count = Column(Integer, nullable=False)
     columns = Column(Json, nullable=False)
     created_at = Column(DateTime, default=datetime.datetime.now)
+    metadata_: dict = Column("metadata", JSON, nullable=True)
     updated_at = Column(
         DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now
     )

mindsdb/migrations/versions/2025-02-09_4943359e354a_file_metadata.py

@@ -0,0 +1,31 @@
+"""file_metadata
+
+Revision ID: 4943359e354a
+Revises: c06c35f7e8e1
+Create Date: 2025-02-09 10:10:55.577407
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import mindsdb.interfaces.storage.db  # noqa
+
+
+# revision identifiers, used by Alembic.
+revision = '4943359e354a'
+down_revision = 'c06c35f7e8e1'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('file', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('metadata', sa.JSON(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('file', schema=None) as batch_op:
+        batch_op.drop_column('metadata')
+    # ### end Alembic commands ###

mindsdb/utilities/render/sqlalchemy_render.py

@@ -93,9 +93,6 @@ class SqlalchemyRender:
         if hasattr(dialect, 'preparer'):
             class Preparer(dialect.preparer):
 
-                def __init__(self, *args, **kwargs):
-                    super().__init__(*args, **kwargs)
-
                 def _requires_quotes(self, value: str) -> bool:
                     # check force-quote flag
                     if isinstance(value, AttributedStr):
@@ -242,6 +239,8 @@ class SqlalchemyRender:
 
             op = t.op.lower()
             if op in ('in', 'not in'):
+                if t.args[1].parentheses:
+                    arg1 = [arg1]
                 if isinstance(arg1, sa.sql.selectable.ColumnClause):
                     raise NotImplementedError(f'Required list argument for: {op}')
 
@@ -536,12 +535,19 @@ class SqlalchemyRender:
                 query = query.select_from(table)
 
                 # other tables
+                has_explicit_join = False
                 for item in join_list[1:]:
                     table = self.to_table(item['table'])
                     if item['is_implicit']:
                         # add to from clause
-                        query = query.select_from(table)
+                        if has_explicit_join:
+                            # sqlalchemy doesn't support implicit join after explicit
+                            # convert it to explicit
+                            query = query.join(table, sa.text('1=1'))
+                        else:
+                            query = query.select_from(table)
                     else:
+                        has_explicit_join = True
                         if item['condition'] is None:
                             # otherwise, sqlalchemy raises "Don't know how to join to ..."
                             condition = sa.text('1=1')
@@ -564,7 +570,7 @@ class SqlalchemyRender:
                             condition,
                             full=is_full
                         )
-            elif isinstance(from_table, ast.Union):
+            elif isinstance(from_table, (ast.Union, ast.Intersect, ast.Except)):
                 alias = None
                 if from_table.alias:
                     alias = self.get_alias(from_table.alias)