MindsDB 25.5.4.2__py3-none-any.whl → 25.6.3.0__py3-none-any.whl

mindsdb/interfaces/data_catalog/data_catalog_loader.py

@@ -0,0 +1,375 @@
+from typing import List, Union
+
+import pandas as pd
+
+from mindsdb.integrations.libs.response import RESPONSE_TYPE
+from mindsdb.interfaces.data_catalog.base_data_catalog import BaseDataCatalog
+from mindsdb.interfaces.storage import db
+
+
+class DataCatalogLoader(BaseDataCatalog):
+    """
+    This class is responsible for loading the metadata from a data source (via the handler) and storing it in the data catalog.
+    """
+
+    def load_metadata(self) -> None:
+        """
+        Load the metadata from the handler and store it in the database.
+        """
+        if not self.is_data_catalog_supported():
+            return
+
+        loaded_table_names = self._get_loaded_table_names()
+
+        tables = self._load_table_metadata(loaded_table_names)
+
+        if tables:
+            columns = self._load_column_metadata(tables)
+
+            self._load_column_statistics(tables, columns)
+
+            self._load_primary_keys(tables, columns)
+
+            self._load_foreign_keys(tables, columns)
+
+        self.logger.info(f"Metadata loading completed for {self.database_name}.")
+
+    def _get_loaded_table_names(self) -> List[str]:
+        """
+        Retrieve the names of tables that are already present in the data catalog for the current integration.
+        If table_names are provided, only those tables will be checked.
+
+        Returns:
+            List[str]: Names of tables already loaded in the data catalog.
+        """
+        query = db.session.query(db.MetaTables).filter_by(integration_id=self.integration_id)
+        if self.table_names:
+            query = query.filter(db.MetaTables.name.in_(self.table_names))
+
+        tables = query.all()
+        table_names = [table.name for table in tables]
+
+        if table_names:
+            self.logger.info(f"Tables already loaded in the data catalog: {', '.join(table_names)}.")
+
+        return table_names
+
+    def _load_table_metadata(self, loaded_table_names: List[str] = None) -> List[Union[db.MetaTables, None]]:
+        """
+        Load the table metadata from the handler.
+        """
+        self.logger.info(f"Loading tables for {self.database_name}")
+        response = self.data_handler.meta_get_tables(self.table_names)
+        if response.resp_type == RESPONSE_TYPE.ERROR:
+            self.logger.error(f"Failed to load tables for {self.database_name}: {response.error_message}")
+            return []
+        elif response.resp_type == RESPONSE_TYPE.OK:
+            self.logger.error(f"No tables found for {self.database_name}.")
+            return []
+
+        df = response.data_frame
+        if df.empty:
+            self.logger.info(f"No tables to add for {self.database_name}.")
+            return []
+
+        df.columns = df.columns.str.lower()
+
+        # Filter out tables that are already loaded in the data catalog
+        if loaded_table_names:
+            df = df[~df["table_name"].isin(loaded_table_names)]
+
+        if df.empty:
+            self.logger.info(f"No new tables to load for {self.database_name}.")
+            return []
+
+        tables = self._add_table_metadata(df)
+        self.logger.info(f"Tables loaded for {self.database_name}.")
+        return tables
+
+    def _add_table_metadata(self, df: pd.DataFrame) -> List[db.MetaTables]:
+        """
+        Add the table metadata to the database.
+        """
+        tables = []
+        try:
+            for row in df.to_dict(orient="records"):
+                # Convert the distinct_values_count to an integer if it is not NaN, otherwise set it to None.
+                val = row.get("row_count")
+                row_count = int(val) if pd.notna(val) else None
+
+                record = db.MetaTables(
+                    integration_id=self.integration_id,
+                    name=row.get("table_name") or row.get("name"),
+                    schema=row.get("table_schema"),
+                    description=row.get("table_description"),
+                    type=row.get("table_type"),
+                    row_count=row_count,
+                )
+                tables.append(record)
+
+            db.session.add_all(tables)
+            db.session.commit()
+        except Exception as e:
+            self.logger.error(f"Failed to add tables: {e}")
+            db.session.rollback()
+            raise
+        return tables
+
+    def _load_column_metadata(self, tables: db.MetaTables) -> List[db.MetaColumns]:
+        """
+        Load the column metadata from the handler.
+        """
+        self.logger.info(f"Loading columns for {self.database_name}")
+        response = self.data_handler.meta_get_columns(self.table_names)
+        if response.resp_type == RESPONSE_TYPE.ERROR:
+            self.logger.error(f"Failed to load columns for {self.database_name}: {response.error_message}")
+            return []
+        elif response.resp_type == RESPONSE_TYPE.OK:
+            self.logger.error(f"No columns found for {self.database_name}.")
+            return []
+
+        df = response.data_frame
+        if df.empty:
+            self.logger.info(f"No columns to load for {self.database_name}.")
+            return []
+
+        df.columns = df.columns.str.lower()
+        columns = self._add_column_metadata(df, tables)
+        self.logger.info(f"Columns loaded for {self.database_name}.")
+        return columns
+
+    def _add_column_metadata(self, df: pd.DataFrame, tables: db.MetaTables) -> List[db.MetaColumns]:
+        """
+        Add the column metadata to the database.
+        """
+        columns = []
+        try:
+            for row in df.to_dict(orient="records"):
+                record = db.MetaColumns(
+                    table_id=next((table.id for table in tables if table.name == row.get("table_name"))),
+                    name=row.get("column_name"),
+                    data_type=row.get("data_type"),
+                    default_value=row.get("column_default"),
+                    description=row.get("description"),
+                    is_nullable=row.get("is_nullable"),
+                )
+                columns.append(record)
+
+            db.session.add_all(columns)
+            db.session.commit()
+        except Exception as e:
+            self.logger.error(f"Failed to add columns: {e}")
+            db.session.rollback()
+            raise
+        return columns
+
+    def _load_column_statistics(self, tables: db.MetaTables, columns: db.MetaColumns) -> None:
+        """
+        Load the column statistics metadata from the handler.
+        """
+        self.logger.info(f"Loading column statistics for {self.database_name}")
+        response = self.data_handler.meta_get_column_statistics(self.table_names)
+        if response.resp_type == RESPONSE_TYPE.ERROR:
+            self.logger.error(f"Failed to load column statistics for {self.database_name}: {response.error_message}")
+            return
+        elif response.resp_type == RESPONSE_TYPE.OK:
+            self.logger.error(f"No column statistics found for {self.database_name}.")
+            return
+
+        df = response.data_frame
+        if df.empty:
+            self.logger.info(f"No column statistics to load for {self.database_name}.")
+            return
+
+        df.columns = df.columns.str.lower()
+        self._add_column_statistics(df, tables, columns)
+        self.logger.info(f"Column statistics loaded for {self.database_name}.")
+
+    def _add_column_statistics(self, df: pd.DataFrame, tables: db.MetaTables, columns: db.MetaColumns) -> None:
+        """
+        Add the column statistics metadata to the database.
+        """
+        column_statistics = []
+        try:
+            for row in df.to_dict(orient="records"):
+                table_id = next((table.id for table in tables if table.name == row.get("table_name")))
+                column_id = next(
+                    (
+                        column.id
+                        for column in columns
+                        if column.name == row.get("column_name") and column.table_id == table_id
+                    )
+                )
+
+                # Convert the distinct_values_count to an integer if it is not NaN, otherwise set it to None.
+                val = row.get("distinct_values_count")
+                distinct_values_count = int(val) if pd.notna(val) else None
+
+                # Convert the most_common_frequencies to a list of strings.
+                most_common_frequencies = [str(val) for val in row.get("most_common_frequencies") or []]
+
+                record = db.MetaColumnStatistics(
+                    column_id=column_id,
+                    most_common_values=row.get("most_common_values"),
+                    most_common_frequencies=most_common_frequencies,
+                    null_percentage=row.get("null_percentage"),
+                    distinct_values_count=distinct_values_count,
+                    minimum_value=row.get("minimum_value"),
+                    maximum_value=row.get("maximum_value"),
+                )
+                column_statistics.append(record)
+
+            db.session.add_all(column_statistics)
+            db.session.commit()
+        except Exception as e:
+            self.logger.error(f"Failed to add column statistics: {e}")
+            db.session.rollback()
+            raise
+
+    def _load_primary_keys(self, tables: db.MetaTables, columns: db.MetaColumns) -> None:
+        """
+        Load the primary keys metadata from the handler.
+        """
+        self.logger.info(f"Loading primary keys for {self.database_name}")
+        response = self.data_handler.meta_get_primary_keys(self.table_names)
+        if response.resp_type == RESPONSE_TYPE.ERROR:
+            self.logger.error(f"Failed to load primary keys for {self.database_name}: {response.error_message}")
+            return
+        elif response.resp_type == RESPONSE_TYPE.OK:
+            self.logger.error(f"No primary keys found for {self.database_name}.")
+            return
+
+        df = response.data_frame
+        if df.empty:
+            self.logger.info(f"No primary keys to load for {self.database_name}.")
+            return
+
+        df.columns = df.columns.str.lower()
+        self._add_primary_keys(df, tables, columns)
+        self.logger.info(f"Primary keys loaded for {self.database_name}.")
+
+    def _add_primary_keys(self, df: pd.DataFrame, tables: db.MetaTables, columns: db.MetaColumns) -> None:
+        """
+        Add the primary keys metadata to the database.
+        """
+        primary_keys = []
+        try:
+            for row in df.to_dict(orient="records"):
+                table_id = next((table.id for table in tables if table.name == row.get("table_name")))
+                column_id = next(
+                    (
+                        column.id
+                        for column in columns
+                        if column.name == row.get("column_name") and column.table_id == table_id
+                    )
+                )
+
+                record = db.MetaPrimaryKeys(
+                    table_id=table_id,
+                    column_id=column_id,
+                    constraint_name=row.get("constraint_name"),
+                )
+                primary_keys.append(record)
+
+            db.session.add_all(primary_keys)
+            db.session.commit()
+        except Exception as e:
+            self.logger.error(f"Failed to add primary keys: {e}")
+            db.session.rollback()
+            raise
+
+    def _load_foreign_keys(self, tables: db.MetaTables, columns: db.MetaColumns) -> None:
+        """
+        Load the foreign keys metadata from the handler.
+        """
+        self.logger.info(f"Loading foreign keys for {self.database_name}")
+        response = self.data_handler.meta_get_foreign_keys(self.table_names)
+        if response.resp_type == RESPONSE_TYPE.ERROR:
+            self.logger.error(f"Failed to foreign keys for {self.database_name}: {response.error_message}")
+            return
+        elif response.resp_type == RESPONSE_TYPE.OK:
+            self.logger.error(f"No foreign keys found for {self.database_name}.")
+            return
+
+        df = response.data_frame
+        if df.empty:
+            self.logger.info(f"No foreign keys to load for {self.database_name}.")
+            return
+
+        df.columns = df.columns.str.lower()
+        self._add_foreign_keys(df, tables, columns)
+        self.logger.info(f"Foreign keys loaded for {self.database_name}.")
+
+    def _add_foreign_keys(self, df: pd.DataFrame, tables: db.MetaTables, columns: db.MetaColumns) -> None:
+        """
+        Add the foreign keys metadata to the database.
+        """
+        foreign_keys = []
+        try:
+            for row in df.to_dict(orient="records"):
+                try:
+                    parent_table_id = next((table.id for table in tables if table.name == row.get("parent_table_name")))
+                    parent_column_id = next(
+                        (
+                            column.id
+                            for column in columns
+                            if column.name == row.get("parent_column_name") and column.table_id == parent_table_id
+                        )
+                    )
+                    child_table_id = next((table.id for table in tables if table.name == row.get("child_table_name")))
+                    child_column_id = next(
+                        (
+                            column.id
+                            for column in columns
+                            if column.name == row.get("child_column_name") and column.table_id == child_table_id
+                        )
+                    )
+                except StopIteration:
+                    self.logger.warning(
+                        f"The foreign key relationship for {row.get('parent_table_name')} -> {row.get('child_table_name')} "
+                        f"could not be established. One or more tables or columns may not exist in the metadata."
+                    )
+                    continue
+
+                record = db.MetaForeignKeys(
+                    parent_table_id=parent_table_id,
+                    parent_column_id=parent_column_id,
+                    child_table_id=child_table_id,
+                    child_column_id=child_column_id,
+                    constraint_name=row.get("constraint_name"),
+                )
+                foreign_keys.append(record)
+
+            db.session.add_all(foreign_keys)
+            db.session.commit()
+        except Exception as e:
+            self.logger.error(f"Failed to add foreign keys: {e}")
+            db.session.rollback()
+            raise
+
+    def unload_metadata(self) -> None:
+        """
+        Remove the metadata for the specified database from the data catalog.
+        """
+        if not self.is_data_catalog_supported():
+            return
+
+        meta_tables = db.session.query(db.MetaTables).filter_by(integration_id=self.integration_id).all()
+
+        if not meta_tables:
+            self.logger.info(f"No metadata found for {self.database_name}. Nothing to remove.")
+            return
+
+        for table in meta_tables:
+            db.session.query(db.MetaPrimaryKeys).filter_by(table_id=table.id).delete()
+            db.session.query(db.MetaForeignKeys).filter(
+                (db.MetaForeignKeys.parent_table_id == table.id) | (db.MetaForeignKeys.child_table_id == table.id)
+            ).delete()
+            meta_columns = db.session.query(db.MetaColumns).filter_by(table_id=table.id).all()
+            for col in meta_columns:
+                db.session.query(db.MetaColumnStatistics).filter_by(column_id=col.id).delete()
+                db.session.delete(col)
+
+            db.session.delete(table)
+        db.session.commit()
+        self.logger.info(f"Metadata for {self.database_name} removed successfully.")

mindsdb/interfaces/data_catalog/data_catalog_reader.py

@@ -0,0 +1,38 @@
+from mindsdb.interfaces.data_catalog.base_data_catalog import BaseDataCatalog
+from mindsdb.interfaces.storage import db
+
+
+class DataCatalogReader(BaseDataCatalog):
+    """
+    This class is responsible for reading the metadata from the data catalog and providing it in a structured format.
+    """
+
+    def read_metadata_as_string(self) -> str:
+        """
+        Read the metadata from the data catalog and return it as a string.
+        """
+        if not self.is_data_catalog_supported():
+            return f"Data catalog is not supported for database '{self.database_name}'."
+        tables = self._read_metadata()
+        if not tables:
+            self.logger.warning(f"No metadata found for database '{self.database_name}'")
+            return f"No metadata found for database '{self.database_name}'"
+
+        metadata_str = "Data Catalog: \n"
+        if hasattr(self.data_handler, "meta_get_handler_info"):
+            metadata_str += self.data_handler.meta_get_handler_info() + "\n\n"
+
+        for table in tables:
+            metadata_str += table.as_string() + "\n\n"
+        return metadata_str
+
+    def _read_metadata(self) -> list:
+        """
+        Read the metadata from the data catalog and return it in a structured format.
+        """
+        query = db.session.query(db.MetaTables).filter_by(integration_id=self.integration_id)
+        if self.table_names:
+            cleaned_table_names = [name.strip("`").split(".")[-1] for name in self.table_names]
+            query = query.filter(db.MetaTables.name.in_(cleaned_table_names))
+        tables = query.all()
+        return tables

mindsdb/interfaces/database/database.py

@@ -11,6 +11,7 @@ from mindsdb.interfaces.database.log import LogDBController
 class DatabaseController:
     def __init__(self):
         from mindsdb.interfaces.database.integrations import integration_controller
+
         self.integration_controller = integration_controller
         self.project_controller = ProjectController()
 
@@ -21,13 +22,13 @@ class DatabaseController:
         databases = self.get_dict()
         name = name.lower()
         if name not in databases:
-            raise EntityNotExistsError('Database does not exists', name)
-        db_type = databases[name]['type']
-        if db_type == 'project':
+            raise EntityNotExistsError("Database does not exists", name)
+        db_type = databases[name]["type"]
+        if db_type == "project":
             project = self.get_project(name)
             project.delete()
             return
-        elif db_type == 'data':
+        elif db_type == "data":
             self.integration_controller.delete(name)
             return
         else:
@@ -37,59 +38,52 @@ class DatabaseController:
     def get_list(self, filter_type: Optional[str] = None, with_secrets: Optional[bool] = True):
         projects = self.project_controller.get_list()
         integrations = self.integration_controller.get_all(show_secrets=with_secrets)
-        result = [{
-            'name': 'information_schema',
-            'type': 'system',
-            'id': None,
-            'engine': None,
-            'visible': True,
-            'deletable': False
-        }, {
-            'name': 'log',
-            'type': 'system',
-            'id': None,
-            'engine': None,
-            'visible': True,
-            'deletable': False
-        }]
+        result = [
+            {
+                "name": "information_schema",
+                "type": "system",
+                "id": None,
+                "engine": None,
+                "visible": True,
+                "deletable": False,
+            },
+            {"name": "log", "type": "system", "id": None, "engine": None, "visible": True, "deletable": False},
+        ]
         for x in projects:
-            result.append({
-                'name': x.name,
-                'type': 'project',
-                'id': x.id,
-                'engine': None,
-                'visible': True,
-                'deletable': x.name.lower() != config.get('default_project')
-            })
+            result.append(
+                {
+                    "name": x.name,
+                    "type": "project",
+                    "id": x.id,
+                    "engine": None,
+                    "visible": True,
+                    "deletable": x.name.lower() != config.get("default_project"),
+                }
+            )
         for key, value in integrations.items():
-            db_type = value.get('type', 'data')
-            if db_type != 'ml':
-                result.append({
-                    'name': key,
-                    'type': value.get('type', 'data'),
-                    'id': value.get('id'),
-                    'engine': value.get('engine'),
-                    'class_type': value.get('class_type'),
-                    'connection_data': value.get('connection_data'),
-                    'visible': True,
-                    'deletable': value.get('permanent', False) is False
-                })
+            db_type = value.get("type", "data")
+            if db_type != "ml":
+                result.append(
+                    {
+                        "name": key,
+                        "type": value.get("type", "data"),
+                        "id": value.get("id"),
+                        "engine": value.get("engine"),
+                        "class_type": value.get("class_type"),
+                        "connection_data": value.get("connection_data"),
+                        "visible": True,
+                        "deletable": value.get("permanent", False) is False,
+                    }
+                )
 
         if filter_type is not None:
-            result = [x for x in result if x['type'] == filter_type]
+            result = [x for x in result if x["type"] == filter_type]
 
         return result
 
     def get_dict(self, filter_type: Optional[str] = None):
         return OrderedDict(
-            (
-                x['name'].lower(),
-                {
-                    'type': x['type'],
-                    'engine': x['engine'],
-                    'id': x['id']
-                }
-            )
+            (x["name"].lower(), {"type": x["type"], "engine": x["engine"], "id": x["id"]})
             for x in self.get_list(filter_type=filter_type)
         )
 
@@ -98,13 +92,8 @@ class DatabaseController:
 
         # TODO get directly from db?
         for rec in self.get_list():
-            if rec['id'] == integration_id and rec['type'] == 'data':
-                return {
-                    'name': rec['name'],
-                    'type': rec['type'],
-                    'engine': rec['engine'],
-                    'id': rec['id']
-                }
+            if rec["id"] == integration_id and rec["type"] == "data":
+                return {"name": rec["name"], "type": rec["type"], "engine": rec["engine"], "id": rec["id"]}
 
     def exists(self, db_name: str) -> bool:
         return db_name.lower() in self.get_dict()
@@ -113,11 +102,46 @@ class DatabaseController:
         return self.project_controller.get(name=name)
 
     def get_system_db(self, name: str):
-        if name == 'log':
+        if name == "log":
             return self.logs_db_controller
-        elif name == 'information_schema':
+        elif name == "information_schema":
             from mindsdb.api.executor.controllers.session_controller import SessionController
+
             session = SessionController()
             return session.datahub
         else:
             raise Exception(f"Database '{name}' does not exists")
+
+    def update(self, name: str, data: dict):
+        """
+        Updates the database with the given name using the provided data.
+
+        Parameters:
+            name (str): The name of the database to update.
+            data (dict): The data to update the database with.
+
+        Raises:
+            EntityNotExistsError: If the database does not exist.
+        """
+        databases = self.get_dict()
+        name = name.lower()
+        if name not in databases:
+            raise EntityNotExistsError("Database does not exist.", name)
+
+        db_type = databases[name]["type"]
+        if db_type == "project":
+            # Only the name of the project can be updated.
+            if {"name"} != set(data):
+                raise ValueError("Only the 'name' field can be updated for projects.")
+            self.project_controller.update(name=name, new_name=str(data["name"]))
+            return
+
+        elif db_type == "data":
+            # Only the parameters (connection data) of the integration can be updated.
+            if {"parameters"} != set(data):
+                raise ValueError("Only the 'parameters' field can be updated for integrations.")
+            self.integration_controller.modify(name, data["parameters"])
+            return
+
+        else:
+            raise ValueError(f"Database with type '{db_type}' cannot be updated")