MindsDB 25.5.4.2__py3-none-any.whl → 25.6.2.0__py3-none-any.whl

mindsdb/interfaces/storage/db.py

@@ -5,6 +5,7 @@ from typing import Dict, List
 import numpy as np
 from sqlalchemy import (
     JSON,
+    BigInteger,
     Boolean,
     Column,
     DateTime,
@@ -16,7 +17,7 @@ from sqlalchemy import (
     UniqueConstraint,
     create_engine,
     text,
-    types
+    types,
 )
 from sqlalchemy.exc import OperationalError
 from sqlalchemy.orm import (
@@ -45,7 +46,7 @@ session, engine = None, None
 def init(connection_str: str = None):
     global Base, session, engine
     if connection_str is None:
-        connection_str = config['storage_db']
+        connection_str = config["storage_db"]
     base_args = {
         "pool_size": 30,
         "max_overflow": 200,
@@ -144,15 +145,11 @@ class Predictor(Base):
     __tablename__ = "predictor"
 
     id = Column(Integer, primary_key=True)
-    updated_at = Column(
-        DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now
-    )
+    updated_at = Column(DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now)
     created_at = Column(DateTime, default=datetime.datetime.now)
     deleted_at = Column(DateTime)
     name = Column(String)
-    data = Column(
-        Json
-    )  # A JSON -- should be everything returned by `get_model_data`, I think
+    data = Column(Json)  # A JSON -- should be everything returned by `get_model_data`, I think
     to_predict = Column(Array)
     company_id = Column(Integer)
     mindsdb_version = Column(String)
@@ -173,9 +170,7 @@ class Predictor(Base):
     code = Column(String, nullable=True)
     lightwood_version = Column(String, nullable=True)
     dtype_dict = Column(Json, nullable=True)
-    project_id = Column(
-        Integer, ForeignKey("project.id", name="fk_project_id"), nullable=False
-    )
+    project_id = Column(Integer, ForeignKey("project.id", name="fk_project_id"), nullable=False)
     training_phase_current = Column(Integer)
     training_phase_total = Column(Integer)
     training_phase_name = Column(String)
@@ -199,7 +194,7 @@ Index(
     Predictor.version,
     Predictor.active,
     Predictor.deleted_at,  # would be good to have here nullsfirst(Predictor.deleted_at)
-    unique=True
+    unique=True,
 )
 
 
@@ -208,34 +203,27 @@ class Project(Base):
 
     id = Column(Integer, primary_key=True)
     created_at = Column(DateTime, default=datetime.datetime.now)
-    updated_at = Column(
-        DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now
-    )
+    updated_at = Column(DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now)
     deleted_at = Column(DateTime)
     name = Column(String, nullable=False)
     company_id = Column(Integer, default=0)
     metadata_: dict = Column("metadata", JSON, nullable=True)
-    __table_args__ = (
-        UniqueConstraint("name", "company_id", name="unique_project_name_company_id"),
-    )
+    __table_args__ = (UniqueConstraint("name", "company_id", name="unique_project_name_company_id"),)
 
 
 class Integration(Base):
     __tablename__ = "integration"
     id = Column(Integer, primary_key=True)
-    updated_at = Column(
-        DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now
-    )
+    updated_at = Column(DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now)
     created_at = Column(DateTime, default=datetime.datetime.now)
     name = Column(String, nullable=False)
     engine = Column(String, nullable=False)
     data = Column(Json)
     company_id = Column(Integer)
-    __table_args__ = (
-        UniqueConstraint(
-            "name", "company_id", name="unique_integration_name_company_id"
-        ),
-    )
+
+    meta_tables = relationship("MetaTables", back_populates="integration")
+
+    __table_args__ = (UniqueConstraint("name", "company_id", name="unique_integration_name_company_id"),)
 
 
 class File(Base):
@@ -249,12 +237,8 @@ class File(Base):
     columns = Column(Json, nullable=False)
     created_at = Column(DateTime, default=datetime.datetime.now)
     metadata_: dict = Column("metadata", JSON, nullable=True)
-    updated_at = Column(
-        DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now
-    )
-    __table_args__ = (
-        UniqueConstraint("name", "company_id", name="unique_file_name_company_id"),
-    )
+    updated_at = Column(DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now)
+    __table_args__ = (UniqueConstraint("name", "company_id", name="unique_file_name_company_id"),)
 
 
 class View(Base):
@@ -263,12 +247,8 @@ class View(Base):
     name = Column(String, nullable=False)
     company_id = Column(Integer)
     query = Column(String, nullable=False)
-    project_id = Column(
-        Integer, ForeignKey("project.id", name="fk_project_id"), nullable=False
-    )
-    __table_args__ = (
-        UniqueConstraint("name", "company_id", name="unique_view_name_company_id"),
-    )
+    project_id = Column(Integer, ForeignKey("project.id", name="fk_project_id"), nullable=False)
+    __table_args__ = (UniqueConstraint("name", "company_id", name="unique_view_name_company_id"),)
 
 
 class JsonStorage(Base):
@@ -310,9 +290,7 @@ class Jobs(Base):
     schedule_str = Column(String)
 
     deleted_at = Column(DateTime)
-    updated_at = Column(
-        DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now
-    )
+    updated_at = Column(DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now)
     created_at = Column(DateTime, default=datetime.datetime.now)
 
 
@@ -331,9 +309,7 @@ class JobsHistory(Base):
     created_at = Column(DateTime, default=datetime.datetime.now)
     updated_at = Column(DateTime, default=datetime.datetime.now)
 
-    __table_args__ = (
-        UniqueConstraint("job_id", "start_at", name="uniq_job_history_job_id_start"),
-    )
+    __table_args__ = (UniqueConstraint("job_id", "start_at", name="uniq_job_history_job_id_start"),)
 
 
 class ChatBots(Base):
@@ -349,9 +325,7 @@ class ChatBots(Base):
     database_id = Column(Integer)
     params = Column(JSON)
 
-    updated_at = Column(
-        DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now
-    )
+    updated_at = Column(DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now)
     created_at = Column(DateTime, default=datetime.datetime.now)
     webhook_token = Column(String)
 
@@ -393,9 +367,7 @@ class Triggers(Base):
     query_str = Column(String, nullable=False)
     columns = Column(String)  # list of columns separated by delimiter
 
-    updated_at = Column(
-        DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now
-    )
+    updated_at = Column(DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now)
     created_at = Column(DateTime, default=datetime.datetime.now)
 
 
@@ -417,9 +389,7 @@ class Tasks(Base):
     run_by = Column(String)
     alive_time = Column(DateTime(timezone=True))
 
-    updated_at = Column(
-        DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now
-    )
+    updated_at = Column(DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now)
     created_at = Column(DateTime, default=datetime.datetime.now)
 
 
@@ -444,9 +414,7 @@ class Skills(Base):
     params = Column(JSON)
 
     created_at = Column(DateTime, default=datetime.datetime.now)
-    updated_at = Column(
-        DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now
-    )
+    updated_at = Column(DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now)
     deleted_at = Column(DateTime)
 
     def as_dict(self) -> Dict:
@@ -475,9 +443,7 @@ class Agents(Base):
     provider = Column(String, nullable=True)
     params = Column(JSON)
 
-    updated_at = Column(
-        DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now
-    )
+    updated_at = Column(DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now)
     created_at = Column(DateTime, default=datetime.datetime.now)
     deleted_at = Column(DateTime)
 
@@ -520,21 +486,13 @@ class KnowledgeBase(Base):
         doc="fk to the embedding model",
     )
 
-    embedding_model = relationship(
-        "Predictor", foreign_keys=[embedding_model_id], doc="embedding model"
-    )
+    embedding_model = relationship("Predictor", foreign_keys=[embedding_model_id], doc="embedding model")
     query_id = Column(Integer, nullable=True)
 
     created_at = Column(DateTime, default=datetime.datetime.now)
-    updated_at = Column(
-        DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now
-    )
+    updated_at = Column(DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now)
 
-    __table_args__ = (
-        UniqueConstraint(
-            "name", "project_id", name="unique_knowledge_base_name_project_id"
-        ),
-    )
+    __table_args__ = (UniqueConstraint("name", "project_id", name="unique_knowledge_base_name_project_id"),)
 
     def as_dict(self) -> Dict:
         return {
@@ -546,7 +504,7 @@ class KnowledgeBase(Base):
             "vector_database_table": self.vector_database_table,
             "updated_at": self.updated_at,
             "created_at": self.created_at,
-            "params": self.params
+            "params": self.params,
         }
 
 
@@ -559,9 +517,7 @@ class QueryContext(Base):
     context_name: str = Column(String, nullable=False)
     values: dict = Column(JSON)
 
-    updated_at: datetime.datetime = Column(
-        DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now
-    )
+    updated_at: datetime.datetime = Column(DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now)
     created_at: datetime.datetime = Column(DateTime, default=datetime.datetime.now)
 
 
@@ -581,9 +537,7 @@ class Queries(Base):
     processed_rows = Column(Integer, default=0)
     error: str = Column(String, nullable=True)
 
-    updated_at: datetime.datetime = Column(
-        DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now
-    )
+    updated_at: datetime.datetime = Column(DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now)
     created_at: datetime.datetime = Column(DateTime, default=datetime.datetime.now)
 
 
@@ -610,10 +564,11 @@ class LLMLog(Base):
 
 
 class LLMData(Base):
-    '''
+    """
     Stores the question/answer pairs of an LLM call so examples can be used
     for self improvement with DSPy
-    '''
+    """
+
     __tablename__ = "llm_data"
     id: int = Column(Integer, primary_key=True)
     input: str = Column(String, nullable=False)
@@ -621,3 +576,192 @@ class LLMData(Base):
     model_id: int = Column(Integer, nullable=False)
     created_at: datetime = Column(DateTime, default=datetime.datetime.now)
     updated_at: datetime = Column(DateTime, onupdate=datetime.datetime.now)
+
+
+# Data Catalog
+class MetaTables(Base):
+    __tablename__ = "meta_tables"
+    id: int = Column(Integer, primary_key=True)
+
+    integration_id: int = Column(Integer, ForeignKey("integration.id"))
+    integration = relationship("Integration", back_populates="meta_tables")
+
+    name: str = Column(String, nullable=False)
+    schema: str = Column(String, nullable=True)
+    description: str = Column(String, nullable=True)
+    type: str = Column(String, nullable=True)
+    row_count: int = Column(BigInteger, nullable=True)
+
+    meta_columns: Mapped[List["MetaColumns"]] = relationship("MetaColumns", back_populates="meta_tables")
+    meta_primary_keys: Mapped[List["MetaPrimaryKeys"]] = relationship("MetaPrimaryKeys", back_populates="meta_tables")
+    meta_foreign_keys_parents: Mapped[List["MetaForeignKeys"]] = relationship(
+        "MetaForeignKeys", foreign_keys="MetaForeignKeys.parent_table_id", back_populates="parent_table"
+    )
+    meta_foreign_keys_children: Mapped[List["MetaForeignKeys"]] = relationship(
+        "MetaForeignKeys", foreign_keys="MetaForeignKeys.child_table_id", back_populates="child_table"
+    )
+
+    def as_string(self, indent: int = 0) -> str:
+        pad = " " * indent
+
+        table_info = f"`{self.integration.name}`.`{self.name}` ({self.type})"
+
+        if self.description:
+            table_info += f" : {self.description}"
+
+        if self.schema:
+            table_info += f"\n{pad}Schema: {self.schema}"
+
+        if self.row_count and self.row_count > 0:
+            table_info += f"\n{pad}Estimated Row Count: {self.row_count}"
+
+        if self.meta_primary_keys:
+            table_info += f"\n{pad}Primary Keys (in defined order): {', '.join([pk.as_string() for pk in self.meta_primary_keys])}"
+
+        if self.meta_columns:
+            table_info += f"\n\n{pad}Columns:"
+            for index, column in enumerate(self.meta_columns, start=1):
+                table_info += f"\n{index}. {column.as_string(indent + 4)}\n"
+
+        if self.meta_foreign_keys_children:
+            table_info += f"\n\n{pad}Key Relationships:"
+            for fk in self.meta_foreign_keys_children:
+                table_info += f"\n{pad}  {fk.as_string()}"
+
+        return table_info
+
+
+class MetaColumns(Base):
+    __tablename__ = "meta_columns"
+    id: int = Column(Integer, primary_key=True)
+
+    table_id: int = Column(Integer, ForeignKey("meta_tables.id"))
+    meta_tables = relationship("MetaTables", back_populates="meta_columns")
+
+    name: str = Column(String, nullable=False)
+    data_type: str = Column(String, nullable=False)
+    description: str = Column(String, nullable=True)
+    default_value: str = Column(String, nullable=True)
+    is_nullable: bool = Column(Boolean, nullable=True)
+
+    meta_column_statistics: Mapped[List["MetaColumnStatistics"]] = relationship(
+        "MetaColumnStatistics", back_populates="meta_columns"
+    )
+    meta_primary_keys: Mapped[List["MetaPrimaryKeys"]] = relationship("MetaPrimaryKeys", back_populates="meta_columns")
+    meta_foreign_keys_parents: Mapped[List["MetaForeignKeys"]] = relationship(
+        "MetaForeignKeys", foreign_keys="MetaForeignKeys.parent_column_id", back_populates="parent_column"
+    )
+    meta_foreign_keys_children: Mapped[List["MetaForeignKeys"]] = relationship(
+        "MetaForeignKeys", foreign_keys="MetaForeignKeys.child_column_id", back_populates="child_column"
+    )
+
+    def as_string(self, indent: int = 0) -> str:
+        pad = " " * indent
+
+        column_info = f"{self.name} ({self.data_type}):"
+        if self.description:
+            column_info += f"\n{pad}Description: {self.description}"
+
+        if self.is_nullable:
+            column_info += f"\n{pad}- Nullable: Yes"
+
+        if self.default_value:
+            column_info += f"\n{pad}- Default Value: {self.default_value}"
+
+        if self.meta_column_statistics:
+            column_info += f"\n\n{pad}- Column Statistics:"
+            column_info += f"\n{self.meta_column_statistics[0].as_string(indent + 4)}"
+
+        return column_info
+
+
+class MetaColumnStatistics(Base):
+    __tablename__ = "meta_column_statistics"
+    column_id: int = Column(Integer, ForeignKey("meta_columns.id"), primary_key=True)
+    meta_columns = relationship("MetaColumns", back_populates="meta_column_statistics")
+
+    most_common_values: str = Column(Array, nullable=True)
+    most_common_frequencies: str = Column(Array, nullable=True)
+    null_percentage: float = Column(Numeric(5, 2), nullable=True)
+    distinct_values_count: int = Column(BigInteger, nullable=True)
+    minimum_value: str = Column(String, nullable=True)
+    maximum_value: str = Column(String, nullable=True)
+
+    def as_string(self, indent: int = 0) -> str:
+        pad = " " * indent
+        inner_pad = " " * (indent + 4)
+
+        column_statistics = ""
+
+        if any(self.most_common_values) and any(self.most_common_frequencies):
+            column_statistics += f"{pad}- Top 10 Most Common Values and Frequencies:"
+            for i in range(min(10, len(self.most_common_values))):
+                freq = self.most_common_frequencies[i]
+                try:
+                    percent = float(freq) * 100
+                    freq_str = f"{percent:.2f}%"
+                except (ValueError, TypeError):
+                    freq_str = str(freq)
+
+                column_statistics += f"\n{inner_pad}- {self.most_common_values[i]}: {freq_str}"
+            column_statistics += "\n"
+
+        if self.null_percentage:
+            column_statistics += f"{pad}- Null Percentage: {self.null_percentage}\n"
+
+        if self.distinct_values_count:
+            column_statistics += f"{pad}- No. of Distinct Values: {self.distinct_values_count}\n"
+
+        if self.minimum_value:
+            column_statistics += f"{pad}- Minimum Value: {self.minimum_value}\n"
+
+        if self.maximum_value:
+            column_statistics += f"{pad}- Maximum Value: {self.maximum_value}"
+
+        return column_statistics
+
+
+class MetaPrimaryKeys(Base):
+    __tablename__ = "meta_primary_keys"
+    table_id: int = Column(Integer, ForeignKey("meta_tables.id"), primary_key=True)
+    meta_tables = relationship("MetaTables", back_populates="meta_primary_keys")
+
+    column_id: int = Column(Integer, ForeignKey("meta_columns.id"), primary_key=True)
+    meta_columns = relationship("MetaColumns", back_populates="meta_primary_keys")
+
+    ordinal_position: int = Column(Integer, nullable=True)
+    constraint_name: str = Column(String, nullable=True)
+
+    def as_string(self) -> str:
+        pk_list = sorted(
+            self.meta_tables.meta_primary_keys,
+            key=lambda pk: pk.ordinal_position if pk.ordinal_position is not None else 0,
+        )
+
+        return ", ".join(f"{pk.meta_columns.name} ({pk.meta_columns.data_type})" for pk in pk_list)
+
+
+class MetaForeignKeys(Base):
+    __tablename__ = "meta_foreign_keys"
+    parent_table_id: int = Column(Integer, ForeignKey("meta_tables.id"), primary_key=True)
+    parent_table = relationship(
+        "MetaTables", back_populates="meta_foreign_keys_parents", foreign_keys=[parent_table_id]
+    )
+
+    parent_column_id: int = Column(Integer, ForeignKey("meta_columns.id"), primary_key=True)
+    parent_column = relationship(
+        "MetaColumns", back_populates="meta_foreign_keys_parents", foreign_keys=[parent_column_id]
+    )
+
+    child_table_id: int = Column(Integer, ForeignKey("meta_tables.id"), primary_key=True)
+    child_table = relationship("MetaTables", back_populates="meta_foreign_keys_children", foreign_keys=[child_table_id])
+
+    child_column_id: int = Column(Integer, ForeignKey("meta_columns.id"), primary_key=True)
+    child_column = relationship(
+        "MetaColumns", back_populates="meta_foreign_keys_children", foreign_keys=[child_column_id]
+    )
+
+    constraint_name: str = Column(String, nullable=True)
+
+    def as_string(self) -> str:
+        return f"{self.child_column.name} in {self.child_table.name} references {self.parent_column.name} in {self.parent_table.name}"

mindsdb/migrations/versions/2025-05-28_a44643042fe8_added_data_catalog_tables.py

@@ -0,0 +1,118 @@
+"""added data catalog tables
+
+Revision ID: a44643042fe8
+Revises: 9f150e4f9a05
+Create Date: 2025-05-28 17:20:57.300313
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+import mindsdb.interfaces.storage.db  # noqa
+from mindsdb.interfaces.storage.db import Array
+
+
+# revision identifiers, used by Alembic.
+revision = "a44643042fe8"
+down_revision = "9f150e4f9a05"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "meta_tables",
+        sa.Column("id", sa.Integer(), primary_key=True),
+        sa.Column(
+            "integration_id",
+            sa.Integer(),
+            sa.ForeignKey("integration.id"),
+            nullable=False,
+        ),
+        sa.Column("name", sa.String(), nullable=False),
+        sa.Column("schema", sa.String(), nullable=True),
+        sa.Column("description", sa.String(), nullable=True),
+        sa.Column("type", sa.String(), nullable=True),
+        sa.Column("row_count", sa.Integer(), nullable=True),
+    )
+
+    op.create_table(
+        "meta_columns",
+        sa.Column("id", sa.Integer(), primary_key=True),
+        sa.Column("table_id", sa.Integer(), sa.ForeignKey("meta_tables.id"), nullable=False),
+        sa.Column("name", sa.String(), nullable=False),
+        sa.Column("data_type", sa.String(), nullable=False),
+        sa.Column("default_value", sa.String(), nullable=True),
+        sa.Column("description", sa.String(), nullable=True),
+        sa.Column("is_nullable", sa.Boolean(), nullable=True),
+    )
+
+    op.create_table(
+        "meta_column_statistics",
+        sa.Column(
+            "column_id",
+            sa.Integer(),
+            sa.ForeignKey("meta_columns.id"),
+            primary_key=True,
+        ),
+        sa.Column("most_common_values", Array(), nullable=True),
+        sa.Column("most_common_frequencies", Array(), nullable=True),
+        sa.Column("null_percentage", sa.Numeric(5, 2), nullable=True),
+        sa.Column("distinct_values_count", sa.Integer(), nullable=True),
+        sa.Column("minimum_value", sa.String(), nullable=True),
+        sa.Column("maximum_value", sa.String(), nullable=True),
+    )
+
+    op.create_table(
+        "meta_primary_keys",
+        sa.Column("table_id", sa.Integer(), sa.ForeignKey("meta_tables.id"), primary_key=True),
+        sa.Column(
+            "column_id",
+            sa.Integer(),
+            sa.ForeignKey("meta_columns.id"),
+            primary_key=True,
+        ),
+        sa.Column("ordinal_position", sa.Integer(), nullable=True),
+        sa.Column("constraint_name", sa.String(), nullable=True),
+    )
+
+    op.create_table(
+        "meta_foreign_keys",
+        sa.Column(
+            "parent_table_id",
+            sa.Integer(),
+            sa.ForeignKey("meta_tables.id"),
+            primary_key=True,
+        ),
+        sa.Column(
+            "parent_column_id",
+            sa.Integer(),
+            sa.ForeignKey("meta_columns.id"),
+            primary_key=True,
+        ),
+        sa.Column(
+            "child_table_id",
+            sa.Integer(),
+            sa.ForeignKey("meta_tables.id"),
+            primary_key=True,
+        ),
+        sa.Column(
+            "child_column_id",
+            sa.Integer(),
+            sa.ForeignKey("meta_columns.id"),
+            primary_key=True,
+        ),
+        sa.Column("constraint_name", sa.String(), nullable=True),
+    )
+
+
+def downgrade():
+    op.drop_table("meta_tables")
+
+    op.drop_table("meta_columns")
+
+    op.drop_table("meta_column_statistics")
+
+    op.drop_table("meta_primary_keys")
+
+    op.drop_table("meta_foreign_keys")

mindsdb/migrations/versions/2025-06-09_608e376c19a7_updated_data_catalog_data_types.py

@@ -0,0 +1,58 @@
+"""updated data catalog data types
+
+Revision ID: 608e376c19a7
+Revises: a44643042fe8
+Create Date: 2025-06-09 23:20:34.739735
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+import mindsdb.interfaces.storage.db  # noqa
+
+
+# revision identifiers, used by Alembic.
+revision = "608e376c19a7"
+down_revision = "a44643042fe8"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("meta_tables", schema=None) as batch_op:
+        batch_op.alter_column(
+            "row_count",
+            type_=sa.BigInteger(),
+            existing_type=sa.Integer(),
+            existing_nullable=True,
+            existing_server_default=None,
+        )
+
+    with op.batch_alter_table("meta_column_statistics", schema=None) as batch_op:
+        batch_op.alter_column(
+            "distinct_values_count",
+            type_=sa.BigInteger(),
+            existing_type=sa.Integer(),
+            existing_nullable=True,
+            existing_server_default=None,
+        )
+
+
+def downgrade():
+    with op.batch_alter_table("meta_tables", schema=None) as batch_op:
+        batch_op.alter_column(
+            "row_count",
+            type_=sa.Integer(),
+            existing_type=sa.BigInteger(),
+            existing_nullable=True,
+            existing_server_default=None,
+        )
+
+    with op.batch_alter_table("meta_column_statistics", schema=None) as batch_op:
+        batch_op.alter_column(
+            "distinct_values_count",
+            type_=sa.Integer(),
+            existing_type=sa.BigInteger(),
+            existing_nullable=True,
+            existing_server_default=None,
+        )

mindsdb/utilities/config.py

@@ -217,6 +217,9 @@ class Config:
                 "project_name": "mindsdb",
                 "enabled": False,
             },
+            "data_catalog": {
+                "enabled": False,
+            },
         }
         # endregion
 
@@ -360,6 +363,8 @@ class Config:
             self._env_config["default_reranking_model"] = {
                 "api_key": os.environ["MINDSDB_DEFAULT_RERANKING_MODEL_API_KEY"]
             }
+        if os.environ.get("MINDSDB_DATA_CATALOG_ENABLED", "").lower() in ("1", "true"):
+            self._env_config["data_catalog"] = {"enabled": True}
 
         # region vars: a2a configuration
         a2a_config = {}
@@ -395,11 +400,13 @@ class Config:
             bool: True if config was loaded or updated
         """
 
-        if self.auto_config_mtime != self.auto_config_path.stat().st_mtime:
+        if self.auto_config_path.is_file() and self.auto_config_mtime != self.auto_config_path.stat().st_mtime:
             try:
                 self._auto_config = json.loads(self.auto_config_path.read_text())
             except json.JSONDecodeError as e:
-                raise ValueError(f"The 'auto' configuration file ({self.auto_config_path}) contains invalid JSON: {e}")
+                raise ValueError(
+                    f"The 'auto' configuration file ({self.auto_config_path}) contains invalid JSON: {e}\nFile content: {self.auto_config_path.read_text()}"
+                )
             self.auto_config_mtime = self.auto_config_path.stat().st_mtime
             return True
         return False