MindsDB 25.6.2.0__py3-none-any.whl → 25.6.3.1__py3-none-any.whl

mindsdb/interfaces/knowledge_base/controller.py

@@ -9,6 +9,7 @@ import numpy as np
 
 from mindsdb_sql_parser.ast import BinaryOperation, Constant, Identifier, Select, Update, Delete, Star
 from mindsdb_sql_parser.ast.mindsdb import CreatePredictor
+from mindsdb_sql_parser import parse_sql
 
 from mindsdb.integrations.utilities.query_traversal import query_traversal
 
@@ -52,6 +53,13 @@ def get_model_params(model_params: dict, default_config_key: str):
     """
     Get model parameters by combining default config with user provided parameters.
     """
+    # If the default config key is for reranking and the switch to use the default LLM is enabled,
+    # switch to the default LLM model.
+    if default_config_key == "default_reranking_model" and config.get("default_reranking_model").get(
+        "use_default_llm", False
+    ):
+        default_config_key = "default_llm_model"
+
     combined_model_params = copy.deepcopy(config.get(default_config_key, {}))
 
     if model_params:
@@ -97,6 +105,8 @@ def get_reranking_model_from_params(reranking_model_params: dict):
         params_copy["api_key"] = get_api_key(provider, params_copy, strict=False)
     params_copy["model"] = params_copy.pop("model_name", None)
 
+    params_copy.pop("use_default_llm", None)
+
     return BaseLLMReranker(**params_copy)
 
 
@@ -359,23 +369,30 @@ class KnowledgeBaseTable:
 
     def insert_query_result(self, query: str, project_name: str):
         """Process and insert SQL query results"""
-        if not self.document_loader:
-            raise ValueError("Document loader not configured")
+        ast_query = parse_sql(query)
 
-        documents = list(self.document_loader.load_query_result(query, project_name))
-        if documents:
-            self.insert_documents(documents)
+        command_executor = ExecuteCommands(self.session)
+        response = command_executor.execute_command(ast_query, project_name)
+
+        if response.error_code is not None:
+            raise ValueError(f"Error executing query: {response.error_message}")
+
+        if response.data is None:
+            raise ValueError("Query returned no data")
+
+        records = response.data.records
+        df = pd.DataFrame(records)
+
+        self.insert(df)
 
     def insert_rows(self, rows: List[Dict]):
         """Process and insert raw data rows"""
         if not rows:
             return
 
-        documents = [
-            Document(content=row.get("content", ""), id=row.get("id"), metadata=row.get("metadata", {})) for row in rows
-        ]
+        df = pd.DataFrame(rows)
 
-        self.insert_documents(documents)
+        self.insert(df)
 
     def insert_documents(self, documents: List[Document]):
         """Process and insert documents with preprocessing if configured"""
@@ -1201,22 +1218,10 @@ class KnowledgeBaseController:
         project_names = {i.id: i.name for i in project_controller.get_list()}
 
         for record in query:
-            vector_database = record.vector_database
-            embedding_model = record.embedding_model
+            kb = record.as_dict(with_secrets=self.session.show_secrets)
+            kb["project_name"] = project_names[record.project_id]
 
-            data.append(
-                {
-                    "id": record.id,
-                    "name": record.name,
-                    "project_id": record.project_id,
-                    "project_name": project_names[record.project_id],
-                    "embedding_model": embedding_model.name if embedding_model is not None else None,
-                    "vector_database": None if vector_database is None else vector_database.name,
-                    "vector_database_table": record.vector_database_table,
-                    "query_id": record.query_id,
-                    "params": record.params,
-                }
-            )
+            data.append(kb)
 
         return data
 

mindsdb/interfaces/knowledge_base/evaluate.py

@@ -492,8 +492,6 @@ class EvaluateDocID(EvaluateBase):
         total_questions = len(stats)
         total_found = sum([1 for stat in stats if stat["doc_found"]])
 
-        total_accurately_retrieved = sum([1 for stat in stats if stat["doc_found"]])
-
         accurate_in_top_10 = sum([1 for stat in stats if stat["doc_found"] and stat["doc_position"] < 10])
 
         # calculate recall curve by position
@@ -512,7 +510,6 @@ class EvaluateDocID(EvaluateBase):
         return {
             "total": total_questions,
             "total_found": total_found,
-            "retrieved_in_top_k": total_accurately_retrieved,
             "retrieved_in_top_10": accurate_in_top_10,
             "cumulative_recall": cumulative_recall,
             "avg_query_time": avg_query_time,

mindsdb/interfaces/knowledge_base/preprocessing/document_loader.py

@@ -2,7 +2,6 @@ import os
 from typing import List, Iterator
 from langchain_core.documents import Document as LangchainDocument
 from langchain_text_splitters import MarkdownHeaderTextSplitter
-import pandas as pd
 
 from mindsdb.interfaces.file.file_controller import FileController
 from mindsdb.integrations.utilities.rag.loaders.file_loader import FileLoader
@@ -20,12 +19,12 @@ class DocumentLoader:
     """Handles loading documents from various sources including SQL queries"""
 
     def __init__(
-            self,
-            file_controller: FileController,
-            file_splitter: FileSplitter,
-            markdown_splitter: MarkdownHeaderTextSplitter,
-            file_loader_class=FileLoader,
-            mysql_proxy=None
+        self,
+        file_controller: FileController,
+        file_splitter: FileSplitter,
+        markdown_splitter: MarkdownHeaderTextSplitter,
+        file_loader_class=FileLoader,
+        mysql_proxy=None,
     ):
         """
         Initialize with required dependencies
@@ -52,8 +51,8 @@ class DocumentLoader:
             for doc in loader.lazy_load():
                 # Add file extension to metadata for proper splitting
                 extension = os.path.splitext(file_path)[1].lower()
-                doc.metadata['extension'] = extension
-                doc.metadata['source'] = file_name
+                doc.metadata["extension"] = extension
+                doc.metadata["source"] = file_name
 
                 # Use FileSplitter to handle the document based on its type
                 split_docs = self.file_splitter.split_documents([doc])
@@ -62,34 +61,22 @@ class DocumentLoader:
                     metadata = doc.metadata.copy()
                     metadata.update(split_doc.metadata or {})
 
-                    yield Document(
-                        content=split_doc.page_content,
-                        metadata=metadata
-                    )
+                    yield Document(content=split_doc.page_content, metadata=metadata)
 
     def load_web_pages(
-            self,
-            urls: List[str],
-            crawl_depth: int,
-            limit: int,
-            filters: List[str] = None,
+        self,
+        urls: List[str],
+        crawl_depth: int,
+        limit: int,
+        filters: List[str] = None,
     ) -> Iterator[Document]:
         """Load and split documents from web pages"""
-        websites_df = get_all_websites(
-            urls,
-            crawl_depth=crawl_depth,
-            limit=limit,
-            filters=filters
-        )
+        websites_df = get_all_websites(urls, crawl_depth=crawl_depth, limit=limit, filters=filters)
 
         for _, row in websites_df.iterrows():
             # Create a document with HTML extension for proper splitting
             doc = LangchainDocument(
-                page_content=row['text_content'],
-                metadata={
-                    'extension': '.html',
-                    'url': row['url']
-                }
+                page_content=row["text_content"], metadata={"extension": ".html", "url": row["url"]}
             )
 
             # Use FileSplitter to handle HTML content
@@ -98,60 +85,4 @@ class DocumentLoader:
                 metadata = doc.metadata.copy()
                 metadata.update(split_doc.metadata or {})
 
-                yield Document(
-                    content=split_doc.page_content,
-                    metadata=metadata
-                )
-
-    def load_query_result(self, query: str, project_name: str) -> Iterator[Document]:
-        """
-        Load documents from SQL query results
-
-        Args:
-            query: SQL query to execute
-            project_name: Name of the project context
-
-        Returns:
-            Iterator of Document objects
-
-        Raises:
-            ValueError: If mysql_proxy is not configured or query returns no data
-        """
-        if not self.mysql_proxy:
-            raise ValueError("MySQL proxy not configured")
-
-        if not query:
-            return
-
-        # Set project context and execute query
-        self.mysql_proxy.set_context({'db': project_name})
-        query_result = self.mysql_proxy.process_query(query)
-
-        if query_result.type != 'table':
-            raise ValueError('Query returned no data')
-
-        # Convert query result to DataFrame
-        df = query_result.data.to_df()
-
-        # Process each row into a Document
-        for _, row in df.iterrows():
-            # Extract id, content  and metadata
-            content = str(row.get('content', ''))
-            id = row.get('id', None)
-
-            # Convert remaining columns to metadata
-            metadata = {
-                col: str(row[col])
-                for col in df.columns
-                if col != 'content' and not pd.isna(row[col])
-            }
-            metadata['source'] = 'query'
-
-            # Split content using recursive splitter
-            if content:
-
-                yield Document(
-                    id=id,
-                    content=content,
-                    metadata=metadata
-                )
+                yield Document(content=split_doc.page_content, metadata=metadata)

mindsdb/interfaces/skills/custom/text2sql/mindsdb_kb_tools.py

@@ -3,6 +3,7 @@ import re
 import json
 from pydantic import BaseModel, Field
 from langchain_core.tools import BaseTool
+from mindsdb_sql_parser.ast import Describe, Select, Identifier, Constant, Star
 
 
 class KnowledgeBaseListToolInput(BaseModel):
@@ -55,6 +56,26 @@ class KnowledgeBaseInfoTool(BaseTool):
         except (json.JSONDecodeError, TypeError):
             pass
 
+        def strip(s):
+            length = -1
+            while length != len(s):
+                length = len(s)
+
+                # remove ```
+                if s.startswith("```"):
+                    s = s[3:]
+                if s.endswith("```"):
+                    s = s[:-3]
+
+                # remove trailing new lines
+                s = s.strip("\n")
+
+                # remove extra quotes
+                for q in ('"', "'", "`"):
+                    if s.count(q) == 1:
+                        s = s.strip(q)
+            return s
+
         # Finally, try the original regex pattern for $START$ and $STOP$ markers
         match = re.search(r"\$START\$(.*?)\$STOP\$", tool_input, re.DOTALL)
         if not match:
@@ -63,12 +84,14 @@ class KnowledgeBaseInfoTool(BaseTool):
                 return [kb.strip() for kb in tool_input.split(",")]
             # If it's just a single string without formatting, return it as a single item
             if tool_input.strip():
-                return [tool_input.strip()]
+                return [strip(tool_input)]
             return []
 
         # Extract and clean the knowledge base names
         kb_names_str = match.group(1).strip()
         kb_names = re.findall(r"`([^`]+)`", kb_names_str)
+
+        kb_names = [strip(n) for n in kb_names]
         return kb_names
 
     def _run(self, tool_input: str) -> str:
@@ -83,7 +106,7 @@ class KnowledgeBaseInfoTool(BaseTool):
         for kb_name in kb_names:
             try:
                 # Get knowledge base schema
-                schema_result = self.db.run_no_throw(f"DESCRIBE KNOWLEDGE_BASE `{kb_name}`;")
+                schema_result = self.db.run_no_throw(str(Describe(kb_name, type="knowledge_base")))
 
                 if not schema_result:
                     results.append(f"Knowledge base `{kb_name}` not found or has no schema information.")
@@ -111,7 +134,9 @@ class KnowledgeBaseInfoTool(BaseTool):
                 kb_info += "```\n\n"
 
                 # Get sample data
-                sample_data = self.db.run_no_throw(f"SELECT * FROM `{kb_name}` LIMIT 10;")
+                sample_data = self.db.run_no_throw(
+                    str(Select(targets=[Star()], from_table=Identifier(kb_name), limit=Constant(20)))
+                )
 
                 # Sample data
                 kb_info += "### Sample Data:\n"

mindsdb/interfaces/skills/skills_controller.py

@@ -6,8 +6,6 @@ from sqlalchemy.orm.attributes import flag_modified
 
 from mindsdb.interfaces.storage import db
 from mindsdb.interfaces.database.projects import ProjectController
-from mindsdb.interfaces.data_catalog.data_catalog_loader import DataCatalogLoader
-from mindsdb.interfaces.skills.skill_tool import SkillType
 from mindsdb.utilities.config import config
 from mindsdb.utilities import log
 
@@ -99,27 +97,6 @@ class SkillsController:
         if skill is not None:
             raise ValueError(f"Skill with name already exists: {name}")
 
-        # Load metadata to data catalog (if enabled) if the skill is Text-to-SQL.
-        if config.get("data_catalog", {}).get("enabled", False):
-            if type == SkillType.TEXT2SQL.value and "include_tables" in params:
-                # TODO: Is it possible to create a skill with complete access to the database with the new agent syntax?
-                # TODO: Handle the case where `ignore_tables` is provided. Is this a valid parameter?
-                # TODO: Knowledge Bases?
-                database_table_map = {}
-                for table in params["include_tables"]:
-                    parts = table.split(".", 1)
-                    database_table_map[parts[0]] = database_table_map.get(parts[0], []) + [parts[1]]
-
-                for database_name, table_names in database_table_map.items():
-                    data_catalog_loader = DataCatalogLoader(database_name=database_name, table_names=table_names)
-                    data_catalog_loader.load_metadata()
-
-            elif type in [SkillType.TEXT2SQL.value, SkillType.TEXT2SQL_LEGACY.value] and "database" in params:
-                data_catalog_loader = DataCatalogLoader(
-                    database_name=params["database"], table_names=params["tables"] if "tables" in params else None
-                )
-                data_catalog_loader.load_metadata()
-
         new_skill = db.Skills(
             name=name,
             project_id=project.id,

mindsdb/interfaces/skills/sql_agent.py

@@ -76,7 +76,7 @@ def split_table_name(table_name: str) -> List[str]:
         result.append(current.strip("`"))
 
     # ensure we split the table name
-    result = [r.split(".") for r in result][0]
+    # result = [r.split(".") for r in result][0]
 
     return result
 
@@ -402,11 +402,15 @@ class SQLAgent:
         """
         if config.get("data_catalog", {}).get("enabled", False):
             database_table_map = {}
-            for name in self.get_usable_table_names():
+            for name in table_names or self.get_usable_table_names():
                 name = name.replace("`", "")
 
-                # TODO: Can there be situations where the database name is returned from the above method?
                 parts = name.split(".", 1)
+                # TODO: Will there be situations where parts has more than 2 elements? Like a schema?
+                # This is unlikely given that we default to a single schema per database.
+                if len(parts) == 1:
+                    raise ValueError(f"Invalid table name: {name}. Expected format is 'database.table'.")
+
                 database_table_map[parts[0]] = database_table_map.get(parts[0], []) + [parts[1]]
 
             data_catalog_str = ""
@@ -430,8 +434,8 @@ class SQLAgent:
                 else:
                     all_tables.append(Identifier(name))
 
-            # if table_names is not None:
-            #     all_tables = self._resolve_table_names(table_names, all_tables)
+            if table_names is not None:
+                all_tables = self._resolve_table_names(table_names, all_tables)
 
             tables_info = []
             for table in all_tables:

mindsdb/interfaces/storage/db.py

@@ -1,6 +1,6 @@
 import json
 import datetime
-from typing import Dict, List
+from typing import Dict, List, Optional
 
 import numpy as np
 from sqlalchemy import (
@@ -494,17 +494,33 @@ class KnowledgeBase(Base):
 
     __table_args__ = (UniqueConstraint("name", "project_id", name="unique_knowledge_base_name_project_id"),)
 
-    def as_dict(self) -> Dict:
+    def as_dict(self, with_secrets: Optional[bool] = True) -> Dict:
+        params = self.params.copy()
+        embedding_model = params.pop("embedding_model", None)
+        reranking_model = params.pop("reranking_model", None)
+
+        if not with_secrets:
+            if embedding_model and "api_key" in embedding_model:
+                embedding_model["api_key"] = "******"
+
+            if reranking_model and "api_key" in reranking_model:
+                reranking_model["api_key"] = "******"
+
         return {
             "id": self.id,
             "name": self.name,
             "project_id": self.project_id,
-            "embedding_model": None if self.embedding_model is None else self.embedding_model.name,
             "vector_database": None if self.vector_database is None else self.vector_database.name,
             "vector_database_table": self.vector_database_table,
             "updated_at": self.updated_at,
             "created_at": self.created_at,
-            "params": self.params,
+            "query_id": self.query_id,
+            "embedding_model": embedding_model,
+            "reranking_model": reranking_model,
+            "metadata_columns": params.pop("metadata_columns", None),
+            "content_columns": params.pop("content_columns", None),
+            "id_column": params.pop("id_column", None),
+            "params": params,
         }
 
 

mindsdb/utilities/config.py

@@ -400,7 +400,11 @@ class Config:
             bool: True if config was loaded or updated
         """
 
-        if self.auto_config_path.is_file() and self.auto_config_mtime != self.auto_config_path.stat().st_mtime:
+        if (
+            self.auto_config_path.is_file()
+            and self.auto_config_path.read_text() != ""
+            and self.auto_config_mtime != self.auto_config_path.stat().st_mtime
+        ):
             try:
                 self._auto_config = json.loads(self.auto_config_path.read_text())
             except json.JSONDecodeError as e: