MindsDB 25.9.2.0a1__py3-none-any.whl → 25.10.0rc1__py3-none-any.whl

mindsdb/interfaces/skills/custom/text2sql/mindsdb_sql_toolkit.py

@@ -55,7 +55,7 @@ class MindsDBSQLToolkit(SQLDatabaseToolkit):
             f"""\
             Input: A detailed and well-structured SQL query. The query must be enclosed between the symbols $START$ and $STOP$.
             Output: Database result or error message. For errors, rewrite and retry the query. For 'Unknown column' errors, use '{info_sql_database_tool.name}' to check table fields.
-            This system is a highly intelligent and reliable PostgreSQL SQL skill designed to work with databases.
+            This system is a highly intelligent and reliable SQL skill designed to work with databases.
             Follow these instructions with utmost precision:
             1. Final Response Format:
                - Assume the frontend fully supports Markdown unless the user specifies otherwise.
@@ -73,7 +73,7 @@ class MindsDBSQLToolkit(SQLDatabaseToolkit):
                - Let the user know they can request additional results and/or specify how they would like the results ordered or grouped.
             5. Date Handling:
                - **System current date and time: {current_date_time} (UTC or local timezone based on server settings).**
-               - **Always** use PostgreSQL-compatible `CURRENT_DATE` or `NOW()` functions when working with dates—never assume or guess the current date.
+               - **Always** use `CURRENT_DATE` or `NOW()` functions when working with dates—never assume or guess the current date.
                - For any date-related comparisons in the query, *always* ensure that your query casts the column being compared using `column_name::DATE [operator] ..`
                - Do not compare date values without casting columns to date.
                - For date interval operations, use Interval units as keywords. You can use keywords to specify units like days, hours, months, years, etc., directly without quotes. Examples:
@@ -95,6 +95,8 @@ class MindsDBSQLToolkit(SQLDatabaseToolkit):
             8. Identity and Purpose:
                - When asked about yourself or your maker, state that you are a Data-Mind, created by MindsDB to help answer data questions.
                - When asked about your purpose or how you can help, explore the available data sources and then explain that you can answer questions based on the connected data. Provide a few relevant example questions that you could answer for the user about their data.
+            9. Important: you can use only mysql quoting rules to compose queries: backticks (`) for identifiers, and single quotes (') for constants
+
             Adhere to these guidelines for all queries and responses. Ask for clarification if needed.
         """
         )
@@ -110,7 +112,7 @@ class MindsDBSQLToolkit(SQLDatabaseToolkit):
             "If the query is correct, it will be parsed and returned. "
             f"ALWAYS run this tool before executing a query with {query_sql_database_tool.name}. "
         )
-        mindsdb_sql_parser_tool = MindsDBSQLParserTool(
+        mindsdb_sql_parser_tool = MindsDBSQLParserTool(  # noqa: F841
             name=f"mindsdb_sql_parser_tool{prefix}", description=mindsdb_sql_parser_tool_description
         )
 
@@ -118,7 +120,7 @@ class MindsDBSQLToolkit(SQLDatabaseToolkit):
             query_sql_database_tool,
             info_sql_database_tool,
             list_sql_database_tool,
-            mindsdb_sql_parser_tool,
+            # mindsdb_sql_parser_tool,
         ]
         if not self.include_knowledge_base_tools:
             return sql_tools
@@ -175,29 +177,23 @@ class MindsDBSQLToolkit(SQLDatabaseToolkit):
 
                 Query Types and Examples:
                 1. Basic semantic search:
-                   kb_query_tool("SELECT * FROM kb_name WHERE content = 'your search query';")
+                   kb_query_tool("SELECT * FROM kb_name WHERE chunk_content = 'your search query';")
 
                 2. Metadata filtering:
                    kb_query_tool("SELECT * FROM kb_name WHERE metadata_field = 'value';")
 
                 3. Combined search:
-                   kb_query_tool("SELECT * FROM kb_name WHERE content = 'query' AND metadata_field = 'value';")
+                   kb_query_tool("SELECT * FROM kb_name WHERE chunk_content = 'query' AND metadata_field = 'value';")
 
                 4. Setting relevance threshold:
-                   kb_query_tool("SELECT * FROM kb_name WHERE content = 'query' AND relevance_threshold = 0.7;")
+                   kb_query_tool("SELECT * FROM kb_name WHERE chunk_content = 'query' AND relevance_threshold = 0.7;")
 
                 5. Limiting results:
-                   kb_query_tool("SELECT * FROM kb_name WHERE content = 'query' LIMIT 5;")
+                   kb_query_tool("SELECT * FROM kb_name WHERE chunk_content = 'query' LIMIT 5;")
 
                 6. Getting sample data:
                    kb_query_tool("SELECT * FROM kb_name LIMIT 3;")
 
-                7. Don't use LIKE operator on content filter ie semantic search:
-                SELECT * FROM `test_kb` WHERE content LIKE '%population of New York%' $STOP$
-
-                Like is not supported, use the following instead:
-                SELECT * FROM `test_kb` WHERE content = 'population of New York'
-
                 Result Format:
                 - Results include: id, chunk_id, chunk_content, metadata, distance, and relevance columns
                 - The metadata column contains a JSON object with all metadata fields

mindsdb/interfaces/skills/retrieval_tool.py

@@ -1,32 +1,32 @@
-import traceback
+from langchain_core.documents import Document
+from langchain_core.tools import Tool
 
 from mindsdb.integrations.utilities.rag.rag_pipeline_builder import RAG
 from mindsdb.integrations.utilities.rag.config_loader import load_rag_config
 from mindsdb.integrations.utilities.rag.settings import RAGPipelineModel
 from mindsdb.integrations.utilities.sql_utils import FilterCondition, FilterOperator
-
+from mindsdb.integrations.libs.response import RESPONSE_TYPE
+from mindsdb.integrations.handlers.langchain_embedding_handler.langchain_embedding_handler import (
+    construct_model_from_args,
+)
 from mindsdb.interfaces.agents.constants import DEFAULT_EMBEDDINGS_MODEL_CLASS
 from mindsdb.interfaces.skills.skill_tool import skill_tool
 from mindsdb.interfaces.storage import db
 from mindsdb.interfaces.storage.db import KnowledgeBase
 from mindsdb.utilities import log
-from langchain_core.documents import Document
-from langchain_core.tools import Tool
-from mindsdb.integrations.libs.response import RESPONSE_TYPE
-from mindsdb.integrations.handlers.langchain_embedding_handler.langchain_embedding_handler import construct_model_from_args
 
 logger = log.getLogger(__name__)
 
 
 def _load_rag_config(tool: dict, pred_args: dict, skill: db.Skills) -> RAGPipelineModel:
-    tools_config = tool['config']
+    tools_config = tool["config"]
     tools_config.update(pred_args)
 
     kb_params = {}
     embeddings_model = None
 
-    if 'source' in tool:
-        kb_name = tool['source']
+    if "source" in tool:
+        kb_name = tool["source"]
         executor = skill_tool.get_command_executor()
         kb = _get_knowledge_base(kb_name, skill.project_id, executor)
 
@@ -34,30 +34,26 @@ def _load_rag_config(tool: dict, pred_args: dict, skill: db.Skills) -> RAGPipeli
             raise ValueError(f"Knowledge base not found: {kb_name}")
 
         kb_table = executor.session.kb_controller.get_table(kb.name, kb.project_id)
-        vector_store_config = {
-            'kb_table': kb_table
-        }
-        is_sparse = tools_config.pop('is_sparse', None)
-        vector_size = tools_config.pop('vector_size', None)
+        vector_store_config = {"kb_table": kb_table}
+        is_sparse = tools_config.pop("is_sparse", None)
+        vector_size = tools_config.pop("vector_size", None)
         if is_sparse is not None:
-            vector_store_config['is_sparse'] = is_sparse
+            vector_store_config["is_sparse"] = is_sparse
         if vector_size is not None:
-            vector_store_config['vector_size'] = vector_size
-        kb_params = {
-            'vector_store_config': vector_store_config
-        }
+            vector_store_config["vector_size"] = vector_size
+        kb_params = {"vector_store_config": vector_store_config}
 
         # Get embedding model from knowledge base table
         if kb_table._kb.embedding_model:
             # Extract embedding model args from knowledge base table
-            embedding_args = kb_table._kb.embedding_model.learn_args.get('using', {})
+            embedding_args = kb_table._kb.embedding_model.learn_args.get("using", {})
             # Construct the embedding model directly
             embeddings_model = construct_model_from_args(embedding_args)
             logger.debug(f"Using knowledge base embedding model with args: {embedding_args}")
         else:
             embeddings_model = DEFAULT_EMBEDDINGS_MODEL_CLASS()
             logger.debug("Using default embedding model as knowledge base has no embedding model")
-    elif 'embedding_model' not in tools_config:
+    elif "embedding_model" not in tools_config:
         embeddings_model = DEFAULT_EMBEDDINGS_MODEL_CLASS()
         logger.debug("Using default embedding model as no knowledge base provided")
 
@@ -75,29 +71,28 @@ def _build_rag_pipeline_tool(tool: dict, pred_args: dict, skill: db.Skills):
         try:
             result = rag_pipeline(query)
             logger.debug(f"RAG pipeline result: {result}")
-            return result['answer']
+            return result["answer"]
         except Exception as e:
-            logger.error(f"Error in RAG pipeline: {str(e)}")
-            logger.error(traceback.format_exc())
+            logger.exception("Error in RAG pipeline:")
             return f"Error in retrieval: {str(e)}"
 
     # Create RAG tool
-    tools_config = tool['config']
+    tools_config = tool["config"]
     tools_config.update(pred_args)
     return Tool(
         func=rag_wrapper,
-        name=tool['name'],
-        description=tool['description'],
-        response_format='content',
+        name=tool["name"],
+        description=tool["description"],
+        response_format="content",
         # Return directly by default since we already use an LLM against retrieved context to generate a response.
-        return_direct=tools_config.get('return_direct', True)
+        return_direct=tools_config.get("return_direct", True),
     )
 
 
 def _build_name_lookup_tool(tool: dict, pred_args: dict, skill: db.Skills):
-    if 'source' not in tool:
+    if "source" not in tool:
         raise ValueError("Knowledge base for tool not found")
-    kb_name = tool['source']
+    kb_name = tool["source"]
     executor = skill_tool.get_command_executor()
     kb = _get_knowledge_base(kb_name, skill.project_id, executor)
     if not kb:
@@ -110,16 +105,16 @@ def _build_name_lookup_tool(tool: dict, pred_args: dict, skill: db.Skills):
 
     def _get_document_by_name(name: str):
         if metadata_config.name_column_index is not None:
-            tsquery_str = ' & '.join(name.split(' '))
+            tsquery_str = " & ".join(name.split(" "))
             documents_response = vector_db_handler.native_query(
-                f'SELECT * FROM {metadata_config.table} WHERE {metadata_config.name_column_index} @@ to_tsquery(\'{tsquery_str}\') LIMIT 1;'
+                f"SELECT * FROM {metadata_config.table} WHERE {metadata_config.name_column_index} @@ to_tsquery('{tsquery_str}') LIMIT 1;"
             )
         else:
             documents_response = vector_db_handler.native_query(
-                f'SELECT * FROM {metadata_config.table} WHERE "{metadata_config.name_column}" ILIKE \'%{name}%\' LIMIT 1;'
+                f"SELECT * FROM {metadata_config.table} WHERE \"{metadata_config.name_column}\" ILIKE '%{name}%' LIMIT 1;"
             )
         if documents_response.resp_type == RESPONSE_TYPE.ERROR:
-            raise RuntimeError(f'There was an error looking up documents: {documents_response.error_message}')
+            raise RuntimeError(f"There was an error looking up documents: {documents_response.error_message}")
         if documents_response.data_frame.empty:
             return None
         document_row = documents_response.data_frame.head(1)
@@ -127,38 +122,36 @@ def _build_name_lookup_tool(tool: dict, pred_args: dict, skill: db.Skills):
         id_filter_condition = FilterCondition(
             f"{metadata_config.embeddings_metadata_column}->>'{metadata_config.doc_id_key}'",
             FilterOperator.EQUAL,
-            str(document_row.get(metadata_config.id_column).item())
+            str(document_row.get(metadata_config.id_column).item()),
         )
         document_chunks_df = vector_db_handler.select(
-            metadata_config.embeddings_table,
-            conditions=[id_filter_condition]
+            metadata_config.embeddings_table, conditions=[id_filter_condition]
         )
         if document_chunks_df.empty:
             return None
-        sort_col = 'chunk_id' if 'chunk_id' in document_chunks_df.columns else 'id'
+        sort_col = "chunk_id" if "chunk_id" in document_chunks_df.columns else "id"
         document_chunks_df.sort_values(by=sort_col)
-        content = ''
+        content = ""
         for _, chunk in document_chunks_df.iterrows():
             if len(content) > metadata_config.max_document_context:
                 break
-            content += chunk.get(metadata_config.content_column, '')
+            content += chunk.get(metadata_config.content_column, "")
 
-        return Document(
-            page_content=content,
-            metadata=document_row.to_dict(orient='records')[0]
-        )
+        return Document(page_content=content, metadata=document_row.to_dict(orient="records")[0])
 
     def _lookup_document_by_name(name: str):
         found_document = _get_document_by_name(name)
         if found_document is None:
-            return f'I could not find any document with name {name}. Please make sure the document name matches exactly.'
+            return (
+                f"I could not find any document with name {name}. Please make sure the document name matches exactly."
+            )
         return f"I found document {found_document.metadata.get(metadata_config.id_column)} with name {found_document.metadata.get(metadata_config.name_column)}. Here is the full document to use as context:\n\n{found_document.page_content}"
 
     return Tool(
         func=_lookup_document_by_name,
-        name=tool.get('name', '') + '_name_lookup',
-        description='You must use this tool ONLY when the user is asking about a specific document by name or title. The input should be the exact name of the document the user is looking for.',
-        return_direct=False
+        name=tool.get("name", "") + "_name_lookup",
+        description="You must use this tool ONLY when the user is asking about a specific document by name or title. The input should be the exact name of the document the user is looking for.",
+        return_direct=False,
     )
 
 
@@ -181,7 +174,7 @@ def build_retrieval_tools(tool: dict, pred_args: dict, skill: db.Skills):
     try:
         rag_config = _load_rag_config(tool, pred_args, skill)
     except Exception as e:
-        logger.error(f"Error building RAG pipeline: {str(e)}")
+        logger.exception("Error building RAG pipeline:")
         raise ValueError(f"Failed to build RAG pipeline: {str(e)}")
     tools = [_build_rag_pipeline_tool(tool, pred_args, skill)]
     if rag_config.metadata_config is None:

mindsdb/interfaces/skills/skill_tool.py

@@ -274,8 +274,8 @@ class SkillToolController:
                     else:
                         for table_name in response.data_frame.iloc[:, name_idx]:
                             tables_list.append(f"{database}.{escape_table_name(table_name)}")
-                except Exception as e:
-                    logger.warning(f"Could not get tables from database {database}: {str(e)}")
+                except Exception:
+                    logger.warning(f"Could not get tables from database {database}:", exc_info=True)
                 continue
 
             # Handle table restrictions

mindsdb/interfaces/skills/skills_controller.py

@@ -100,10 +100,7 @@ class SkillsController:
             project_name = default_project
         project = self.project_controller.get(name=project_name)
 
-        if not name.islower():
-            raise ValueError(f"The name must be in lower case: {name}")
-
-        skill = self.get_skill(name, project_name)
+        skill = self.get_skill(name, project_name, strict_case=True)
 
         if skill is not None:
             raise ValueError(f"Skill with name already exists: {name}")

mindsdb/interfaces/skills/sql_agent.py

@@ -1,7 +1,6 @@
 import re
 import csv
 import inspect
-import traceback
 from io import StringIO
 from typing import Iterable, List, Optional, Any, Tuple
 from collections import defaultdict
@@ -254,8 +253,16 @@ class SQLAgent:
                             self.check_table_permission(node)
                         except ValueError as origin_exc:
                             # was it badly quoted by llm?
-                            if len(node.parts) == 1 and node.is_quoted[0] and "." in node.parts[0]:
-                                node2 = Identifier(node.parts[0])
+                            #
+                            if "." in node.parts[0]:
+                                # extract quoted parts (with dots) to sub-parts
+                                parts = []
+                                for i, item in enumerate(node.parts):
+                                    if node.is_quoted[i] and "." in item:
+                                        parts.extend(Identifier(item).parts)
+                                    else:
+                                        parts.append(item)
+                                node2 = Identifier(parts=parts)
                                 try:
                                     _check_f(node2, is_table=True)
                                     return node2
@@ -382,9 +389,9 @@ class SQLAgent:
             #     self._cache.set(cache_key, set(kb_names))
 
             return kb_names
-        except Exception as e:
+        except Exception:
             # If there's an error, log it and return an empty list
-            logger.error(f"Error in get_usable_knowledge_base_names: {str(e)}")
+            logger.exception("Error in get_usable_knowledge_base_names")
             return []
 
     def _resolve_table_names(self, table_names: List[str], all_tables: List[Identifier]) -> List[Identifier]:
@@ -483,9 +490,9 @@ class SQLAgent:
                 # remove backticks
                 name = name.replace("`", "")
 
-                split = name.split(".")
-                if len(split) > 1:
-                    all_tables.append(Identifier(parts=[split[0], split[-1]]))
+                parts = name.split(".")
+                if len(parts) > 1:
+                    all_tables.append(Identifier(parts=parts))
                 else:
                     all_tables.append(Identifier(name))
 
@@ -526,8 +533,8 @@ class SQLAgent:
 
             sample_rows = list(map(lambda row: [truncate_value(value) for value in row], sample_rows))
             sample_rows_str = "\n" + f"{kb_name}:" + list_to_csv_str(sample_rows)
-        except Exception as e:
-            logger.info(f"_get_sample_rows error: {e}")
+        except Exception:
+            logger.info("_get_sample_rows error:", exc_info=True)
             sample_rows_str = "\n" + "\t [error] Couldn't retrieve sample rows!"
 
         return sample_rows_str
@@ -560,7 +567,7 @@ class SQLAgent:
                 )
             ]
         except Exception as e:
-            logger.error(f"Failed processing column info for {table_str}: {e}", exc_info=True)
+            logger.exception(f"Failed processing column info for {table_str}:")
             raise ValueError(f"Failed to process column info for {table_str}") from e
 
         if not fields:
@@ -569,8 +576,8 @@ class SQLAgent:
 
         try:
             sample_rows_info = self._get_sample_rows(table_str, fields)
-        except Exception as e:
-            logger.warning(f"Could not get sample rows for {table_str}: {e}")
+        except Exception:
+            logger.warning(f"Could not get sample rows for {table_str}:", exc_info=True)
             sample_rows_info = "\n\t [error] Couldn't retrieve sample rows!"
 
         info = f"Table named `{table_str}`:\n"
@@ -585,7 +592,7 @@ class SQLAgent:
 
     def _get_sample_rows(self, table: str, fields: List[str]) -> str:
         logger.info(f"_get_sample_rows: table={table} fields={fields}")
-        command = f"select {', '.join(fields)} from {table} limit {self._sample_rows_in_table_info};"
+        command = f"select * from {table} limit {self._sample_rows_in_table_info};"
         try:
             ret = self._call_engine(command)
             sample_rows = ret.data.to_lists()
@@ -596,8 +603,8 @@ class SQLAgent:
 
             sample_rows = list(map(lambda row: [truncate_value(value) for value in row], sample_rows))
             sample_rows_str = "\n" + list_to_csv_str([fields] + sample_rows)
-        except Exception as e:
-            logger.info(f"_get_sample_rows error: {e}")
+        except Exception:
+            logger.info("_get_sample_rows error:", exc_info=True)
             sample_rows_str = "\n" + "\t [error] Couldn't retrieve sample rows!"
 
         return sample_rows_str
@@ -647,7 +654,7 @@ class SQLAgent:
             logger.info(f"get_table_info_safe: {table_names}")
             return self.get_table_info(table_names)
         except Exception as e:
-            logger.info(f"get_table_info_safe error: {e}")
+            logger.info("get_table_info_safe error:", exc_info=True)
             return f"Error: {e}"
 
     def query_safe(self, command: str, fetch: str = "all") -> str:
@@ -655,8 +662,7 @@ class SQLAgent:
             logger.info(f"query_safe (fetch={fetch}): {command}")
             return self.query(command, fetch)
         except Exception as e:
-            logger.error(f"Error in query_safe: {str(e)}\n{traceback.format_exc()}")
-            logger.info(f"query_safe error: {e}")
+            logger.exception("Error in query_safe:")
             msg = f"Error: {e}"
             if "does not exist" in msg and " relation " in msg:
                 msg += "\nAvailable tables: " + ", ".join(self.get_usable_table_names())

mindsdb/interfaces/storage/db.py

@@ -1,4 +1,5 @@
 import json
+import orjson
 import datetime
 from typing import Dict, List, Optional
 
@@ -47,10 +48,20 @@ def init(connection_str: str = None):
     global Base, session, engine
     if connection_str is None:
         connection_str = config["storage_db"]
+    # Use orjson with our CustomJSONEncoder.default for JSON serialization
+    _default_json = CustomJSONEncoder().default
+
+    def _json_serializer(value):
+        return orjson.dumps(
+            value,
+            default=_default_json,
+            option=orjson.OPT_SERIALIZE_NUMPY | orjson.OPT_PASSTHROUGH_DATETIME,
+        ).decode("utf-8")
+
     base_args = {
         "pool_size": 30,
         "max_overflow": 200,
-        "json_serializer": CustomJSONEncoder().encode,
+        "json_serializer": _json_serializer,
     }
     engine = create_engine(connection_str, echo=False, **base_args)
     session = scoped_session(sessionmaker(bind=engine, autoflush=True))
@@ -534,11 +545,10 @@ class KnowledgeBase(Base):
         reranking_model = params.pop("reranking_model", None)
 
         if not with_secrets:
-            if embedding_model and "api_key" in embedding_model:
-                embedding_model["api_key"] = "******"
-
-            if reranking_model and "api_key" in reranking_model:
-                reranking_model["api_key"] = "******"
+            for key in ("api_key", "private_key"):
+                for el in (embedding_model, reranking_model):
+                    if el and key in el:
+                        el[key] = "******"
 
         return {
             "id": self.id,