MindsDB 25.4.5.0__py3-none-any.whl → 25.5.3.0__py3-none-any.whl

mindsdb/interfaces/database/projects.py

@@ -4,7 +4,6 @@ from typing import List, Optional
 from collections import OrderedDict
 
 import sqlalchemy as sa
-from sqlalchemy.orm.attributes import flag_modified
 import numpy as np
 
 from mindsdb_sql_parser.ast.base import ASTNode
@@ -457,7 +456,7 @@ class ProjectController:
         project.create(name=name)
         return project
 
-    def update(self, id: Optional[int] = None, name: Optional[str] = None, new_name: str = None, new_metadata: dict = None) -> Project:
+    def update(self, id: Optional[int] = None, name: Optional[str] = None, new_name: str = None) -> Project:
         if id is not None and name is not None:
             raise ValueError("Both 'id' and 'name' can't be provided at the same time")
 
@@ -470,10 +469,5 @@ class ProjectController:
             project.name = new_name
             project.record.name = new_name
 
-        if new_metadata is not None:
-            project.metadata = new_metadata
-            project.record.metadata_ = new_metadata
-            flag_modified(project.record, 'metadata_')
-
         db.session.commit()
         return project

mindsdb/interfaces/functions/controller.py

@@ -1,7 +1,7 @@
 import os
+import copy
 
 from duckdb.typing import BIGINT, DOUBLE, VARCHAR, BLOB, BOOLEAN
-from mindsdb.interfaces.functions.to_markdown import ToMarkdown
 from mindsdb.interfaces.storage.model_fs import HandlerStorage
 from mindsdb.utilities.config import config
 
@@ -159,31 +159,28 @@ class FunctionController(BYOMFunctionsController):
         return meta
 
     def to_markdown_call_function(self, node):
+        # load on-demand because lib is heavy
+        from mindsdb.interfaces.functions.to_markdown import ToMarkdown
         name = node.op.lower()
 
         if name in self.callbacks:
             return self.callbacks[name]
 
-        def callback(file_path_or_url, use_llm):
+        def callback(file_path_or_url):
             chat_model_params = self._parse_chat_model_params('TO_MARKDOWN_FUNCTION_')
 
-            llm_client = None
-            llm_model = None
-            try:
-                from mindsdb.interfaces.agents.langchain_agent import create_chat_model
-                llm = create_chat_model(chat_model_params)
-                llm_client = llm.root_client
-                llm_model = llm.model_name
-            except Exception:
-                pass
+            params_copy = copy.deepcopy(chat_model_params)
+            params_copy['model'] = params_copy.pop('model_name')
+            params_copy.pop('api_keys')
+            params_copy.pop('provider')
 
-            to_markdown = ToMarkdown(use_llm, llm_client, llm_model)
-            return to_markdown.call(file_path_or_url)
+            to_markdown = ToMarkdown()
+            return to_markdown.call(file_path_or_url, **params_copy)
 
         meta = {
             'name': name,
             'callback': callback,
-            'input_types': ['str', 'bool'],
+            'input_types': ['str'],
             'output_type': 'str'
         }
         self.callbacks[name] = meta

mindsdb/interfaces/functions/to_markdown.py

@@ -1,13 +1,10 @@
-import base64
 from io import BytesIO
 import os
 from typing import Union
 from urllib.parse import urlparse
 
-import fitz  # PyMuPDF
-from markitdown import MarkItDown
+from aipdf import ocr
 import mimetypes
-from openai import OpenAI
 import requests
 
 
@@ -15,41 +12,22 @@ class ToMarkdown:
     """
     Extracts the content of documents of various formats in markdown format.
     """
-    def __init__(self, use_llm: bool, llm_client: OpenAI = None, llm_model: str = None):
+    def __init__(self):
         """
         Initializes the ToMarkdown class.
         """
-        # If use_llm is True, llm_client and llm_model must be provided.
-        if use_llm and (llm_client is None or llm_model is None):
-            raise ValueError('LLM client and model must be provided when use_llm is True.')
 
-        # If use_llm is False, set llm_client and llm_model to None even if they are provided.
-        if not use_llm:
-            llm_client = None
-            llm_model = None
-
-        # Only OpenAI is supported for now.
-        # TODO: Add support for other LLMs.
-        if llm_client is not None and not isinstance(llm_client, OpenAI):
-            raise ValueError('Only OpenAI models are supported at the moment.')
-
-        self.use_llm = use_llm
-        self.llm_client = llm_client
-        self.llm_model = llm_model
-
-    def call(self, file_path_or_url: str) -> str:
+    def call(self, file_path_or_url: str, **kwargs) -> str:
         """
         Converts a file to markdown.
         """
         file_extension = self._get_file_extension(file_path_or_url)
-        file = self._get_file_content(file_path_or_url)
+        file_content = self._get_file_content(file_path_or_url)
 
         if file_extension == '.pdf':
-            return self._pdf_to_markdown(file)
-        elif file_extension in ['.jpg', '.jpeg', '.png', '.gif']:
-            return self._image_to_markdown(file)
+            return self._pdf_to_markdown(file_content, **kwargs)
         else:
-            return self._other_to_markdown(file)
+            raise ValueError(f"Unsupported file type: {file_extension}.")
 
     def _get_file_content(self, file_path_or_url: str) -> str:
         """
@@ -90,105 +68,12 @@ class ToMarkdown:
         else:
             return os.path.splitext(file_path_or_url)[1]
 
-    def _pdf_to_markdown(self, file_content: Union[requests.Response, bytes]) -> str:
+    def _pdf_to_markdown(self, file_content: Union[requests.Response, bytes], **kwargs) -> str:
         """
         Converts a PDF file to markdown.
         """
-        if self.llm_client is None:
-            return self._pdf_to_markdown_no_llm(file_content)
-        else:
-            return self._pdf_to_markdown_llm(file_content)
-
-    def _pdf_to_markdown_llm(self, file_content: Union[requests.Response, BytesIO]) -> str:
-        """
-        Converts a PDF file to markdown using LLM.
-        The LLM is used mainly for the purpose of generating descriptions of any images in the PDF.
-        """
         if isinstance(file_content, requests.Response):
             file_content = BytesIO(file_content.content)
 
-        document = fitz.open(stream=file_content, filetype="pdf")
-
-        markdown_content = []
-        for page_num in range(len(document)):
-            page = document.load_page(page_num)
-
-            # Get text blocks with coordinates.
-            page_content = []
-            blocks = page.get_text("blocks")
-            for block in blocks:
-                x0, y0, x1, y1, text, _, _ = block
-                if text.strip():  # Skip empty or whitespace blocks.
-                    page_content.append((y0, text.strip()))
-
-            # Extract images from the page.
-            image_list = page.get_images(full=True)
-            for img_index, img in enumerate(image_list):
-                xref = img[0]
-                base_image = document.extract_image(xref)
-                image_bytes = base_image["image"]
-
-                # Use actual image y-coordinate if available.
-                y0 = float(base_image.get("y", 0))
-                image_description = self._generate_image_description(image_bytes)
-                page_content.append((y0, f"![{image_description}](image_{page_num + 1}_{img_index + 1}.png)"))
-
-            # Sort the content by y0 coordinate
-            page_content.sort(key=lambda x: x[0])
-
-            # Add sorted content to the markdown
-            for _, text in page_content:
-                markdown_content.append(text)
-            markdown_content.append("\n")
-
-        document.close()
-
-        return "\n".join(markdown_content)
-
-    def _generate_image_description(self, image_bytes: bytes) -> str:
-        """
-        Generates a description of the image using LLM.
-        """
-        image_base64 = base64.b64encode(image_bytes).decode("utf-8")
-
-        response = self.llm_client.chat.completions.create(
-            model=self.llm_model,
-            messages=[
-                {
-                    "role": "user",
-                    "content": [
-                        {"type": "text", "text": "Describe this image"},
-                        {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{image_base64}"}},
-                    ],
-                }
-            ],
-        )
-        description = response.choices[0].message.content
-        return description
-
-    def _pdf_to_markdown_no_llm(self, file_content: Union[requests.Response, BytesIO]) -> str:
-        """
-        Converts a PDF file to markdown without using LLM.
-        """
-        md = MarkItDown(enable_plugins=True)
-        result = md.convert(file_content)
-        return result.markdown
-
-    def _image_to_markdown(self, file_content: Union[requests.Response, BytesIO]) -> str:
-        """
-        Converts images to markdown.
-        """
-        if not self.use_llm or self.llm_client is None:
-            raise ValueError('LLM client must be enabled to convert images to markdown.')
-
-        md = MarkItDown(llm_client=self.llm_client, llm_model=self.llm_model, enable_plugins=True)
-        result = md.convert(file_content)
-        return result.markdown
-
-    def _other_to_markdown(self, file_content: Union[requests.Response, BytesIO]) -> str:
-        """
-        Converts other file formats to markdown.
-        """
-        md = MarkItDown(enable_plugins=True)
-        result = md.convert(file_content)
-        return result.markdown
+        markdown_pages = ocr(file_content, **kwargs)
+        return "\n\n---\n\n".join(markdown_pages)

mindsdb/interfaces/knowledge_base/controller.py

@@ -41,13 +41,14 @@ from mindsdb.utilities.config import config
 from mindsdb.utilities.context import context as ctx
 
 from mindsdb.api.executor.command_executor import ExecuteCommands
+from mindsdb.api.executor.utilities.sql import query_df
 from mindsdb.utilities import log
 from mindsdb.integrations.utilities.rag.rerankers.base_reranker import BaseLLMReranker
 
 logger = log.getLogger(__name__)
 
 KB_TO_VECTORDB_COLUMNS = {
-    'id': 'original_row_id',
+    'id': 'original_doc_id',
     'chunk_id': 'id',
     'chunk_content': 'content'
 }
@@ -150,13 +151,8 @@ class KnowledgeBaseTable:
         query.from_table = Identifier(parts=[self._kb.vector_database_table])
         logger.debug(f"Set table name to: {self._kb.vector_database_table}")
 
-        requested_kb_columns = []
-        for target in query.targets:
-            if isinstance(target, Star):
-                requested_kb_columns = None
-                break
-            else:
-                requested_kb_columns.append(target.parts[-1].lower())
+        # Copy query for complex execution via DuckDB: DISTINCT, GROUP BY etc.
+        query_copy = copy.deepcopy(query)
 
         query.targets = [
             Identifier(TableField.ID.value),
@@ -220,9 +216,17 @@ class KnowledgeBaseTable:
 
         df = self.add_relevance(df, query_text, relevance_threshold)
 
-        # filter by targets
-        if requested_kb_columns is not None:
-            df = df[requested_kb_columns]
+        if (
+            query.group_by is not None
+            or query.order_by is not None
+            or query.having is not None
+            or query.distinct is True
+            or len(query.targets) != 1
+            or not isinstance(query.targets[0], Star)
+        ):
+            query_copy.where = None
+            df = query_df(df, query_copy, session=self.session)
+
         return df
 
     def add_relevance(self, df, query_text, relevance_threshold=None):
@@ -290,7 +294,7 @@ class KnowledgeBaseTable:
         columns = list(df.columns)
         # update id, get from metadata
         df[TableField.ID.value] = df[TableField.METADATA.value].apply(
-            lambda m: None if m is None else m.get('original_row_id')
+            lambda m: None if m is None else m.get('original_doc_id')
         )
 
         # id on first place
@@ -479,12 +483,9 @@ class KnowledgeBaseTable:
                     # Use provided_id directly if it exists, otherwise generate one
                     doc_id = self._generate_document_id(content_str, col, provided_id)
 
-                    # Need provided ID to link chunks back to original source (e.g. database row).
-                    row_id = provided_id if provided_id else idx
-
                     metadata = {
                         **base_metadata,
-                        'original_row_id': str(row_id),
+                        'original_row_index': str(idx),  # provide link to original row index
                         'content_column': col,
                     }
 
@@ -787,7 +788,7 @@ class KnowledgeBaseTable:
     def _generate_document_id(self, content: str, content_column: str, provided_id: str = None) -> str:
         """Generate a deterministic document ID using the utility function."""
         from mindsdb.interfaces.knowledge_base.utils import generate_document_id
-        return generate_document_id(content, content_column, provided_id)
+        return generate_document_id(content=content, provided_id=provided_id)
 
     def _convert_metadata_value(self, value):
         """
@@ -1004,6 +1005,8 @@ class KnowledgeBaseController:
         if 'provider' in params:
             engine = params.pop('provider').lower()
 
+        api_key = get_api_key(engine, params, strict=False) or params.pop('api_key')
+
         if engine == 'azure_openai':
             engine = 'openai'
             params['provider'] = 'azure'
@@ -1011,8 +1014,8 @@ class KnowledgeBaseController:
         if engine == 'openai':
             if 'question_column' not in params:
                 params['question_column'] = 'content'
-            if 'api_key' in params:
-                params[f"{engine}_api_key"] = params.pop('api_key')
+            if api_key:
+                params[f"{engine}_api_key"] = api_key
             if 'base_url' in params:
                 params['api_base'] = params.pop('base_url')
 

mindsdb/interfaces/knowledge_base/preprocessing/document_preprocessor.py

@@ -25,6 +25,8 @@ from langchain_core.documents import Document as LangchainDocument
 
 logger = log.getLogger(__name__)
 
+_DEFAULT_CONTENT_COLUMN_NAME = "content"
+
 
 class DocumentPreprocessor:
     """Base class for document preprocessing"""
@@ -90,14 +92,18 @@ class DocumentPreprocessor:
         start_char: Optional[int] = None,
         end_char: Optional[int] = None,
         provided_id: str = None,
+        content_column: str = None,
     ) -> str:
         """Generate human-readable deterministic ID for a chunk
-        Format: <doc_id>:<chunk_number>of<total_chunks>:<start_char>to<end_char>
+        Format: <doc_id>:<content_column>:<chunk_number>of<total_chunks>:<start_char>to<end_char>
         """
         if provided_id is None:
             raise ValueError("Document ID must be provided for chunk ID generation")
 
-        chunk_id = f"{provided_id}:{chunk_index + 1}of{total_chunks}:{start_char}to{end_char}"
+        if content_column is None:
+            raise ValueError("Content column must be provided for chunk ID generation")
+
+        chunk_id = f"{provided_id}:{content_column}:{chunk_index + 1}of{total_chunks}:{start_char}to{end_char}"
         logger.debug(f"Generated chunk ID: {chunk_id}")
         return chunk_id
 
@@ -254,8 +260,15 @@ Please give a short succinct context to situate this chunk within the overall do
             if doc.metadata:
                 metadata.update(doc.metadata)
 
+            # Get content_column from metadata or use default
+            content_column = metadata.get('content_column')
+            if content_column is None:
+                # If content_column is not in metadata, use the default column name
+                content_column = _DEFAULT_CONTENT_COLUMN_NAME
+                logger.debug(f"No content_column found in metadata, using default: {_DEFAULT_CONTENT_COLUMN_NAME}")
+
             chunk_id = self._generate_chunk_id(
-                chunk_index=chunk_index, provided_id=doc.id
+                chunk_index=chunk_index, provided_id=doc.id, content_column=content_column
             )
             processed_chunks.append(
                 ProcessedChunk(
@@ -324,13 +337,23 @@ class TextChunkingPreprocessor(DocumentPreprocessor):
                 metadata["start_char"] = start_char
                 metadata["end_char"] = end_char
 
-                # Generate chunk ID with total chunks
+                # Get content_column from metadata or use default
+                content_column = None
+                if doc.metadata:
+                    content_column = doc.metadata.get('content_column')
+
+                if content_column is None:
+                    # If content_column is not in metadata, use the default column name
+                    content_column = _DEFAULT_CONTENT_COLUMN_NAME
+                    logger.debug(f"No content_column found in metadata, using default: {_DEFAULT_CONTENT_COLUMN_NAME}")
+
                 chunk_id = self._generate_chunk_id(
                     chunk_index=i,
                     total_chunks=total_chunks,
                     start_char=start_char,
                     end_char=end_char,
-                    provided_id=doc.id
+                    provided_id=doc.id,
+                    content_column=content_column
                 )
 
                 processed_chunks.append(

mindsdb/interfaces/knowledge_base/utils.py

@@ -2,27 +2,22 @@
 import hashlib
 
 
-def generate_document_id(content: str, content_column: str, provided_id: str = None) -> str:
+def generate_document_id(content: str, content_column: str = None, provided_id: str = None) -> str:
     """
-    Generate a deterministic document ID from content and column name.
-    If provided_id exists, combines it with content_column.
-    For generated IDs, uses a short hash of just the content to ensure
-    same content gets same base ID across different columns.
+    Generate a deterministic document ID from content.
+    If provided_id exists, returns it directly.
+    For generated IDs, uses a short hash of just the content.
 
     Args:
         content: The content string
-        content_column: Name of the content column
+        content_column: Name of the content column (not used in ID generation, kept for backward compatibility)
         provided_id: Optional user-provided ID
     Returns:
-        Deterministic document ID in format: <base_id>_<column>
-        where base_id is either the provided_id or a 16-char hash of content
+        Deterministic document ID (either provided_id or a 16-char hash of content)
     """
     if provided_id is not None:
-        base_id = provided_id
-    else:
-        # Generate a shorter 16-character hash based only on content
-        hash_obj = hashlib.md5(content.encode())
-        base_id = hash_obj.hexdigest()[:16]
+        return provided_id
 
-    # Append column name to maintain uniqueness across columns
-    return f"{base_id}_{content_column}"
+    # Generate a shorter 16-character hash based only on content
+    hash_obj = hashlib.md5(content.encode())
+    return hash_obj.hexdigest()[:16]

mindsdb/interfaces/model/model_controller.py

@@ -1,4 +1,3 @@
-import sys
 import copy
 import datetime as dt
 from copy import deepcopy
@@ -28,7 +27,6 @@ from mindsdb.utilities import log
 
 logger = log.getLogger(__name__)
 
-IS_PY36 = sys.version_info[1] <= 6
 
 default_project = config.get('default_project')
 

mindsdb/interfaces/skills/sql_agent.py

@@ -1,4 +1,3 @@
-
 import re
 import csv
 import inspect
@@ -13,6 +12,7 @@ from mindsdb.utilities import log
 from mindsdb.utilities.context import context as ctx
 from mindsdb.integrations.utilities.query_traversal import query_traversal
 from mindsdb.integrations.libs.response import INF_SCHEMA_COLUMNS_NAMES
+from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import MYSQL_DATA_TYPE
 
 logger = log.getLogger(__name__)
 
@@ -253,7 +253,7 @@ class SQLAgent:
         for table in all_tables:
             key = f"{ctx.company_id}_{table}_info"
             table_info = self._cache.get(key) if self._cache else None
-            if table_info is None:
+            if True or table_info is None:
                 table_info = self._get_single_table_info(table)
                 if self._cache:
                     self._cache.set(key, table_info)
@@ -276,19 +276,41 @@ class SQLAgent:
         dn = self._command_executor.session.datahub.get(integration)
 
         fields, dtypes = [], []
-        for df in dn.get_table_columns_df(table_name, schema_name):
-            df_records = df.to_dict(orient='records')
-            fields.append(df_records[INF_SCHEMA_COLUMNS_NAMES.COLUMN_NAME])
-            if df_records[INF_SCHEMA_COLUMNS_NAMES.MYSQL_DATA_TYPE] is not None:
-                dtypes.append(df_records[INF_SCHEMA_COLUMNS_NAMES.MYSQL_DATA_TYPE].value)
-            else:
-                dtypes.append(df_records[INF_SCHEMA_COLUMNS_NAMES.DATA_TYPE])
+        try:
+            df = dn.get_table_columns_df(table_name, schema_name)
+            if not isinstance(df, pd.DataFrame) or df.empty:
+                logger.warning(f"Received empty or invalid DataFrame for table columns of {table_str}")
+                return f"Table named `{table_str}`:\n [No column information available]"
+
+            fields = df[INF_SCHEMA_COLUMNS_NAMES.COLUMN_NAME].to_list()
+            dtypes = [
+                mysql_data_type.value if isinstance(mysql_data_type, MYSQL_DATA_TYPE) else (data_type or 'UNKNOWN')
+                for mysql_data_type, data_type
+                in zip(
+                    df[INF_SCHEMA_COLUMNS_NAMES.MYSQL_DATA_TYPE],
+                    df[INF_SCHEMA_COLUMNS_NAMES.DATA_TYPE]
+                )
+            ]
+        except Exception as e:
+            logger.error(f"Failed processing column info for {table_str}: {e}", exc_info=True)
+            raise ValueError(f"Failed to process column info for {table_str}") from e
+
+        if not fields:
+            logger.error(f"Could not extract column fields for {table_str}.")
+            return f"Table named `{table_str}`:\n [Could not extract column information]"
+
+        try:
+            sample_rows_info = self._get_sample_rows(table_str, fields)
+        except Exception as e:
+            logger.warning(f"Could not get sample rows for {table_str}: {e}")
+            sample_rows_info = "\n\t [error] Couldn't retrieve sample rows!"
 
         info = f'Table named `{table_str}`:\n'
         info += f"\nSample with first {self._sample_rows_in_table_info} rows from table {table_str} in CSV format (dialect is 'excel'):\n"
-        info += self._get_sample_rows(table_str, fields) + "\n"
+        info += sample_rows_info + "\n"
         info += '\nColumn data types: ' + ",\t".join(
-            [f'\n`{field}` : `{dtype}`' for field, dtype in zip(fields, dtypes)]) + '\n'  # noqa
+            [f'\n`{field}` : `{dtype}`' for field, dtype in zip(fields, dtypes)]
+        ) + '\n'
         return info
 
     def _get_sample_rows(self, table: str, fields: List[str]) -> str:

mindsdb/migrations/migrate.py

@@ -47,7 +47,5 @@ def migrate_to_head():
 if __name__ == "__main__":
     # have to import this because
     # all env initialization happens here
-    from mindsdb.utilities.config import Config as MDBConfig
-    MDBConfig()
     db.init()
     migrate_to_head()

mindsdb/utilities/config.py

@@ -143,7 +143,8 @@ class Config:
             'auth': {
                 'http_auth_enabled': False,
                 "http_permanent_session_lifetime": datetime.timedelta(days=31),
-                "username": "mindsdb"
+                "username": "mindsdb",
+                "password": ""
             },
             "logging": {
                 "handlers": {
@@ -459,7 +460,7 @@ class Config:
         """
         updated = self.fetch_auto_config()
         if updated:
-            self.init_config()
+            self.merge_configs()
 
     def merge_configs(self) -> None:
         """Merge multiple configs to one.

mindsdb/utilities/context.py

@@ -54,7 +54,7 @@ class Context:
     def load(self, storage: dict) -> None:
         self._storage.set(storage)
 
-    def metadata(self, **kwargs) -> dict:
+    def get_metadata(self, **kwargs) -> dict:
         return {
             'user_id': self.user_id or "",
             'company_id': self.company_id or "",

mindsdb/utilities/functions.py

@@ -7,7 +7,6 @@ import textwrap
 from functools import wraps
 from collections.abc import Callable
 
-import requests
 from cryptography.fernet import Fernet
 from mindsdb_sql_parser.ast import Identifier
 
@@ -72,41 +71,6 @@ def mark_process(name: str, custom_mark: str = None) -> Callable:
     return mark_process_wrapper
 
 
-def get_versions_where_predictors_become_obsolete():
-    """ Get list of MindsDB versions in which predictors should be retrained
-        Returns:
-            list of str or False
-    """
-    versions_for_updating_predictors = []
-    try:
-        try:
-            res = requests.get(
-                'https://mindsdb-cloud-public-service-files.s3.us-east-2.amazonaws.com/version_for_updating_predictors.txt',
-                timeout=0.5
-            )
-        except (ConnectionError, requests.exceptions.ConnectionError) as e:
-            logger.error(f'Is no connection. {e}')
-            raise
-        except Exception as e:
-            logger.error(f'Is something wrong with getting version_for_updating_predictors.txt: {e}')
-            raise
-
-        if res.status_code != 200:
-            logger.error(f'Cant get version_for_updating_predictors.txt: returned status code = {res.status_code}')
-            raise
-
-        try:
-            versions_for_updating_predictors = res.text.replace(' \t\r', '').split('\n')
-        except Exception as e:
-            logger.error(f'Cant decode version_for_updating_predictors.txt: {e}')
-            raise
-    except Exception:
-        return False, versions_for_updating_predictors
-
-    versions_for_updating_predictors = [x for x in versions_for_updating_predictors if len(x) > 0]
-    return True, versions_for_updating_predictors
-
-
 def init_lexer_parsers():
     from mindsdb_sql_parser.lexer import MindsDBLexer
     from mindsdb_sql_parser.parser import MindsDBParser