MindsDB 25.4.1.0__py3-none-any.whl → 25.4.2.0__py3-none-any.whl

mindsdb/interfaces/functions/controller.py

@@ -1,6 +1,7 @@
 import os
 
 from duckdb.typing import BIGINT, DOUBLE, VARCHAR, BLOB, BOOLEAN
+from mindsdb.interfaces.functions.to_markdown import ToMarkdown
 from mindsdb.interfaces.storage.model_fs import HandlerStorage
 
 
@@ -121,32 +122,20 @@ class FunctionController(BYOMFunctionsController):
         if meta is not None:
             return meta
 
-        # builtin function
+        # builtin functions
         if node.op.lower() == 'llm':
             return self.llm_call_function(node)
 
+        elif node.op.lower() == 'to_markdown':
+            return self.to_markdown_call_function(node)
+
     def llm_call_function(self, node):
         name = node.op.lower()
 
         if name in self.callbacks:
             return self.callbacks[name]
 
-        param_prefix = 'LLM_FUNCTION_'
-        chat_model_params = {}
-        for k, v in os.environ.items():
-            if k.startswith(param_prefix):
-                param_name = k[len(param_prefix):]
-                if param_name == 'MODEL':
-                    chat_model_params['model_name'] = v
-                else:
-                    chat_model_params[param_name.lower()] = v
-
-        if 'provider' not in chat_model_params:
-            chat_model_params['provider'] = 'openai'
-
-        if 'api_key' in chat_model_params:
-            # move to api_keys dict
-            chat_model_params["api_keys"] = {chat_model_params['provider']: chat_model_params['api_key']}
+        chat_model_params = self._parse_chat_model_params()
 
         try:
             from langchain_core.messages import HumanMessage
@@ -168,6 +157,59 @@ class FunctionController(BYOMFunctionsController):
         self.callbacks[name] = meta
         return meta
 
+    def to_markdown_call_function(self, node):
+        name = node.op.lower()
+
+        if name in self.callbacks:
+            return self.callbacks[name]
+
+        def callback(file_path_or_url, use_llm):
+            chat_model_params = self._parse_chat_model_params()
+
+            llm_client = None
+            llm_model = None
+            try:
+                from mindsdb.interfaces.agents.langchain_agent import create_chat_model
+                llm = create_chat_model(chat_model_params)
+                llm_client = llm.root_client
+                llm_model = llm.model_name
+            except Exception:
+                pass
+
+            to_markdown = ToMarkdown(use_llm, llm_client, llm_model)
+            return to_markdown.call(file_path_or_url)
+
+        meta = {
+            'name': name,
+            'callback': callback,
+            'input_types': ['str', 'bool'],
+            'output_type': 'str'
+        }
+        self.callbacks[name] = meta
+        return meta
+
+    def _parse_chat_model_params(self, param_prefix: str = 'LLM_FUNCTION_'):
+        """
+        Parses the environment variables for chat model parameters.
+        """
+        chat_model_params = {}
+        for k, v in os.environ.items():
+            if k.startswith(param_prefix):
+                param_name = k[len(param_prefix):]
+                if param_name == 'MODEL':
+                    chat_model_params['model_name'] = v
+                else:
+                    chat_model_params[param_name.lower()] = v
+
+        if 'provider' not in chat_model_params:
+            chat_model_params['provider'] = 'openai'
+
+        if 'api_key' in chat_model_params:
+            # move to api_keys dict
+            chat_model_params["api_keys"] = {chat_model_params['provider']: chat_model_params['api_key']}
+
+        return chat_model_params
+
 
 class DuckDBFunctions:
     def __init__(self, controller):

mindsdb/interfaces/functions/to_markdown.py

@@ -0,0 +1,194 @@
+import base64
+from io import BytesIO
+import os
+from typing import Union
+from urllib.parse import urlparse
+
+import fitz  # PyMuPDF
+from markitdown import MarkItDown
+import mimetypes
+from openai import OpenAI
+import requests
+
+
+class ToMarkdown:
+    """
+    Extracts the content of documents of various formats in markdown format.
+    """
+    def __init__(self, use_llm: bool, llm_client: OpenAI = None, llm_model: str = None):
+        """
+        Initializes the ToMarkdown class.
+        """
+        # If use_llm is True, llm_client and llm_model must be provided.
+        if use_llm and (llm_client is None or llm_model is None):
+            raise ValueError('LLM client and model must be provided when use_llm is True.')
+
+        # If use_llm is False, set llm_client and llm_model to None even if they are provided.
+        if not use_llm:
+            llm_client = None
+            llm_model = None
+
+        # Only OpenAI is supported for now.
+        # TODO: Add support for other LLMs.
+        if llm_client is not None and not isinstance(llm_client, OpenAI):
+            raise ValueError('Only OpenAI models are supported at the moment.')
+
+        self.use_llm = use_llm
+        self.llm_client = llm_client
+        self.llm_model = llm_model
+
+    def call(self, file_path_or_url: str) -> str:
+        """
+        Converts a file to markdown.
+        """
+        file_extension = self._get_file_extension(file_path_or_url)
+        file = self._get_file_content(file_path_or_url)
+
+        if file_extension == '.pdf':
+            return self._pdf_to_markdown(file)
+        elif file_extension in ['.jpg', '.jpeg', '.png', '.gif']:
+            return self._image_to_markdown(file)
+        else:
+            return self._other_to_markdown(file)
+
+    def _get_file_content(self, file_path_or_url: str) -> str:
+        """
+        Retrieves the content of a file.
+        """
+        parsed_url = urlparse(file_path_or_url)
+        if parsed_url.scheme in ('http', 'https'):
+            response = requests.get(file_path_or_url)
+            if response.status_code == 200:
+                return response
+            else:
+                raise RuntimeError(f'Unable to retrieve file from URL: {file_path_or_url}')
+        else:
+            with open(file_path_or_url, 'rb') as file:
+                return BytesIO(file.read())
+
+    def _get_file_extension(self, file_path_or_url: str) -> str:
+        """
+        Retrieves the file extension from a file path or URL.
+        """
+        parsed_url = urlparse(file_path_or_url)
+        if parsed_url.scheme in ('http', 'https'):
+            try:
+                # Make a HEAD request to get headers without downloading the file.
+                response = requests.head(file_path_or_url, allow_redirects=True)
+                content_type = response.headers.get('Content-Type', '')
+                if content_type:
+                    ext = mimetypes.guess_extension(content_type.split(';')[0].strip())
+                    if ext:
+                        return ext
+
+                # Fallback to extracting extension from the URL path
+                ext = os.path.splitext(parsed_url.path)[1]
+                if ext:
+                    return ext
+            except requests.RequestException:
+                raise RuntimeError(f'Unable to retrieve file extension from URL: {file_path_or_url}')
+        else:
+            return os.path.splitext(file_path_or_url)[1]
+
+    def _pdf_to_markdown(self, file_content: Union[requests.Response, bytes]) -> str:
+        """
+        Converts a PDF file to markdown.
+        """
+        if self.llm_client is None:
+            return self._pdf_to_markdown_no_llm(file_content)
+        else:
+            return self._pdf_to_markdown_llm(file_content)
+
+    def _pdf_to_markdown_llm(self, file_content: Union[requests.Response, BytesIO]) -> str:
+        """
+        Converts a PDF file to markdown using LLM.
+        The LLM is used mainly for the purpose of generating descriptions of any images in the PDF.
+        """
+        if isinstance(file_content, requests.Response):
+            file_content = BytesIO(file_content.content)
+
+        document = fitz.open(stream=file_content, filetype="pdf")
+
+        markdown_content = []
+        for page_num in range(len(document)):
+            page = document.load_page(page_num)
+
+            # Get text blocks with coordinates.
+            page_content = []
+            blocks = page.get_text("blocks")
+            for block in blocks:
+                x0, y0, x1, y1, text, _, _ = block
+                if text.strip():  # Skip empty or whitespace blocks.
+                    page_content.append((y0, text.strip()))
+
+            # Extract images from the page.
+            image_list = page.get_images(full=True)
+            for img_index, img in enumerate(image_list):
+                xref = img[0]
+                base_image = document.extract_image(xref)
+                image_bytes = base_image["image"]
+
+                # Use actual image y-coordinate if available.
+                y0 = float(base_image.get("y", 0))
+                image_description = self._generate_image_description(image_bytes)
+                page_content.append((y0, f"![{image_description}](image_{page_num + 1}_{img_index + 1}.png)"))
+
+            # Sort the content by y0 coordinate
+            page_content.sort(key=lambda x: x[0])
+
+            # Add sorted content to the markdown
+            for _, text in page_content:
+                markdown_content.append(text)
+            markdown_content.append("\n")
+
+        document.close()
+
+        return "\n".join(markdown_content)
+
+    def _generate_image_description(self, image_bytes: bytes) -> str:
+        """
+        Generates a description of the image using LLM.
+        """
+        image_base64 = base64.b64encode(image_bytes).decode("utf-8")
+
+        response = self.llm_client.chat.completions.create(
+            model=self.llm_model,
+            messages=[
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": "Describe this image"},
+                        {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{image_base64}"}},
+                    ],
+                }
+            ],
+        )
+        description = response.choices[0].message.content
+        return description
+
+    def _pdf_to_markdown_no_llm(self, file_content: Union[requests.Response, BytesIO]) -> str:
+        """
+        Converts a PDF file to markdown without using LLM.
+        """
+        md = MarkItDown(enable_plugins=True)
+        result = md.convert(file_content)
+        return result.markdown
+
+    def _image_to_markdown(self, file_content: Union[requests.Response, BytesIO]) -> str:
+        """
+        Converts images to markdown.
+        """
+        if not self.use_llm or self.llm_client is None:
+            raise ValueError('LLM client must be enabled to convert images to markdown.')
+
+        md = MarkItDown(llm_client=self.llm_client, llm_model=self.llm_model, enable_plugins=True)
+        result = md.convert(file_content)
+        return result.markdown
+
+    def _other_to_markdown(self, file_content: Union[requests.Response, BytesIO]) -> str:
+        """
+        Converts other file formats to markdown.
+        """
+        md = MarkItDown(enable_plugins=True)
+        result = md.convert(file_content)
+        return result.markdown

mindsdb/interfaces/jobs/jobs_controller.py

@@ -337,10 +337,10 @@ class JobsController:
                 BinaryOperation(op='=', args=[Identifier('project'), Constant(project_name)])
             ])
         )
-        data, columns = logs_db_controller.query(query)
+        response = logs_db_controller.query(query)
 
-        names = [i['name'] for i in columns]
-        return data[names].to_dict(orient='records')
+        names = [i['name'] for i in response.columns]
+        return response.data_frame[names].to_dict(orient='records')
 
 
 class JobsExecutor:

mindsdb/interfaces/knowledge_base/controller.py

@@ -35,6 +35,7 @@ from mindsdb.interfaces.knowledge_base.preprocessing.models import Preprocessing
 from mindsdb.interfaces.knowledge_base.preprocessing.document_preprocessor import PreprocessorFactory
 from mindsdb.interfaces.model.functions import PredictorRecordNotFound
 from mindsdb.utilities.exception import EntityExistsError, EntityNotExistsError
+from mindsdb.integrations.utilities.sql_utils import FilterCondition, FilterOperator
 
 from mindsdb.api.executor.command_executor import ExecuteCommands
 from mindsdb.utilities import log
@@ -85,88 +86,124 @@ class KnowledgeBaseTable:
         """
         logger.debug(f"Processing select query: {query}")
 
-        # replace content with embeddings
-        query_traversal(query.where, self._replace_query_content)
-        logger.debug("Replaced content with embeddings in where clause")
+        # Extract the content query text for potential reranking
+
+        db_handler = self.get_vector_db()
 
+        logger.debug("Replaced content with embeddings in where clause")
         # set table name
         query.from_table = Identifier(parts=[self._kb.vector_database_table])
         logger.debug(f"Set table name to: {self._kb.vector_database_table}")
 
-        # remove embeddings from result
-        targets = []
+        requested_kb_columns = []
         for target in query.targets:
             if isinstance(target, Star):
-                targets.extend([
-                    Identifier(TableField.ID.value),
-                    Identifier(TableField.CONTENT.value),
-                    Identifier(TableField.METADATA.value),
-                ])
-            elif isinstance(target, Identifier) and target.parts[-1].lower() != TableField.EMBEDDINGS.value:
-                targets.append(target)
-        query.targets = targets
-        logger.debug(f"Modified query targets: {targets}")
+                requested_kb_columns = None
+                break
+            else:
+                requested_kb_columns.append(target.parts[-1].lower())
+
+        query.targets = [
+            Identifier(TableField.ID.value),
+            Identifier(TableField.CONTENT.value),
+            Identifier(TableField.METADATA.value),
+            Identifier(TableField.DISTANCE.value),
+        ]
 
         # Get response from vector db
-        db_handler = self.get_vector_db()
         logger.debug(f"Using vector db handler: {type(db_handler)}")
 
-        conditions = db_handler.extract_conditions(query.where)
+        # extract values from conditions and prepare for vectordb
+        conditions = []
+        query_text = None
+        reranking_threshold = None
+        query_conditions = db_handler.extract_conditions(query.where)
+        if query_conditions is not None:
+            for item in query_conditions:
+                if item.column == "reranking_threshold" and item.op.value == "=":
+                    try:
+                        reranking_threshold = float(item.value)
+                        # Validate range: must be between 0 and 1
+                        if not (0 <= reranking_threshold <= 1):
+                            raise ValueError(f"reranking_threshold must be between 0 and 1, got: {reranking_threshold}")
+                        logger.debug(f"Found reranking_threshold in query: {reranking_threshold}")
+                    except (ValueError, TypeError) as e:
+                        error_msg = f"Invalid reranking_threshold value: {item.value}. {str(e)}"
+                        logger.error(error_msg)
+                        raise ValueError(error_msg)
+                elif item.column == TableField.CONTENT.value:
+                    query_text = item.value
+
+                    # replace content with embeddings
+                    conditions.append(FilterCondition(
+                        column=TableField.EMBEDDINGS.value,
+                        value=self._content_to_embeddings(item.value),
+                        op=FilterOperator.EQUAL,
+                    ))
+                else:
+                    conditions.append(item)
+
+        logger.debug(f"Extracted query text: {query_text}")
+
         self.addapt_conditions_columns(conditions)
         df = db_handler.dispatch_select(query, conditions)
+        df = self.addapt_result_columns(df)
 
-        if df is not None:
+        logger.debug(f"Query returned {len(df)} rows")
+        logger.debug(f"Columns in response: {df.columns.tolist()}")
+        # Check if we have a rerank_model configured in KB params
 
-            logger.debug(f"Query returned {len(df)} rows")
-            logger.debug(f"Columns in response: {df.columns.tolist()}")
-            # Log a sample of IDs to help diagnose issues
-            if not df.empty:
-                logger.debug(f"Sample of IDs in response: {df['id'].head().tolist()}")
-        else:
-            logger.warning("Query returned no data")
+        df = self.add_relevance(df, query_text, reranking_threshold)
+
+        # filter by targets
+        if requested_kb_columns is not None:
+            df = df[requested_kb_columns]
+        return df
+
+    def add_relevance(self, df, query_text, reranking_threshold=None):
+        relevance_column = TableField.RELEVANCE.value
 
         rerank_model = self._kb.params.get("rerank_model")
-        if rerank_model and df is not None and not df.empty:
+        if rerank_model and query_text and len(df) > 0:
+            # Use reranker for relevance score
             try:
-                logger.info(f"Using reranker model: {rerank_model}")
-                reranker = LLMReranker(model=rerank_model)
-                # convert response from a dataframe to a list of strings
-                content_column = df[TableField.CONTENT.value]
-                # convert to list
-                documents = content_column.tolist()
-                # Extract query text from WHERE clause if it exists
-                query_text = ""
-                if query.where:
-                    def extract_content(node, **kwargs):
-                        nonlocal query_text
-                        is_binary_op = isinstance(node, BinaryOperation)
-                        is_identifier = isinstance(node.args[0], Identifier)
-                        is_content = node.args[0].parts[-1].lower() == 'content'
-                        is_constant = isinstance(node.args[1], Constant)
-                        if is_binary_op and is_identifier and is_content and is_constant:
-                            query_text = node.args[1].value
-                    query_traversal(query.where, extract_content)
-                    logger.debug(f"Extracted query text: {query_text}")
-                # Get scores from reranker
+                logger.info(f"Using reranker model {rerank_model} for relevance calculation")
+                reranker_params = {"model": rerank_model}
+                # Apply custom filtering threshold if provided
+                if reranking_threshold is not None:
+                    reranker_params["filtering_threshold"] = reranking_threshold
+                    logger.info(f"Using custom filtering threshold: {reranking_threshold}")
+
+                reranker = LLMReranker(**reranker_params)
+                # Get documents to rerank
+                documents = df['chunk_content'].tolist()
+                # Use the get_scores method with disable_events=True
                 scores = reranker.get_scores(query_text, documents)
-                # Add scores as a new column for filtering
+                # Add scores as the relevance column
+                df[relevance_column] = scores
+
+                # Filter by threshold
                 scores_array = np.array(scores)
-                # Add temporary column for sorting
-                df['_relevance_score'] = scores
-                # Filter by score threshold using numpy array for element-wise comparison
                 df = df[scores_array > reranker.filtering_threshold]
-                # Sort by relevance (higher score = more relevant)
-                df = df.sort_values(by='_relevance_score', ascending=False)
-                # Remove temporary column
-                # df = df.drop(columns=['_relevance_score'])
-                # Apply original limit if it exists
-                if query.limit and len(df) > query.limit.value:
-                    df = df.iloc[:query.limit.value]
-                logger.debug(f"Applied reranking with model {rerank_model}")
+                logger.debug(f"Applied reranking with model {rerank_model}, threshold: {reranker.filtering_threshold}")
             except Exception as e:
                 logger.error(f"Error during reranking: {str(e)}")
+                # Fallback to distance-based relevance
+                if 'distance' in df.columns:
+                    df[relevance_column] = 1 / (1 + df['distance'])
+                else:
+                    logger.info("No distance or reranker available")
 
-        df = self.addapt_result_columns(df)
+        elif 'distance' in df.columns:
+            # Calculate relevance from distance
+            logger.info("Calculating relevance from vector distance")
+            df[relevance_column] = 1 / (1 + df['distance'])
+
+        else:
+            df[relevance_column] = None
+            df['distance'] = None
+        # Sort by relevance
+        df = df.sort_values(by=relevance_column, ascending=False)
         return df
 
     def addapt_conditions_columns(self, conditions):
@@ -186,7 +223,9 @@ class KnowledgeBaseTable:
 
         columns = list(df.columns)
         # update id, get from metadata
-        df[TableField.ID.value] = df[TableField.METADATA.value].apply(lambda m: m.get('original_row_id'))
+        df[TableField.ID.value] = df[TableField.METADATA.value].apply(
+            lambda m: None if m is None else m.get('original_row_id')
+        )
 
         # id on first place
         return df[[TableField.ID.value] + columns]
@@ -276,7 +315,9 @@ class KnowledgeBaseTable:
 
         # send to vectordb
         db_handler = self.get_vector_db()
-        db_handler.query(query)
+        conditions = db_handler.extract_conditions(query.where)
+        self.addapt_conditions_columns(conditions)
+        db_handler.dispatch_update(query, conditions)
 
     def delete_query(self, query: Delete):
         """

mindsdb/interfaces/knowledge_base/preprocessing/document_preprocessor.py

@@ -92,9 +92,7 @@ class DocumentPreprocessor:
 
     def _generate_chunk_id(
         self,
-        content: str,
         chunk_index: Optional[int] = None,
-        content_column: str = None,
         provided_id: str = None,
     ) -> str:
         """Generate deterministic ID for a chunk"""
@@ -262,15 +260,8 @@ Please give a short succinct context to situate this chunk within the overall do
             if doc.metadata:
                 metadata.update(doc.metadata)
 
-            # Pass through doc.id and content_column
-            content_column = (
-                doc.metadata.get("content_column") if doc.metadata else None
-            )
             chunk_id = self._generate_chunk_id(
-                processed_content,
-                chunk_index,
-                content_column=content_column,
-                provided_id=doc.id,
+                chunk_index=chunk_index, provided_id=doc.id
             )
             processed_chunks.append(
                 ProcessedChunk(
@@ -335,7 +326,7 @@ class TextChunkingPreprocessor(DocumentPreprocessor):
 
                 # Pass through doc.id and content_column
                 id = self._generate_chunk_id(
-                    chunk_doc.content, content_column=content_column, provided_id=doc.id
+                    chunk_index=0, provided_id=doc.id
                 )
                 processed_chunks.append(
                     ProcessedChunk(
@@ -358,9 +349,7 @@ class TextChunkingPreprocessor(DocumentPreprocessor):
 
                     # Pass through doc.id and content_column
                     chunk_id = self._generate_chunk_id(
-                        chunk_doc.content,
-                        i,
-                        content_column=content_column,
+                        chunk_index=i,
                         provided_id=doc.id,
                     )
                     processed_chunks.append(

mindsdb/interfaces/query_context/context_controller.py

@@ -156,10 +156,12 @@ class QueryContextController:
         last_values = {}
         for query, info in l_query.get_init_queries():
 
-            data, columns_info = dn.query(
+            response = dn.query(
                 query=query,
                 session=session
             )
+            data = response.data_frame
+            columns_info = response.columns
 
             if len(data) == 0:
                 value = None