MindsDB 25.2.2.2__py3-none-any.whl → 25.2.4.0__py3-none-any.whl

mindsdb/integrations/utilities/files/file_reader.py

@@ -4,6 +4,7 @@ import csv
 from io import BytesIO, StringIO, IOBase
 from pathlib import Path
 import codecs
+from typing import List
 
 import filetype
 import pandas as pd
@@ -65,6 +66,7 @@ def decode(file_obj: IOBase) -> StringIO:
 class FormatDetector:
 
     supported_formats = ['parquet', 'csv', 'xlsx', 'pdf', 'json', 'txt']
+    multipage_formats = ['xlsx']
 
     def __init__(
         self,
@@ -200,16 +202,62 @@ class FormatDetector:
 
 class FileReader(FormatDetector):
 
-    def to_df(self, **kwargs) -> pd.DataFrame:
+    def _get_fnc(self):
         format = self.get_format()
-
         func = getattr(self, f'read_{format}', None)
         if func is None:
             raise FileDetectError(f'Unsupported format: {format}')
+        return func
+
+    def get_pages(self, **kwargs) -> List[str]:
+        """
+            Get list of tables in file
+        """
+        format = self.get_format()
+        if format not in self.multipage_formats:
+            # only one table
+            return ['main']
+
+        func = self._get_fnc()
+        self.file_obj.seek(0)
 
+        return [
+            name for name, _ in
+            func(self.file_obj, only_names=True, **kwargs)
+        ]
+
+    def get_contents(self, **kwargs):
+        """
+            Get all info(pages with content) from file as dict: {tablename, content}
+        """
+        func = self._get_fnc()
         self.file_obj.seek(0)
-        kwargs.update(self.parameters)
-        return func(self.file_obj, name=self.name, **kwargs)
+
+        format = self.get_format()
+        if format not in self.multipage_formats:
+            # only one table
+            return {'main': func(self.file_obj, name=self.name, **kwargs)}
+
+        return {
+            name: df
+            for name, df in
+            func(self.file_obj, **kwargs)
+        }
+
+    def get_page_content(self, page_name: str = None, **kwargs) -> pd.DataFrame:
+        """
+            Get content of a single table
+        """
+        func = self._get_fnc()
+        self.file_obj.seek(0)
+
+        format = self.get_format()
+        if format not in self.multipage_formats:
+            # only one table
+            return func(self.file_obj, name=self.name, **kwargs)
+
+        for _, df in func(self.file_obj, name=self.name, page_name=page_name, **kwargs):
+            return df
 
     @staticmethod
     def _get_csv_dialect(buffer, delimiter=None) -> csv.Dialect:
@@ -304,14 +352,18 @@ class FileReader(FormatDetector):
         return pd.read_parquet(file_obj)
 
     @staticmethod
-    def read_xlsx(file_obj: BytesIO, sheet_name=None, **kwargs) -> pd.DataFrame:
-
-        file_obj.seek(0)
+    def read_xlsx(file_obj: BytesIO, page_name=None, only_names=False, **kwargs):
         with pd.ExcelFile(file_obj) as xls:
-            if sheet_name is None:
-                # No sheet specified: Return list of sheets
-                sheet_list = xls.sheet_names
-                return pd.DataFrame(sheet_list, columns=["Sheet_Name"])
-            else:
-                # Specific sheet requested: Load that sheet
-                return pd.read_excel(xls, sheet_name=sheet_name)
+
+            if page_name is not None:
+                # return specific page
+                yield page_name, pd.read_excel(xls, sheet_name=page_name)
+
+            for page_name in xls.sheet_names:
+
+                if only_names:
+                    # extract only pages names
+                    df = None
+                else:
+                    df = pd.read_excel(xls, sheet_name=page_name)
+                yield page_name, df

mindsdb/integrations/utilities/rag/chains/local_context_summarizer_chain.py

@@ -0,0 +1,227 @@
+import asyncio
+from collections import namedtuple
+from typing import Any, Dict, List, Optional
+
+from mindsdb.interfaces.agents.langchain_agent import create_chat_model
+from langchain.chains.base import Chain
+from langchain.chains.combine_documents.stuff import StuffDocumentsChain
+from langchain.chains.llm import LLMChain
+from langchain.chains.combine_documents.map_reduce import MapReduceDocumentsChain, ReduceDocumentsChain
+from langchain_core.callbacks import dispatch_custom_event
+from langchain_core.callbacks.manager import CallbackManagerForChainRun
+from langchain_core.documents import Document
+from langchain_core.prompts import PromptTemplate
+from pandas import DataFrame
+
+from mindsdb.integrations.libs.vectordatabase_handler import VectorStoreHandler
+from mindsdb.integrations.utilities.rag.settings import SummarizationConfig
+from mindsdb.integrations.utilities.sql_utils import FilterCondition, FilterOperator
+from mindsdb.utilities import log
+
+logger = log.getLogger(__name__)
+
+Summary = namedtuple('Summary', ['source_id', 'content'])
+
+
+def create_map_reduce_documents_chain(summarization_config: SummarizationConfig, input: str) -> ReduceDocumentsChain:
+    """Creates a chain that map-reduces documents into a single consolidated summary."""
+    summarization_llm = create_chat_model({
+        'model_name': summarization_config.llm_config.model_name,
+        'provider': summarization_config.llm_config.provider,
+        **summarization_config.llm_config.params
+    })
+
+    reduce_prompt_template = summarization_config.reduce_prompt_template
+    reduce_prompt = PromptTemplate.from_template(reduce_prompt_template)
+    if 'input' in reduce_prompt.input_variables:
+        reduce_prompt = reduce_prompt.partial(input=input)
+
+    reduce_chain = LLMChain(llm=summarization_llm, prompt=reduce_prompt)
+
+    combine_documents_chain = StuffDocumentsChain(
+        llm_chain=reduce_chain,
+        document_variable_name='docs'
+    )
+
+    return ReduceDocumentsChain(
+        combine_documents_chain=combine_documents_chain,
+        collapse_documents_chain=combine_documents_chain,
+        token_max=summarization_config.max_summarization_tokens
+    )
+
+
+class LocalContextSummarizerChain(Chain):
+    """Summarizes M chunks before and after a given chunk in a document."""
+
+    doc_id_key: str = 'original_row_id'
+    chunk_index_key: str = 'chunk_index'
+
+    vector_store_handler: VectorStoreHandler
+    table_name: str = 'embeddings'
+    content_column_name: str = 'content'
+    metadata_column_name: str = 'metadata'
+
+    summarization_config: SummarizationConfig
+    map_reduce_documents_chain: Optional[ReduceDocumentsChain] = None
+
+    def _select_chunks_from_vector_store(self, doc_id: str) -> DataFrame:
+        condition = FilterCondition(
+            f"{self.metadata_column_name}->>'{self.doc_id_key}'",
+            FilterOperator.EQUAL,
+            doc_id
+        )
+        return self.vector_store_handler.select(
+            self.table_name,
+            columns=[self.content_column_name, self.metadata_column_name],
+            conditions=[condition]
+        )
+
+    async def _get_all_chunks_for_document(self, doc_id: str) -> List[Document]:
+        df = await asyncio.get_event_loop().run_in_executor(
+            None, self._select_chunks_from_vector_store, doc_id
+        )
+        chunks = []
+        for _, row in df.iterrows():
+            metadata = row.get(self.metadata_column_name, {})
+            metadata[self.chunk_index_key] = row.get('chunk_id', 0)
+            chunks.append(Document(page_content=row[self.content_column_name], metadata=metadata))
+
+        return sorted(chunks, key=lambda x: x.metadata.get(self.chunk_index_key, 0))
+
+    async def summarize_local_context(self, doc_id: str, target_chunk_index: int, M: int) -> Summary:
+        """
+        Summarizes M chunks before and after the given chunk.
+
+        Args:
+            doc_id (str): Document ID.
+            target_chunk_index (int): Index of the chunk to summarize around.
+            M (int): Number of chunks before and after to include.
+
+        Returns:
+            Summary: Summary object containing source_id and summary content.
+        """
+        logger.debug(f"Fetching chunks for document {doc_id}")
+        all_chunks = await self._get_all_chunks_for_document(doc_id)
+
+        if not all_chunks:
+            logger.warning(f"No chunks found for document {doc_id}")
+            return Summary(source_id=doc_id, content='')
+
+        # Determine window boundaries
+        start_idx = max(0, target_chunk_index - M)
+        end_idx = min(len(all_chunks), target_chunk_index + M + 1)
+        local_chunks = all_chunks[start_idx:end_idx]
+
+        logger.debug(f"Summarizing chunks {start_idx} to {end_idx - 1} for document {doc_id}")
+
+        if not self.map_reduce_documents_chain:
+            self.map_reduce_documents_chain = create_map_reduce_documents_chain(
+                self.summarization_config, input="Summarize these chunks."
+            )
+
+        summary_result = await self.map_reduce_documents_chain.ainvoke(local_chunks)
+        summary_text = summary_result.get('output_text', '')
+
+        logger.debug(f"Generated summary: {summary_text[:100]}...")
+
+        return Summary(source_id=doc_id, content=summary_text)
+
+    @property
+    def input_keys(self) -> List[str]:
+        return [self.context_key, self.question_key]
+
+    @property
+    def output_keys(self) -> List[str]:
+        return [self.context_key, self.question_key]
+
+    async def _get_source_summary(self, source_id: str, map_reduce_documents_chain: MapReduceDocumentsChain) -> Summary:
+        if not source_id:
+            logger.warning("Received empty source_id, returning empty summary")
+            return Summary(source_id='', content='')
+
+        logger.debug(f"Getting summary for source ID: {source_id}")
+        source_chunks = await self._get_all_chunks_for_document(source_id)
+
+        if not source_chunks:
+            logger.warning(f"No chunks found for source ID: {source_id}")
+            return Summary(source_id=source_id, content='')
+
+        logger.debug(f"Summarizing {len(source_chunks)} chunks for source ID: {source_id}")
+        summary = await map_reduce_documents_chain.ainvoke(source_chunks)
+        content = summary.get('output_text', '')
+        logger.debug(f"Generated summary for source ID {source_id}: {content[:100]}...")
+
+        # Stream summarization update.
+        dispatch_custom_event('summary', {'source_id': source_id, 'content': content})
+
+        return Summary(source_id=source_id, content=content)
+
+    async def _get_source_summaries(self, source_ids: List[str], map_reduce_documents_chain: MapReduceDocumentsChain) -> \
+            List[Summary]:
+        summaries = await asyncio.gather(
+            *[self._get_source_summary(source_id, map_reduce_documents_chain) for source_id in source_ids]
+        )
+        return summaries
+
+    def _call(
+            self,
+            inputs: Dict[str, Any],
+            run_manager: Optional[CallbackManagerForChainRun] = None
+    ) -> Dict[str, Any]:
+        # Step 1: Connect to vector store to ensure embeddings are accessible
+        self.vector_store_handler.connect()
+
+        context_chunks: List[Document] = inputs.get(self.context_key, [])
+        logger.debug(f"Found {len(context_chunks)} context chunks.")
+
+        # Step 2: Extract unique document IDs from the provided chunks
+        unique_document_ids = self._get_document_ids_from_chunks(context_chunks)
+        logger.debug(f"Extracted {len(unique_document_ids)} unique document IDs: {unique_document_ids}")
+
+        # Step 3: Initialize the summarization chain if not provided
+        question = inputs.get(self.question_key, '')
+        map_reduce_documents_chain = self.map_reduce_documents_chain or create_map_reduce_documents_chain(
+            self.summarization_config, question
+        )
+
+        # Step 4: Dispatch event to signal summarization start
+        if run_manager:
+            run_manager.on_text("Starting summarization for documents.", verbose=True)
+
+        # Step 5: Process each document ID to summarize chunks with local context
+        for doc_id in unique_document_ids:
+            logger.debug(f"Fetching and summarizing chunks for document ID: {doc_id}")
+
+            # Fetch all chunks for the document
+            chunks = asyncio.get_event_loop().run_until_complete(self._get_all_chunks_for_document(doc_id))
+            if not chunks:
+                logger.warning(f"No chunks found for document ID: {doc_id}")
+                continue
+
+            # Summarize each chunk with M neighboring chunks
+            M = self.neighbor_window
+            for i, chunk in enumerate(chunks):
+                window_chunks = chunks[max(0, i - M): min(len(chunks), i + M + 1)]
+                local_summary = asyncio.get_event_loop().run_until_complete(
+                    map_reduce_documents_chain.ainvoke(window_chunks)
+                )
+                chunk.metadata['summary'] = local_summary.get('output_text', '')
+                logger.debug(f"Chunk {i} summary: {chunk.metadata['summary'][:100]}...")
+
+        # Step 6: Update the original context chunks with the newly generated summaries
+        for chunk in context_chunks:
+            doc_id = str(chunk.metadata.get(self.doc_id_key, ''))
+            matching_chunk = next((c for c in chunks if c.metadata.get(self.doc_id_key) == doc_id and c.metadata.get(
+                'chunk_index') == chunk.metadata.get('chunk_index')), None)
+            if matching_chunk:
+                chunk.metadata['summary'] = matching_chunk.metadata.get('summary', '')
+            else:
+                chunk.metadata['summary'] = ''
+                logger.warning(f"No matching chunk found for doc_id: {doc_id}")
+
+        # Step 7: Signal summarization end
+        if run_manager:
+            run_manager.on_text("Summarization completed.", verbose=True)
+
+        logger.debug(f"Updated {len(context_chunks)} context chunks with summaries.")
+        return inputs

mindsdb/interfaces/agents/agents_controller.py

@@ -256,9 +256,9 @@ class AgentsController:
         if (
             is_demo and (
                 (name is not None and name != agent_name)
-                or (model_name or provider)
-                or (len(skills_to_add) > 0 or len(skills_to_remove) > 0 or len(skills_to_rewrite) > 0)
-                or (isinstance(params, dict) and len(params) > 1 and 'prompt_template' not in params)
+                or (model_name is not None and existing_agent.model_name != model_name)
+                or (provider is not None and existing_agent.provider != provider)
+                or (isinstance(params, dict) and len(params) > 0 and 'prompt_template' not in params)
             )
         ):
             raise ValueError("It is forbidden to change properties of the demo object")

mindsdb/interfaces/agents/callback_handlers.py

@@ -1,9 +1,13 @@
-from typing import Any, Dict, List, Union
+import io
 import logging
+import contextlib
+from typing import Any, Dict, List, Union, Callable
+
 from langchain_core.agents import AgentAction, AgentFinish
 from langchain_core.callbacks.base import BaseCallbackHandler
 from langchain_core.messages.base import BaseMessage
 from langchain_core.outputs import LLMResult
+from langchain_core.callbacks import StdOutCallbackHandler
 
 
 class ContextCaptureCallback(BaseCallbackHandler):
@@ -20,14 +24,49 @@ class ContextCaptureCallback(BaseCallbackHandler):
         return self.context
 
 
+class VerboseLogCallbackHandler(StdOutCallbackHandler):
+    def __init__(self, logger: logging.Logger, verbose: bool):
+        self.logger = logger
+        self.verbose = verbose
+        super().__init__()
+
+    def __call(self, method: Callable, *args: List[Any], **kwargs: Any) -> Any:
+        if self.verbose is False:
+            return
+        f = io.StringIO()
+        with contextlib.redirect_stdout(f):
+            method(*args, **kwargs)
+        output = f.getvalue()
+        self.logger.info(output)
+
+    def on_chain_start(self, *args: List[Any], **kwargs: Any) -> None:
+        self.__call(super().on_chain_start, *args, **kwargs)
+
+    def on_chain_end(self, *args: List[Any], **kwargs: Any) -> None:
+        self.__call(super().on_chain_end, *args, **kwargs)
+
+    def on_agent_action(self, *args: List[Any], **kwargs: Any) -> None:
+        self.__call(super().on_agent_action, *args, **kwargs)
+
+    def on_tool_end(self, *args: List[Any], **kwargs: Any) -> None:
+        self.__call(super().on_tool_end, *args, **kwargs)
+
+    def on_text(self, *args: List[Any], **kwargs: Any) -> None:
+        self.__call(super().on_text, *args, **kwargs)
+
+    def on_agent_finish(self, *args: List[Any], **kwargs: Any) -> None:
+        self.__call(super().on_agent_finish, *args, **kwargs)
+
+
 class LogCallbackHandler(BaseCallbackHandler):
     '''Langchain callback handler that logs agent and chain executions.'''
 
-    def __init__(self, logger: logging.Logger):
+    def __init__(self, logger: logging.Logger, verbose: bool = True):
         logger.setLevel('DEBUG')
         self.logger = logger
         self._num_running_chains = 0
         self.generated_sql = None
+        self.verbose_log_handler = VerboseLogCallbackHandler(logger, verbose)
 
     def on_llm_start(
         self, serialized: Dict[str, Any], prompts: List[str], **kwargs: Any
@@ -36,6 +75,7 @@ class LogCallbackHandler(BaseCallbackHandler):
         self.logger.debug('LLM started with prompts:')
         for prompt in prompts:
             self.logger.debug(prompt[:50])
+        self.verbose_log_handler.on_llm_start(serialized, prompts, **kwargs)
 
     def on_chat_model_start(
             self,
@@ -46,7 +86,7 @@ class LogCallbackHandler(BaseCallbackHandler):
         self.logger.debug('Chat model started with messages:')
         for message_list in messages:
             for message in message_list:
-                self.logger.debug(message.pretty_print())
+                self.logger.debug(message.pretty_repr())
 
     def on_llm_new_token(self, token: str, **kwargs: Any) -> Any:
         '''Run on new LLM token. Only available when streaming is enabled.'''
@@ -72,6 +112,8 @@ class LogCallbackHandler(BaseCallbackHandler):
             self._num_running_chains))
         self.logger.debug('Inputs: {}'.format(inputs))
 
+        self.verbose_log_handler.on_chain_start(serialized=serialized, inputs=inputs, **kwargs)
+
     def on_chain_end(self, outputs: Dict[str, Any], **kwargs: Any) -> Any:
         '''Run when chain ends running.'''
         self._num_running_chains -= 1
@@ -79,6 +121,8 @@ class LogCallbackHandler(BaseCallbackHandler):
             self._num_running_chains))
         self.logger.debug('Outputs: {}'.format(outputs))
 
+        self.verbose_log_handler.on_chain_end(outputs=outputs, **kwargs)
+
     def on_chain_error(
         self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any
     ) -> Any:
@@ -96,7 +140,7 @@ class LogCallbackHandler(BaseCallbackHandler):
 
     def on_tool_end(self, output: str, **kwargs: Any) -> Any:
         '''Run when tool ends running.'''
-        pass
+        self.verbose_log_handler.on_tool_end(output=output, **kwargs)
 
     def on_tool_error(
         self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any
@@ -106,7 +150,7 @@ class LogCallbackHandler(BaseCallbackHandler):
 
     def on_text(self, text: str, **kwargs: Any) -> Any:
         '''Run on arbitrary text.'''
-        pass
+        self.verbose_log_handler.on_text(text=text, **kwargs)
 
     def on_agent_action(self, action: AgentAction, **kwargs: Any) -> Any:
         '''Run on agent action.'''
@@ -124,7 +168,10 @@ class LogCallbackHandler(BaseCallbackHandler):
         # fix for mistral
         action.tool = action.tool.replace('\\', '')
 
+        self.verbose_log_handler.on_agent_action(action=action, **kwargs)
+
     def on_agent_finish(self, finish: AgentFinish, **kwargs: Any) -> Any:
         '''Run on agent end.'''
         self.logger.debug('Agent finished with return values:')
         self.logger.debug(str(finish.return_values))
+        self.verbose_log_handler.on_agent_finish(finish=finish, **kwargs)

mindsdb/interfaces/agents/langchain_agent.py

@@ -400,7 +400,7 @@ class LangchainAgent:
                 "max_iterations", args.get("max_iterations", DEFAULT_MAX_ITERATIONS)
             ),
             memory=memory,
-            verbose=args.get("verbose", args.get("verbose", True)),
+            verbose=args.get("verbose", args.get("verbose", False))
         )
         return agent_executor
 
@@ -435,7 +435,7 @@ class LangchainAgent:
         all_callbacks = []
 
         if self.log_callback_handler is None:
-            self.log_callback_handler = LogCallbackHandler(logger)
+            self.log_callback_handler = LogCallbackHandler(logger, verbose=args.get("verbose", True))
 
         all_callbacks.append(self.log_callback_handler)
 
@@ -599,7 +599,9 @@ AI: {response}"""
                 agent_executor_finished_event.set()
 
         # Enqueue Langchain agent streaming chunks in a separate thread to not block event chunks.
-        executor_stream_thread = threading.Thread(target=stream_worker, daemon=True, args=(ctx.dump(),))
+        executor_stream_thread = threading.Thread(
+            target=stream_worker, daemon=True, args=(ctx.dump(),), name='LangchainAgent.stream_worker'
+        )
         executor_stream_thread.start()
 
         while not agent_executor_finished_event.is_set():

mindsdb/interfaces/database/database.py

@@ -106,7 +106,7 @@ class DatabaseController:
                 }
 
     def exists(self, db_name: str) -> bool:
-        return db_name in self.get_dict()
+        return db_name.lower() in self.get_dict()
 
     def get_project(self, name: str):
         return self.project_controller.get(name=name)

mindsdb/interfaces/database/integrations.py

@@ -64,7 +64,7 @@ class HandlersCache:
         ):
             return
         self._stop_event.clear()
-        self.cleaner_thread = threading.Thread(target=self._clean)
+        self.cleaner_thread = threading.Thread(target=self._clean, name='HandlersCache.clean')
         self.cleaner_thread.daemon = True
         self.cleaner_thread.start()