MindsDB 25.2.3.0__py3-none-any.whl → 25.3.1.0__py3-none-any.whl

mindsdb/integrations/libs/process_cache.py

@@ -186,7 +186,6 @@ class ProcessCache:
         self._keep_alive = {}
         self._stop_event = threading.Event()
         self.cleaner_thread = None
-        self._start_clean()
 
     def __del__(self):
         self._stop_clean()
@@ -200,7 +199,7 @@ class ProcessCache:
         ):
             return
         self._stop_event.clear()
-        self.cleaner_thread = threading.Thread(target=self._clean)
+        self.cleaner_thread = threading.Thread(target=self._clean, name='ProcessCache.clean')
         self.cleaner_thread.daemon = True
         self.cleaner_thread.start()
 
@@ -258,6 +257,7 @@ class ProcessCache:
             Returns:
                 Future
         """
+        self._start_clean()
         handler_module_path = payload['handler_meta']['module_path']
         integration_id = payload['handler_meta']['integration_id']
         if task_type in (ML_TASK_TYPE.LEARN, ML_TASK_TYPE.FINETUNE):

mindsdb/integrations/utilities/handlers/auth_utilities/google/google_user_oauth_utilities.py

@@ -1,9 +1,8 @@
-import os
 import json
+from pathlib import Path
 import requests
 import datetime as dt
 from flask import request
-from shutil import copyfile
 
 from mindsdb.utilities import log
 
@@ -29,73 +28,65 @@ class GoogleUserOAuth2Manager:
         creds = None
 
         if self.credentials_file or self.credentials_url:
-            # get the current directory and checks tokens & creds
-            curr_dir = self.handler_storage.folder_get('config')
+            oauth_user_info = self.handler_storage.encrypted_json_get('oauth_user_info')
 
-            creds_file = os.path.join(curr_dir, 'creds.json')
-            secret_file = os.path.join(curr_dir, 'secret.json')
-
-            if os.path.isfile(creds_file):
-                creds = Credentials.from_authorized_user_file(creds_file, self.scopes)
+            if oauth_user_info:
+                creds = Credentials.from_authorized_user_info(oauth_user_info, self.scopes)
 
             if not creds or not creds.valid:
                 logger.debug("Credentials do not exist or are invalid, attempting to authorize again")
 
-                if self._download_secret_file(secret_file):
-                    # save to storage
-                    self.handler_storage.folder_sync('config')
-                else:
-                    raise ValueError('No valid Gmail Credentials filepath or S3 url found.')
+                oauth_user_info = self._download_oauth_user_info()
 
                 if creds and creds.expired and creds.refresh_token:
                     creds.refresh(Request())
                     logger.debug("Credentials refreshed successfully")
                 else:
-                    creds = self._execute_google_auth_flow(secret_file, self.scopes, self.code)
+                    creds = self._execute_google_auth_flow(oauth_user_info)
                     logger.debug("New credentials obtained")
 
-                self._save_credentials_to_file(creds, creds_file)
-                logger.debug(f"saved session credentials to {creds_file}")
-                self.handler_storage.folder_sync('config')
+                self.handler_storage.encrypted_json_set('oauth_user_info', self._convert_credentials_to_dict(creds))
+                logger.debug("Saving credentials to storage")
 
         return creds
 
-    def _download_secret_file(self, secret_file):
-        # if credentials_url is set, attempt to download the file
+    def _download_oauth_user_info(self):
+        # if credentials_url is set, attempt to download the contents of the files
         # this will be given preference over credentials_file
         if self.credentials_url:
             response = requests.get(self.credentials_url)
             if response.status_code == 200:
-                with open(secret_file, 'w') as creds:
-                    creds.write(response.text)
-                return True
+                return response.json()
             else:
-                logger.error("Failed to get credentials from S3", response.status_code)
+                logger.error("Failed to get credentials from URL", response.status_code)
+
+        # if credentials_file is set, attempt to read the contents of the file
+        if self.credentials_file:
+            path = Path(self.credentials_file).expanduser()
+            if path.exists():
+                with open(path, 'r') as f:
+                    return json.load(f)
+            else:
+                logger.error("Credentials file does not exist")
 
-        # if credentials_file is set, attempt to copy the file
-        if self.credentials_file and os.path.isfile(self.credentials_file):
-            copyfile(self.credentials_file, secret_file)
-            return True
-        return False
+        raise ValueError('OAuth2 credentials could not be found')
 
-    def _execute_google_auth_flow(self, secret_file, scopes, code=None):
-        flow = Flow.from_client_secrets_file(secret_file, scopes)
+    def _execute_google_auth_flow(self, oauth_user_info: dict):
+        flow = Flow.from_client_config(
+            oauth_user_info,
+            scopes=self.scopes
+        )
 
         flow.redirect_uri = request.headers['ORIGIN'] + '/verify-auth'
 
-        if code:
-            flow.fetch_token(code=code)
+        if self.code:
+            flow.fetch_token(code=self.code)
             creds = flow.credentials
             return creds
         else:
             auth_url = flow.authorization_url()[0]
             raise AuthException(f'Authorisation required. Please follow the url: {auth_url}', auth_url=auth_url)
 
-    def _save_credentials_to_file(self, creds, file_path):
-        with open(file_path, 'w') as token:
-            data = self._convert_credentials_to_dict(creds)
-            token.write(json.dumps(data))
-
     def _convert_credentials_to_dict(self, credentials):
         return {
             'token': credentials.token,

mindsdb/integrations/utilities/pydantic_utils.py

@@ -0,0 +1,208 @@
+import pprint
+
+pydantic_schema_description = """## Understanding Pydantic Schemas for JSON Formatting
+
+Pydantic schemas provide a framework for defining the structure and validation rules for JSON output. Below is an overview of key components commonly found in a Pydantic schema:
+
+### Key Components
+
+Each object in the schema represents a Pydantic model in JSON format. Typical fields in a Pydantic model description include:
+
+- **`anyOf`**:
+  - A list describing possible values for a Pydantic model field.
+
+- **`additionalProperties`**:
+  - Describes the keys of a dictionary. Keys are always of type `string` due to this being a JSON Pydantic schema. The corresponding key types supported by Pydantic are:
+    - `string`: a text string
+    - `integer`: an integer number
+    - `number`: a floating-point number
+
+- **`items`**:
+  - Describes the items contained within an `array` (list).
+
+- **`type`**:
+  - Specifies the Pydantic type assigned to the field, defining the expected data type. Common types include:
+    - `string`: a text string
+    - `integer`: an integer number
+    - `number`: a floating-point number
+    - `array`: a list
+    - `object`: a dictionary
+    - `null`: the python null value None. Indicates the field is optional.
+
+- **`description`**:
+  - Provides a textual narrative explaining the purpose and details of the output JSON field.
+
+- **`title`**:
+  - A Pydantic-generated, human-readable title for the field.
+
+- **`default`**:
+  - The default value for this field if no value is provided by the user.
+
+### Schema
+
+Below is the Pydantic schema:
+
+{schema}
+
+### Examples
+
+Below is an example of well-formed output adhering to this schema.
+
+- Dummy text strings are represented as "lorem ipsum."
+
+{example}
+"""
+
+
+def get_dummy_value(field_value):
+    """A function to return a dummy value of a Pydantic model field."""
+    type_str = field_value["type"]
+    example_dict = {
+        "string": "lorem ipsum",
+        "int": 3,
+        "number": 42.0,
+        "null": None,
+        "object": {"lorem ipsum": "lorem_ipsum"},
+    }
+
+    if type_str in example_dict:
+        return example_dict[type_str]
+    else:
+        return None
+
+
+def get_dummy_array(field_value):
+    """A function to return a dummy array of a Pydantic model field."""
+    items = field_value["items"]
+
+    if "type" in items:
+        if items["type"] == "null":  # skip if null
+            pass
+        elif items["type"] == "array":  # is it an array?
+            array_value = get_dummy_array(items)
+        elif (
+            items["type"] == "object" and "additionalProperties" in items
+        ):  # is it a dict?
+            array_value = get_dummy_dict(items)
+        else:  # it is a regular value!
+            array_value = get_dummy_value(items)
+        return [array_value for _ in range(2)]
+
+    elif "AnyOf" in field_value["items"]:
+        array_value = get_any_of(field_value["items"])  # can be one of many types
+        return [array_value for _ in range(2)]
+
+    else:  # is it a pydantic class?
+        array_value = example_generator(items)
+        return [array_value for _ in range(2)]
+
+
+def get_dummy_dict(field_value):
+    """A function to return a dummy dictionary of a Pydantic model field."""
+    return get_dummy_value(field_value)
+
+
+def get_any_of(field_value):
+    """A function to return the first viable pydantic type of an Any() Pydantic model field."""
+    for any_of in field_value["anyOf"]:
+        if "type" in any_of:
+            if any_of["type"] == "null":  # skip if null
+                continue
+            elif any_of["type"] == "array":  # is it an array?
+                out = get_dummy_array(any_of)
+                return out
+            elif (
+                any_of["type"] == "object" and "additionalProperties" in any_of
+            ):  # is it a dict?
+                out = get_dummy_dict(any_of)
+                return out
+            else:  # it is a regular value!
+                out = get_dummy_value(any_of)
+                return out
+        else:  # is it a pydantic class?
+            out = example_generator(any_of)
+            return out
+
+
+def example_generator(pydantic_json_schema):
+    """dynamically parse a pydantic object and generate an example of it's formatting."""
+
+    example_dict = {}
+    for schema_name, schema in pydantic_json_schema.items():
+
+        for field_name, field_value in schema.items():
+            if "type" in field_value:
+
+                if field_value["type"] == "array":  # is it an array?
+                    example_dict[field_name] = get_dummy_array(field_value)
+
+                elif (
+                    field_value["type"] == "object"
+                    and "additionalProperties" in field_value
+                ):  # is it a dict?
+                    example_dict[field_name] = get_dummy_dict(field_value)
+
+                else:  # it is a regular value!
+                    example_dict[field_name] = get_dummy_value(field_value)
+
+            elif "anyOf" in field_value:
+                example_dict[field_name] = get_any_of(field_value)
+
+            else:  # it is a pydantic class
+                example_dict[field_name] = example_generator(field_value)
+    return example_dict
+
+
+def search_and_replace_refs(schema, defs, ref_skip={}, n=0):
+    """Dynamically substitute subclass references in a Pydantic object schema."""
+    for key, value in schema.items():
+        if key in ref_skip:
+            continue
+        if type(value) is dict:
+            if "$ref" in value:
+                definition_key = value["$ref"].split("/")[-1]
+                if definition_key in ref_skip:
+                    schema[key] = {"type": "null"}
+                else:
+                    schema[key] = {definition_key: defs[definition_key]["properties"]}
+            else:
+                search_and_replace_refs(value, defs, ref_skip, n + 1)
+        elif type(value) is list:
+            for val in value:
+                search_and_replace_refs(val, defs, ref_skip, n + 1)
+
+
+def remove_extraneous_fields(schema, ref_skip):
+    """Remove extraneous fields from object descriptions."""
+    reduced_schema = schema["properties"]
+
+    for ref in ref_skip.keys():
+        if ref in reduced_schema:
+            del reduced_schema[ref]
+
+    for key, value in reduced_schema.items():
+        if "title" in value:
+            del value["title"]
+        if "$defs" in value:
+            del value["$defs"]
+        if "required" in value:
+            del value["required"]
+
+    return reduced_schema
+
+
+def format_for_prompt(pydantic_object, ref_skip={}):
+    """Format a Pydantic object description for prompting an LLM."""
+    schema = {k: v for k, v in pydantic_object.schema().items()}
+
+    search_and_replace_refs(
+        schema=schema["properties"], defs=schema["$defs"], ref_skip=ref_skip, n=0
+    )
+
+    reduced_schema = remove_extraneous_fields(schema, ref_skip)
+
+    reduced_schema = {schema["title"]: reduced_schema}
+
+    out = pprint.pformat(reduced_schema)
+
+    return out, reduced_schema

mindsdb/integrations/utilities/rag/chains/local_context_summarizer_chain.py

@@ -0,0 +1,227 @@
+import asyncio
+from collections import namedtuple
+from typing import Any, Dict, List, Optional
+
+from mindsdb.interfaces.agents.langchain_agent import create_chat_model
+from langchain.chains.base import Chain
+from langchain.chains.combine_documents.stuff import StuffDocumentsChain
+from langchain.chains.llm import LLMChain
+from langchain.chains.combine_documents.map_reduce import MapReduceDocumentsChain, ReduceDocumentsChain
+from langchain_core.callbacks import dispatch_custom_event
+from langchain_core.callbacks.manager import CallbackManagerForChainRun
+from langchain_core.documents import Document
+from langchain_core.prompts import PromptTemplate
+from pandas import DataFrame
+
+from mindsdb.integrations.libs.vectordatabase_handler import VectorStoreHandler
+from mindsdb.integrations.utilities.rag.settings import SummarizationConfig
+from mindsdb.integrations.utilities.sql_utils import FilterCondition, FilterOperator
+from mindsdb.utilities import log
+
+logger = log.getLogger(__name__)
+
+Summary = namedtuple('Summary', ['source_id', 'content'])
+
+
+def create_map_reduce_documents_chain(summarization_config: SummarizationConfig, input: str) -> ReduceDocumentsChain:
+    """Creates a chain that map-reduces documents into a single consolidated summary."""
+    summarization_llm = create_chat_model({
+        'model_name': summarization_config.llm_config.model_name,
+        'provider': summarization_config.llm_config.provider,
+        **summarization_config.llm_config.params
+    })
+
+    reduce_prompt_template = summarization_config.reduce_prompt_template
+    reduce_prompt = PromptTemplate.from_template(reduce_prompt_template)
+    if 'input' in reduce_prompt.input_variables:
+        reduce_prompt = reduce_prompt.partial(input=input)
+
+    reduce_chain = LLMChain(llm=summarization_llm, prompt=reduce_prompt)
+
+    combine_documents_chain = StuffDocumentsChain(
+        llm_chain=reduce_chain,
+        document_variable_name='docs'
+    )
+
+    return ReduceDocumentsChain(
+        combine_documents_chain=combine_documents_chain,
+        collapse_documents_chain=combine_documents_chain,
+        token_max=summarization_config.max_summarization_tokens
+    )
+
+
+class LocalContextSummarizerChain(Chain):
+    """Summarizes M chunks before and after a given chunk in a document."""
+
+    doc_id_key: str = 'original_row_id'
+    chunk_index_key: str = 'chunk_index'
+
+    vector_store_handler: VectorStoreHandler
+    table_name: str = 'embeddings'
+    content_column_name: str = 'content'
+    metadata_column_name: str = 'metadata'
+
+    summarization_config: SummarizationConfig
+    map_reduce_documents_chain: Optional[ReduceDocumentsChain] = None
+
+    def _select_chunks_from_vector_store(self, doc_id: str) -> DataFrame:
+        condition = FilterCondition(
+            f"{self.metadata_column_name}->>'{self.doc_id_key}'",
+            FilterOperator.EQUAL,
+            doc_id
+        )
+        return self.vector_store_handler.select(
+            self.table_name,
+            columns=[self.content_column_name, self.metadata_column_name],
+            conditions=[condition]
+        )
+
+    async def _get_all_chunks_for_document(self, doc_id: str) -> List[Document]:
+        df = await asyncio.get_event_loop().run_in_executor(
+            None, self._select_chunks_from_vector_store, doc_id
+        )
+        chunks = []
+        for _, row in df.iterrows():
+            metadata = row.get(self.metadata_column_name, {})
+            metadata[self.chunk_index_key] = row.get('chunk_id', 0)
+            chunks.append(Document(page_content=row[self.content_column_name], metadata=metadata))
+
+        return sorted(chunks, key=lambda x: x.metadata.get(self.chunk_index_key, 0))
+
+    async def summarize_local_context(self, doc_id: str, target_chunk_index: int, M: int) -> Summary:
+        """
+        Summarizes M chunks before and after the given chunk.
+
+        Args:
+            doc_id (str): Document ID.
+            target_chunk_index (int): Index of the chunk to summarize around.
+            M (int): Number of chunks before and after to include.
+
+        Returns:
+            Summary: Summary object containing source_id and summary content.
+        """
+        logger.debug(f"Fetching chunks for document {doc_id}")
+        all_chunks = await self._get_all_chunks_for_document(doc_id)
+
+        if not all_chunks:
+            logger.warning(f"No chunks found for document {doc_id}")
+            return Summary(source_id=doc_id, content='')
+
+        # Determine window boundaries
+        start_idx = max(0, target_chunk_index - M)
+        end_idx = min(len(all_chunks), target_chunk_index + M + 1)
+        local_chunks = all_chunks[start_idx:end_idx]
+
+        logger.debug(f"Summarizing chunks {start_idx} to {end_idx - 1} for document {doc_id}")
+
+        if not self.map_reduce_documents_chain:
+            self.map_reduce_documents_chain = create_map_reduce_documents_chain(
+                self.summarization_config, input="Summarize these chunks."
+            )
+
+        summary_result = await self.map_reduce_documents_chain.ainvoke(local_chunks)
+        summary_text = summary_result.get('output_text', '')
+
+        logger.debug(f"Generated summary: {summary_text[:100]}...")
+
+        return Summary(source_id=doc_id, content=summary_text)
+
+    @property
+    def input_keys(self) -> List[str]:
+        return [self.context_key, self.question_key]
+
+    @property
+    def output_keys(self) -> List[str]:
+        return [self.context_key, self.question_key]
+
+    async def _get_source_summary(self, source_id: str, map_reduce_documents_chain: MapReduceDocumentsChain) -> Summary:
+        if not source_id:
+            logger.warning("Received empty source_id, returning empty summary")
+            return Summary(source_id='', content='')
+
+        logger.debug(f"Getting summary for source ID: {source_id}")
+        source_chunks = await self._get_all_chunks_for_document(source_id)
+
+        if not source_chunks:
+            logger.warning(f"No chunks found for source ID: {source_id}")
+            return Summary(source_id=source_id, content='')
+
+        logger.debug(f"Summarizing {len(source_chunks)} chunks for source ID: {source_id}")
+        summary = await map_reduce_documents_chain.ainvoke(source_chunks)
+        content = summary.get('output_text', '')
+        logger.debug(f"Generated summary for source ID {source_id}: {content[:100]}...")
+
+        # Stream summarization update.
+        dispatch_custom_event('summary', {'source_id': source_id, 'content': content})
+
+        return Summary(source_id=source_id, content=content)
+
+    async def _get_source_summaries(self, source_ids: List[str], map_reduce_documents_chain: MapReduceDocumentsChain) -> \
+            List[Summary]:
+        summaries = await asyncio.gather(
+            *[self._get_source_summary(source_id, map_reduce_documents_chain) for source_id in source_ids]
+        )
+        return summaries
+
+    def _call(
+            self,
+            inputs: Dict[str, Any],
+            run_manager: Optional[CallbackManagerForChainRun] = None
+    ) -> Dict[str, Any]:
+        # Step 1: Connect to vector store to ensure embeddings are accessible
+        self.vector_store_handler.connect()
+
+        context_chunks: List[Document] = inputs.get(self.context_key, [])
+        logger.debug(f"Found {len(context_chunks)} context chunks.")
+
+        # Step 2: Extract unique document IDs from the provided chunks
+        unique_document_ids = self._get_document_ids_from_chunks(context_chunks)
+        logger.debug(f"Extracted {len(unique_document_ids)} unique document IDs: {unique_document_ids}")
+
+        # Step 3: Initialize the summarization chain if not provided
+        question = inputs.get(self.question_key, '')
+        map_reduce_documents_chain = self.map_reduce_documents_chain or create_map_reduce_documents_chain(
+            self.summarization_config, question
+        )
+
+        # Step 4: Dispatch event to signal summarization start
+        if run_manager:
+            run_manager.on_text("Starting summarization for documents.", verbose=True)
+
+        # Step 5: Process each document ID to summarize chunks with local context
+        for doc_id in unique_document_ids:
+            logger.debug(f"Fetching and summarizing chunks for document ID: {doc_id}")
+
+            # Fetch all chunks for the document
+            chunks = asyncio.get_event_loop().run_until_complete(self._get_all_chunks_for_document(doc_id))
+            if not chunks:
+                logger.warning(f"No chunks found for document ID: {doc_id}")
+                continue
+
+            # Summarize each chunk with M neighboring chunks
+            M = self.neighbor_window
+            for i, chunk in enumerate(chunks):
+                window_chunks = chunks[max(0, i - M): min(len(chunks), i + M + 1)]
+                local_summary = asyncio.get_event_loop().run_until_complete(
+                    map_reduce_documents_chain.ainvoke(window_chunks)
+                )
+                chunk.metadata['summary'] = local_summary.get('output_text', '')
+                logger.debug(f"Chunk {i} summary: {chunk.metadata['summary'][:100]}...")
+
+        # Step 6: Update the original context chunks with the newly generated summaries
+        for chunk in context_chunks:
+            doc_id = str(chunk.metadata.get(self.doc_id_key, ''))
+            matching_chunk = next((c for c in chunks if c.metadata.get(self.doc_id_key) == doc_id and c.metadata.get(
+                'chunk_index') == chunk.metadata.get('chunk_index')), None)
+            if matching_chunk:
+                chunk.metadata['summary'] = matching_chunk.metadata.get('summary', '')
+            else:
+                chunk.metadata['summary'] = ''
+                logger.warning(f"No matching chunk found for doc_id: {doc_id}")
+
+        # Step 7: Signal summarization end
+        if run_manager:
+            run_manager.on_text("Summarization completed.", verbose=True)
+
+        logger.debug(f"Updated {len(context_chunks)} context chunks with summaries.")
+        return inputs

mindsdb/integrations/utilities/rag/pipelines/rag.py

@@ -294,16 +294,23 @@ class LangChainRAGPipeline:
         retriever = SQLRetriever(
             fallback_retriever=vector_store_retriever,
             vector_store_handler=knowledge_base_table.get_vector_db(),
-            metadata_schemas=retriever_config.metadata_schemas,
-            examples=retriever_config.examples,
+            min_k=retriever_config.min_k,
+            max_filters=retriever_config.max_filters,
+            filter_threshold=retriever_config.filter_threshold,
+            database_schema=retriever_config.database_schema,
             embeddings_model=embeddings,
+            search_kwargs=config.search_kwargs,
             rewrite_prompt_template=retriever_config.rewrite_prompt_template,
-            metadata_filters_prompt_template=retriever_config.metadata_filters_prompt_template,
+            table_prompt_template=retriever_config.table_prompt_template,
+            column_prompt_template=retriever_config.column_prompt_template,
+            value_prompt_template=retriever_config.value_prompt_template,
+            boolean_system_prompt=retriever_config.boolean_system_prompt,
+            generative_system_prompt=retriever_config.generative_system_prompt,
             num_retries=retriever_config.num_retries,
             embeddings_table=knowledge_base_table._kb.vector_database_table,
             source_table=retriever_config.source_table,
+            source_id_column=retriever_config.source_id_column,
             distance_function=distance_function,
-            search_kwargs=config.search_kwargs,
             llm=sql_llm
         )
         return cls(