MindsDB 25.3.2.0__py3-none-any.whl → 25.3.4.0__py3-none-any.whl

mindsdb/integrations/utilities/handlers/api_utilities/microsoft/ms_graph_api_utilities.py

@@ -87,7 +87,7 @@ class MSGraphAPIBaseClient:
 
         return response
 
-    def fetch_paginated_data(self, endpoint: Text, params: Optional[Dict] = {}) -> Generator:
+    def fetch_paginated_data(self, endpoint: Text, params: Optional[Dict] = None) -> Generator:
         """
         Fetches data from the Microsoft Graph API by making the specified request and handling pagination.
 
@@ -98,6 +98,8 @@ class MSGraphAPIBaseClient:
         Yields:
             List: The data fetched from the Microsoft Graph API.
         """
+        if params is None:
+            params = {}
         api_url = self._get_api_url(endpoint)
 
         # Add the pagination count to the request parameters.
@@ -115,7 +117,7 @@ class MSGraphAPIBaseClient:
             api_url = response_json.get("@odata.nextLink", "")
             yield value
 
-    def fetch_data(self, endpoint: str, params: Optional[Dict] = {}) -> Union[List, Dict, bytes]:
+    def _fetch_data(self, endpoint: str, params: Optional[Dict] = {}) -> Union[List, Dict, bytes]:
         """
         Fetches data from the Microsoft Graph API by making the specified request.
 
@@ -129,4 +131,36 @@ class MSGraphAPIBaseClient:
         api_url = self._get_api_url(endpoint)
 
         response = self._make_request(api_url, params)
+        return response
+
+    def fetch_data_content(self, endpoint: str, params: Optional[Dict] = {}) -> bytes:
+        """
+        Fetches data content from the Microsoft Graph API by making the specified request.
+
+        Args:
+            endpoint (str): The endpoint of the Microsoft Graph API to fetch data from.
+            params (Optional[Dict]): The parameters to include in the request.
+
+        Returns:
+            bytes: The data content fetched from the Microsoft Graph API.
+        """
+        response = self._fetch_data(endpoint, params)
         return response.content
+
+    def fetch_data_json(self, endpoint: str, params: Optional[Dict] = {}) -> Union[List, Dict]:
+        """
+        Fetches data from the Microsoft Graph API by making the specified request and returns the JSON response.
+
+        Args:
+            endpoint (str): The endpoint of the Microsoft Graph API to fetch data from.
+            params (Optional[Dict]): The parameters to include in the request.
+
+        Returns:
+            Union[List, Dict]: The JSON response fetched from the Microsoft Graph API.
+        """
+        response = self._fetch_data(endpoint, params)
+        response_json = response.json()
+
+        if "value" in response_json:
+            return response_json["value"]
+        return response_json

mindsdb/integrations/utilities/rag/settings.py

@@ -551,6 +551,7 @@ class ColumnSchema(BaseModel):
             Dict[Union[str, int, float], ValueSchema],
         ]
     ] = Field(
+        default=None,
         description="One of the following. A dict or ordered dict of {schema_value: ValueSchema, ...}, where schema value is the name given for this value description in the schema."
     )
     example_questions: Optional[List[LLMExample]] = Field(

mindsdb/interfaces/chatbot/chatbot_controller.py

@@ -1,14 +1,13 @@
 from typing import Dict, List
 
+from mindsdb.api.executor.controllers.session_controller import SessionController
 from mindsdb.interfaces.agents.agents_controller import AgentsController
 from mindsdb.interfaces.chatbot.chatbot_task import ChatBotTask
 from mindsdb.interfaces.database.projects import ProjectController
 from mindsdb.interfaces.storage import db
 from mindsdb.interfaces.model.functions import get_project_records
-
+from mindsdb.utilities.exception import EntityNotExistsError
 from mindsdb.utilities.context import context as ctx
-
-from mindsdb.api.executor.controllers.session_controller import SessionController
 from mindsdb.utilities.config import config
 
 
@@ -126,7 +125,7 @@ class ChatBotController:
         Gets all chatbots in a project.
 
         Parameters:
-            project_name (str): The name of the containing project
+            project_name (str): The name of the containing project. If None, then return from all projects
 
         Returns:
             all_bots (List[db.ChatBots]): List of database chatbot object
@@ -138,6 +137,9 @@ class ChatBotController:
                 continue
             project_names[project.id] = project.name
 
+        if project_name is not None and project_name not in project_names.values():
+            raise EntityNotExistsError(f'Project {project_name} not found')
+
         query = db.session.query(
             db.ChatBots, db.Tasks
         ).join(

mindsdb/interfaces/jobs/jobs_controller.py

@@ -340,10 +340,7 @@ class JobsController:
         data, columns = logs_db_controller.query(query)
 
         names = [i['name'] for i in columns]
-        records = []
-        for row in data:
-            records.append(dict(zip(names, row)))
-        return records
+        return data[names].to_dict(orient='records')
 
 
 class JobsExecutor:

mindsdb/interfaces/knowledge_base/controller.py

@@ -26,9 +26,7 @@ from mindsdb.integrations.libs.vectordatabase_handler import (
 )
 from mindsdb.integrations.utilities.rag.rag_pipeline_builder import RAG
 from mindsdb.integrations.utilities.rag.config_loader import load_rag_config
-from mindsdb.integrations.utilities.sql_utils import (
-    extract_comparison_conditions, filter_dataframe, FilterCondition, FilterOperator
-)
+
 from mindsdb.interfaces.agents.constants import DEFAULT_EMBEDDINGS_MODEL_CLASS
 from mindsdb.interfaces.agents.langchain_agent import create_chat_model, get_llm_provider
 from mindsdb.interfaces.database.projects import ProjectController
@@ -105,19 +103,9 @@ class KnowledgeBaseTable:
         db_handler = self.get_vector_db()
         logger.debug(f"Using vector db handler: {type(db_handler)}")
 
-        vector_filters, outer_filters = [], []
-        # update vector handlers, mark conditions as applied inside
-        for op, arg1, arg2 in extract_comparison_conditions(query.where):
-            condition = FilterCondition(arg1, FilterOperator(op.upper()), arg2)
-            if arg1 in (TableField.ID.value, TableField.CONTENT.value, TableField.EMBEDDINGS.value):
-                vector_filters.append(condition)
-            else:
-                outer_filters.append([op, arg1, arg2])
-
-        df = db_handler.dispatch_select(query, conditions=vector_filters)
+        df = db_handler.dispatch_select(query)
 
         if df is not None:
-            df = filter_dataframe(df, outer_filters)
 
             logger.debug(f"Query returned {len(df)} rows")
             logger.debug(f"Columns in response: {df.columns.tolist()}")
@@ -229,7 +217,7 @@ class KnowledgeBaseTable:
 
         # send to vectordb
         db_handler = self.get_vector_db()
-        db_handler.query(query)
+        db_handler.dispatch_delete(query)
 
     def hybrid_search(
         self,
@@ -293,7 +281,6 @@ class KnowledgeBaseTable:
                         **base_metadata,
                         'original_row_id': str(row_id),
                         'content_column': col,
-                        'content_type': col.split('_')[-1] if '_' in col else 'text'
                     }
 
                     raw_documents.append(Document(
@@ -364,7 +351,7 @@ class KnowledgeBaseTable:
             logger.debug(f"Added IDs: {df_out[TableField.ID.value].tolist()}")
 
         # -- prepare content and metadata --
-        content_columns = params.get('content_columns')
+        content_columns = params.get('content_columns', [TableField.CONTENT.value])
         metadata_columns = params.get('metadata_columns')
 
         logger.debug(f"Processing with: content_columns={content_columns}, metadata_columns={metadata_columns}")
@@ -399,17 +386,6 @@ class KnowledgeBaseTable:
                 # all the rest columns
                 metadata_columns = list(set(columns).difference(content_columns))
 
-        elif metadata_columns is not None:
-            metadata_columns = list(set(metadata_columns).intersection(columns))
-            # use all unused columns is content
-            content_columns = list(set(columns).difference(metadata_columns))
-        elif TableField.METADATA.value in columns:
-            metadata_columns = [TableField.METADATA.value]
-            content_columns = list(set(columns).difference(metadata_columns))
-        else:
-            # all columns go to content
-            content_columns = columns
-
         # Add content columns directly (don't combine them)
         for col in content_columns:
             df_out[col] = df[col]
@@ -429,6 +405,9 @@ class KnowledgeBaseTable:
                         value = float(value)
                     elif pd.api.types.is_bool_dtype(value):
                         value = bool(value)
+                    elif isinstance(value, dict):
+                        metadata.update(value)
+                        continue
                     else:
                         value = str(value)
                     metadata[col] = value
@@ -852,6 +831,8 @@ class KnowledgeBaseController:
         # drop objects if they were created automatically
         if 'default_vector_storage' in kb.params:
             try:
+                handler = self.session.datahub.get(kb.params['default_vector_storage']).integration_handler
+                handler.drop_table(kb.vector_database_table)
                 self.session.integration_controller.delete(kb.params['default_vector_storage'])
             except EntityNotExistsError:
                 pass

mindsdb/interfaces/knowledge_base/preprocessing/document_preprocessor.py

@@ -98,7 +98,7 @@ class DocumentPreprocessor:
         provided_id: str = None,
     ) -> str:
         """Generate deterministic ID for a chunk"""
-        base_id = self._generate_deterministic_id(content, content_column, provided_id)
+        base_id = provided_id
         chunk_id = (
             f"{base_id}_chunk_{chunk_index}" if chunk_index is not None else base_id
         )

mindsdb/interfaces/skills/skills_controller.py

@@ -42,12 +42,12 @@ class SkillsController:
             db.Skills.deleted_at == null()
         ).first()
 
-    def get_skills(self, project_name: str) -> List[dict]:
+    def get_skills(self, project_name: Optional[str]) -> List[dict]:
         '''
         Gets all skills in a project.
 
         Parameters:
-            project_name (str): The name of the containing project
+            project_name (Optional[str]): The name of the containing project
 
         Returns:
             all_skills (List[db.Skills]): List of database skill object
@@ -56,11 +56,12 @@ class SkillsController:
             ValueError: If `project_name` does not exist
         '''
 
-        project_controller = ProjectController()
-        projects = project_controller.get_list()
-        if project_name is not None:
-            projects = list([p for p in projects if p.name == project_name])
-        project_ids = list([p.id for p in projects])
+        if project_name is None:
+            projects = self.project_controller.get_list()
+            project_ids = list([p.id for p in projects])
+        else:
+            project = self.project_controller.get(name=project_name)
+            project_ids = [project.id]
 
         query = (
             db.session.query(db.Skills)

mindsdb/utilities/render/sqlalchemy_render.py

@@ -483,7 +483,7 @@ class SqlalchemyRender:
 
         return schema, table_name
 
-    def to_table(self, node):
+    def to_table(self, node, is_lateral=False):
         if isinstance(node, ast.Identifier):
             schema, table_name = self.get_table_name(node)
 
@@ -497,7 +497,10 @@ class SqlalchemyRender:
             alias = None
             if node.alias:
                 alias = self.get_alias(node.alias)
-            table = sub_stmt.subquery(alias)
+            if is_lateral:
+                table = sub_stmt.lateral(alias)
+            else:
+                table = sub_stmt.subquery(alias)
 
         else:
             # TODO tests are failing
@@ -526,8 +529,11 @@ class SqlalchemyRender:
 
                 query = query.add_cte(stmt.cte(self.get_alias(alias), nesting=True))
 
-        if node.distinct:
+        if node.distinct is True:
             query = query.distinct()
+        elif isinstance(node.distinct, list):
+            columns = [self.to_expression(c) for c in node.distinct]
+            query = query.distinct(*columns)
 
         if node.from_table is not None:
             from_table = node.from_table
@@ -541,7 +547,8 @@ class SqlalchemyRender:
                 # other tables
                 has_explicit_join = False
                 for item in join_list[1:]:
-                    table = self.to_table(item['table'])
+                    join_type = item['join_type']
+                    table = self.to_table(item['table'], is_lateral=('LATERAL' in join_type))
                     if item['is_implicit']:
                         # add to from clause
                         if has_explicit_join:
@@ -558,7 +565,6 @@ class SqlalchemyRender:
                         else:
                             condition = self.to_expression(item['condition'])
 
-                        join_type = item['join_type']
                         if 'ASOF' in join_type:
                             raise NotImplementedError(f'Unsupported join type: {join_type}')
                         method = 'join'