MindsDB 25.3.3.0__py3-none-any.whl → 25.3.4.1__py3-none-any.whl

mindsdb/integrations/handlers/ms_teams_handler/ms_teams_tables.py

@@ -0,0 +1,431 @@
+from typing import List
+
+import pandas as pd
+
+from mindsdb.integrations.handlers.ms_teams_handler.ms_graph_api_teams_client import MSGraphAPITeamsDelegatedPermissionsClient
+from mindsdb.integrations.libs.api_handler import APIResource
+from mindsdb.integrations.utilities.sql_utils import (
+    FilterCondition,
+    FilterOperator,
+    SortColumn
+)
+
+
+class TeamsTable(APIResource):
+    """
+    The table abstraction for the 'teams' resource of the Microsoft Graph API.
+    """
+    def list(
+        self,
+        conditions: List[FilterCondition] = None,
+        limit: int = None,
+        sort: List[SortColumn] = None,
+        targets: List[str] = None,
+        **kwargs
+    ):
+        """
+        Executes a parsed SELECT SQL query on the 'teams' resource of the Microsoft Graph API.
+
+        Args:
+            conditions (List[FilterCondition]): The list of parsed filter conditions.
+            limit (int): The maximum number of records to return.
+            sort (List[SortColumn]): The list of parsed sort columns.
+            targets (List[str]): The list of target columns to return.
+        """
+        client: MSGraphAPITeamsDelegatedPermissionsClient = self.handler.connect()
+        teams = client.get_all_groups()
+
+        teams_df = pd.json_normalize(teams, sep="_")
+        teams_df = teams_df[self.get_columns()]
+
+        return teams_df
+
+    def get_columns(self) -> List[str]:
+        """
+        Retrieves the attributes (columns) of the 'teams' resource.
+
+        Returns:
+            List[Text]: A list of attributes (columns) of the 'teams' resource.
+        """
+        return [
+            "id",
+            "createdDateTime",
+            "displayName",
+            "description",
+            "internalId",
+            "classification",
+            "specialization",
+            "visibility",
+            "webUrl",
+            "isArchived",
+            "tenantId",
+            "isMembershipLimitedToOwners",
+        ]
+
+
+class ChannelsTable(APIResource):
+    """
+    The table abstraction for the 'channels' resource of the Microsoft Graph API.
+    """
+    def list(
+        self,
+        conditions: List[FilterCondition] = None,
+        limit: int = None,
+        sort: List[SortColumn] = None,
+        targets: List[str] = None,
+        **kwargs
+    ):
+        """
+        Executes a parsed SELECT SQL query on the 'channels' resource of the Microsoft Graph API.
+
+        Args:
+            conditions (List[FilterCondition]): The list of parsed filter conditions.
+            limit (int): The maximum number of records to return.
+            sort (List[SortColumn]): The list of parsed sort columns.
+            targets (List[str]): The list of target columns to return.
+        """
+        client: MSGraphAPITeamsDelegatedPermissionsClient = self.handler.connect()
+        channels = []
+
+        team_id, channel_ids = None, None
+        for condition in conditions:
+            if condition.column == "teamId":
+                if condition.op == FilterOperator.EQUAL:
+                    team_id = condition.value
+
+                else:
+                    raise ValueError(
+                        f"Unsupported operator '{condition.op}' for column 'teamId'."
+                    )
+
+                condition.applied = True
+
+            if condition.column == "id":
+                if condition.op == FilterOperator.EQUAL:
+                    channel_ids = [condition.value]
+
+                elif condition.op == FilterOperator.IN:
+                    channel_ids = condition.value
+
+                else:
+                    raise ValueError(
+                        f"Unsupported operator '{condition.op}' for column 'id'."
+                    )
+
+                condition.applied = True
+
+        if team_id:
+            if channel_ids:
+                channels = client.get_channels_in_group_by_ids(team_id, channel_ids)
+
+            else:
+                channels = client.get_all_channels_in_group(team_id)
+
+        elif channel_ids:
+            channels = client.get_channels_across_all_groups_by_ids(channel_ids)
+
+        else:
+            channels = client.get_all_channels_across_all_groups()
+
+        channels_df = pd.json_normalize(channels, sep="_")
+        channels_df = channels_df[self.get_columns()]
+
+        return channels_df
+
+    def get_columns(self) -> List[str]:
+        """
+        Retrieves the attributes (columns) of the 'chats' resource.
+
+        Returns:
+            List[Text]: A list of attributes (columns) of the 'chats' resource.
+        """
+        return [
+            "id",
+            "createdDateTime",
+            "displayName",
+            "description",
+            "isFavoriteByDefault",
+            "email",
+            "tenantId",
+            "webUrl",
+            "membershipType",
+            "teamId",
+        ]
+
+
+class ChannelMessagesTable(APIResource):
+    """
+    The table abstraction for the 'channel messages' resource of the Microsoft Graph API.
+    """
+    def list(
+        self,
+        conditions: List[FilterCondition] = None,
+        limit: int = None,
+        sort: List[SortColumn] = None,
+        targets: List[str] = None,
+        **kwargs
+    ):
+        """
+        Executes a parsed SELECT SQL query on the 'channel messages' resource of the Microsoft Graph API.
+
+        Args:
+            conditions (List[FilterCondition]): The list of parsed filter conditions.
+            limit (int): The maximum number of records to return.
+            sort (List[SortColumn]): The list of parsed sort columns.
+            targets (List[str]): The list of target columns to return.
+        """
+        client: MSGraphAPITeamsDelegatedPermissionsClient = self.handler.connect()
+        messages = []
+
+        group_id, channel_id, message_ids = None, None, None
+        for condition in conditions:
+            if condition.column == "channelIdentity_teamId":
+                if condition.op == FilterOperator.EQUAL:
+                    group_id = condition.value
+
+                else:
+                    raise ValueError(
+                        f"Unsupported operator '{condition.op}' for column 'channelIdentity_teamId'."
+                    )
+
+                condition.applied = True
+
+            if condition.column == "channelIdentity_channelId":
+                if condition.op == FilterOperator.EQUAL:
+                    channel_id = condition.value
+
+                else:
+                    raise ValueError(
+                        f"Unsupported operator '{condition.op}' for column 'channelIdentity_channelId'."
+                    )
+
+                condition.applied = True
+
+            if condition.column == "id":
+                if condition.op == FilterOperator.EQUAL:
+                    message_ids = [condition.value]
+
+                elif condition.op == FilterOperator.IN:
+                    message_ids = condition.value
+
+                else:
+                    raise ValueError(
+                        f"Unsupported operator '{condition.op}' for column 'id'."
+                    )
+
+                condition.applied = True
+
+        if not group_id or not channel_id:
+            raise ValueError("The 'channelIdentity_teamId' and 'channelIdentity_channelId' columns are required.")
+        
+        if message_ids:
+            messages = client.get_messages_in_channel_by_ids(group_id, channel_id, message_ids)
+
+        else:
+            messages = client.get_all_messages_in_channel(group_id, channel_id, limit)
+
+        messages_df = pd.json_normalize(messages, sep="_")
+        messages_df = messages_df[self.get_columns()]
+
+        return messages_df
+
+    def get_columns(self) -> List[str]:
+        """
+        Retrieves the attributes (columns) of the 'chat messages' resource.
+
+        Returns:
+            List[Text]: A list of attributes (columns) of the 'chat messages' resource.
+        """
+        return [
+            "id",
+            "replyToId",
+            "etag",
+            "messageType",
+            "createdDateTime",
+            "lastModifiedDateTime",
+            "lastEditedDateTime",
+            "deletedDateTime",
+            "subject",
+            "summary",
+            "chatId",
+            "importance",
+            "locale",
+            "webUrl",
+            "policyViolation",
+            "from_application",
+            "from_device",
+            "from_user_id",
+            "from_user_displayName",
+            "from_user_userIdentityType",
+            "body_contentType",
+            "body_content",
+            "channelIdentity_teamId",
+            "channelIdentity_channelId",
+        ]
+    
+
+class ChatsTable(APIResource):
+    """
+    The table abstraction for the 'chats' resource of the Microsoft Graph API.
+    """
+    def list(
+        self,
+        conditions: List[FilterCondition] = None,
+        limit: int = None,
+        sort: List[SortColumn] = None,
+        targets: List[str] = None,
+        **kwargs
+    ):
+        """
+        Executes a parsed SELECT SQL query on the 'chats' resource of the Microsoft Graph API.
+
+        Args:
+            conditions (List[FilterCondition]): The list of parsed filter conditions.
+            limit (int): The maximum number of records to return.
+            sort (List[SortColumn]): The list of parsed sort columns.
+            targets (List[str]): The list of target columns to return.
+        """
+        client: MSGraphAPITeamsDelegatedPermissionsClient = self.handler.connect()
+        chats = []
+
+        chat_ids = None
+        for condition in conditions:
+            if condition.column == "id":
+                if condition.op == FilterOperator.EQUAL:
+                    chat_ids = [condition.value]
+
+                elif condition.op == FilterOperator.IN:
+                    chat_ids = condition.value
+
+                else:
+                    raise ValueError(
+                        f"Unsupported operator '{condition.op}' for column 'id'."
+                    )
+
+                condition.applied = True
+
+        if chat_ids:
+            chats = client.get_chats_by_ids(chat_ids)
+
+        else:
+            chats = client.get_all_chats(limit)
+
+        chats_df = pd.json_normalize(chats, sep="_")
+        chats_df = chats_df[self.get_columns()]
+
+        return chats_df
+
+    def get_columns(self) -> List[str]:
+        """
+        Retrieves the attributes (columns) of the 'chats' resource.
+
+        Returns:
+            List[Text]: A list of attributes (columns) of the 'chats' resource.
+        """
+        return [
+            "id",
+            "topic",
+            "createdDateTime",
+            "lastUpdatedDateTime",
+            "chatType",
+            "webUrl",
+            "isHiddenForAllMembers"
+        ]
+    
+
+class ChatMessagesTable(APIResource):
+    """
+    The table abstraction for the 'chat messages' resource of the Microsoft Graph API.
+    """
+    def list(
+        self,
+        conditions: List[FilterCondition] = None,
+        limit: int = None,
+        sort: List[SortColumn] = None,
+        targets: List[str] = None,
+        **kwargs
+    ):
+        """
+        Executes a parsed SELECT SQL query on the 'chat messages' resource of the Microsoft Graph API.
+
+        Args:
+            conditions (List[FilterCondition]): The list of parsed filter conditions.
+            limit (int): The maximum number of records to return.
+            sort (List[SortColumn]): The list of parsed sort columns.
+            targets (List[str]): The list of target columns to return.
+        """
+        client: MSGraphAPITeamsDelegatedPermissionsClient = self.handler.connect()
+        messages = []
+
+        chat_id, message_ids = None, None
+        for condition in conditions:
+            if condition.column == "chatId":
+                if condition.op == FilterOperator.EQUAL:
+                    chat_id = condition.value
+
+                else:
+                    raise ValueError(
+                        f"Unsupported operator '{condition.op}' for column 'chatId'."
+                    )
+
+                condition.applied = True
+
+            if condition.column == "id":
+                if condition.op == FilterOperator.EQUAL:
+                    message_ids = [condition.value]
+
+                elif condition.op == FilterOperator.IN:
+                    message_ids = condition.value
+
+                else:
+                    raise ValueError(
+                        f"Unsupported operator '{condition.op}' for column 'id'."
+                    )
+
+                condition.applied = True
+
+        if not chat_id:
+            raise ValueError("The 'chatId' column is required.")
+        
+        if message_ids:
+            messages = client.get_messages_in_chat_by_ids(chat_id, message_ids)
+
+        else:
+            messages = client.get_all_messages_in_chat(chat_id, limit)
+
+        messages_df = pd.json_normalize(messages, sep="_")
+        messages_df = messages_df[self.get_columns()]
+
+        return messages_df
+
+    def get_columns(self) -> List[str]:
+        """
+        Retrieves the attributes (columns) of the 'chat messages' resource.
+
+        Returns:
+            List[Text]: A list of attributes (columns) of the 'chat messages' resource.
+        """
+        return [
+            "id",
+            "replyToId",
+            "etag",
+            "messageType",
+            "createdDateTime",
+            "lastModifiedDateTime",
+            "lastEditedDateTime",
+            "deletedDateTime",
+            "subject",
+            "summary",
+            "chatId",
+            "importance",
+            "locale",
+            "webUrl",
+            "policyViolation",
+            "from_application",
+            "from_device",
+            "from_user_id",
+            "from_user_displayName",
+            "from_user_userIdentityType",
+            "body_contentType",
+            "body_content",
+        ]

mindsdb/integrations/handlers/mysql_handler/mysql_handler.py

@@ -94,6 +94,8 @@ class MySQLHandler(DatabaseHandler):
                 config["ssl_key"] = ssl_key
         if 'collation' not in config:
             config['collation'] = 'utf8mb4_general_ci'
+        if 'use_pure' not in config:
+            config['use_pure'] = True
         try:
             connection = mysql.connector.connect(**config)
             connection.autocommit = True

mindsdb/integrations/handlers/pgvector_handler/pgvector_handler.py

@@ -114,13 +114,27 @@ class PgVectorHandler(PostgresHandler, VectorStoreHandler):
         if conditions is None:
             return {}
 
-        return {
-            condition.column.split(".")[-1]: {
+        filter_conditions = {}
+
+        for condition in conditions:
+
+            parts = condition.column.split(".")
+            key = parts[0]
+            # converts 'col.el1.el2' to col->'el1'->>'el2'
+            if len(parts) > 1:
+                # intermediate elements
+                for el in parts[1:-1]:
+                    key += f" -> '{el}'"
+
+                # last element
+                key += f" ->> '{parts[-1]}'"
+
+            filter_conditions[key] = {
                 "op": condition.op.value,
                 "value": condition.value,
             }
-            for condition in conditions
-        }
+
+        return filter_conditions
 
     @staticmethod
     def _construct_where_clause(filter_conditions=None):

mindsdb/integrations/handlers/ray_serve_handler/ray_serve_handler.py

@@ -81,27 +81,29 @@ class RayServeHandler(BaseMLEngine):
             resp = requests.post(args['predict_url'],
                                  json={'df': df.to_json(orient='records'), 'pred_args': pred_args},
                                  headers={'content-type': 'application/json; format=pandas-records'})
-        try:
-            if args.get('is_parquet', False):
+        content_type = resp.headers.get("Content-Type", "")
+        if "application/octet-stream" in content_type:
+            try:
                 buffer = io.BytesIO(resp.content)
                 table = pq.read_table(buffer)
                 response = table.to_pandas()
-            else:
+            except Exception:
+                error = 'Could not decode parquet.'
+        else:
+            try:
                 response = resp.json()
-        except json.JSONDecodeError:
-            error = resp.text
-        except Exception:
-            error = 'Could not decode parquet.'
+            except json.JSONDecodeError:
+                error = resp.text
+
+        if 'prediction' in response:
+            target = args['target']
+            if target != 'prediction':
+                # rename prediction to target
+                response[target] = response.pop('prediction')
+            return pd.DataFrame(response)
         else:
-            if 'prediction' in response:
-                target = args['target']
-                if target != 'prediction':
-                    # rename prediction to target
-                    response[target] = response.pop('prediction')
-                return pd.DataFrame(response)
-            else:
-                # something wrong
-                error = response
+            # something wrong
+            error = response
 
         raise RayServeException(f"Error: {error}")
 

mindsdb/integrations/handlers/snowflake_handler/snowflake_handler.py

@@ -7,6 +7,8 @@ from snowflake.connector.errors import NotSupportedError
 
 from mindsdb.utilities import log
 from mindsdb_sql_parser.ast.base import ASTNode
+from mindsdb_sql_parser.ast import Select, Identifier
+
 from mindsdb.integrations.libs.base import DatabaseHandler
 from mindsdb.utilities.render.sqlalchemy_render import SqlalchemyRender
 from mindsdb.integrations.libs.response import (
@@ -234,7 +236,30 @@ class SnowflakeHandler(DatabaseHandler):
 
         query_str = self.renderer.get_string(query, with_failback=True)
         logger.debug(f"Executing SQL query: {query_str}")
-        return self.native_query(query_str)
+        result = self.native_query(query_str)
+        return self.lowercase_columns(result, query)
+
+    def lowercase_columns(self, result, query):
+        if not isinstance(query, Select) or result.data_frame is None:
+            return result
+
+        quoted_columns = []
+        if query.targets is not None:
+            for column in query.targets:
+                if hasattr(column, 'alias') and column.alias is not None:
+                    if column.alias.is_quoted[-1]:
+                        quoted_columns.append(column.alias.parts[-1])
+                elif isinstance(column, Identifier):
+                    if column.is_quoted[-1]:
+                        quoted_columns.append(column.parts[-1])
+
+        rename_columns = {}
+        for col in result.data_frame.columns:
+            if col.isupper() and col not in quoted_columns:
+                rename_columns[col] = col.lower()
+        if rename_columns:
+            result.data_frame = result.data_frame.rename(columns=rename_columns)
+        return result
 
     def get_tables(self) -> Response:
         """

mindsdb/integrations/libs/vectordatabase_handler.py

@@ -325,7 +325,7 @@ class VectorStoreHandler(BaseHandler):
         if not df_insert.empty:
             self.insert(table_name, df_insert)
 
-    def _dispatch_delete(self, query: Delete):
+    def dispatch_delete(self, query: Delete):
         """
         Dispatch delete query to the appropriate method.
         """
@@ -382,7 +382,7 @@ class VectorStoreHandler(BaseHandler):
             DropTables: self._dispatch_drop_table,
             Insert: self._dispatch_insert,
             Update: self._dispatch_update,
-            Delete: self._dispatch_delete,
+            Delete: self.dispatch_delete,
             Select: self.dispatch_select,
         }
         if type(query) in dispatch_router:

mindsdb/integrations/utilities/files/file_reader.py

@@ -309,7 +309,7 @@ class FileReader(FormatDetector):
             )
         text = file_obj.read()
 
-        metadata = {"source": name}
+        metadata = {"source_file": name, "file_format": "txt"}
         documents = [Document(page_content=text, metadata=metadata)]
 
         text_splitter = RecursiveCharacterTextSplitter(
@@ -325,7 +325,7 @@ class FileReader(FormatDetector):
         )
 
     @staticmethod
-    def read_pdf(file_obj: BytesIO, **kwargs):
+    def read_pdf(file_obj: BytesIO, name=None, **kwargs):
 
         with fitz.open(stream=file_obj.read()) as pdf:  # open pdf
             text = chr(12).join([page.get_text() for page in pdf])
@@ -337,7 +337,7 @@ class FileReader(FormatDetector):
         split_text = text_splitter.split_text(text)
 
         return pd.DataFrame(
-            {"content": split_text, "metadata": [{}] * len(split_text)}
+            {"content": split_text, "metadata": [{"file_format": "pdf", "source_file": name}] * len(split_text)}
         )
 
     @staticmethod

mindsdb/integrations/utilities/handlers/api_utilities/microsoft/ms_graph_api_utilities.py

@@ -87,7 +87,7 @@ class MSGraphAPIBaseClient:
 
         return response
 
-    def fetch_paginated_data(self, endpoint: Text, params: Optional[Dict] = {}) -> Generator:
+    def fetch_paginated_data(self, endpoint: Text, params: Optional[Dict] = None) -> Generator:
         """
         Fetches data from the Microsoft Graph API by making the specified request and handling pagination.
 
@@ -98,6 +98,8 @@ class MSGraphAPIBaseClient:
         Yields:
             List: The data fetched from the Microsoft Graph API.
         """
+        if params is None:
+            params = {}
         api_url = self._get_api_url(endpoint)
 
         # Add the pagination count to the request parameters.
@@ -115,7 +117,7 @@ class MSGraphAPIBaseClient:
             api_url = response_json.get("@odata.nextLink", "")
             yield value
 
-    def fetch_data(self, endpoint: str, params: Optional[Dict] = {}) -> Union[List, Dict, bytes]:
+    def _fetch_data(self, endpoint: str, params: Optional[Dict] = {}) -> Union[List, Dict, bytes]:
         """
         Fetches data from the Microsoft Graph API by making the specified request.
 
@@ -129,4 +131,36 @@ class MSGraphAPIBaseClient:
         api_url = self._get_api_url(endpoint)
 
         response = self._make_request(api_url, params)
+        return response
+
+    def fetch_data_content(self, endpoint: str, params: Optional[Dict] = {}) -> bytes:
+        """
+        Fetches data content from the Microsoft Graph API by making the specified request.
+
+        Args:
+            endpoint (str): The endpoint of the Microsoft Graph API to fetch data from.
+            params (Optional[Dict]): The parameters to include in the request.
+
+        Returns:
+            bytes: The data content fetched from the Microsoft Graph API.
+        """
+        response = self._fetch_data(endpoint, params)
         return response.content
+
+    def fetch_data_json(self, endpoint: str, params: Optional[Dict] = {}) -> Union[List, Dict]:
+        """
+        Fetches data from the Microsoft Graph API by making the specified request and returns the JSON response.
+
+        Args:
+            endpoint (str): The endpoint of the Microsoft Graph API to fetch data from.
+            params (Optional[Dict]): The parameters to include in the request.
+
+        Returns:
+            Union[List, Dict]: The JSON response fetched from the Microsoft Graph API.
+        """
+        response = self._fetch_data(endpoint, params)
+        response_json = response.json()
+
+        if "value" in response_json:
+            return response_json["value"]
+        return response_json

mindsdb/integrations/utilities/rag/settings.py

@@ -551,6 +551,7 @@ class ColumnSchema(BaseModel):
             Dict[Union[str, int, float], ValueSchema],
         ]
     ] = Field(
+        default=None,
         description="One of the following. A dict or ordered dict of {schema_value: ValueSchema, ...}, where schema value is the name given for this value description in the schema."
     )
     example_questions: Optional[List[LLMExample]] = Field(