MindsDB 25.4.5.0__py3-none-any.whl → 25.5.4.0__py3-none-any.whl

mindsdb/integrations/handlers/youtube_handler/youtube_handler.py

@@ -14,12 +14,12 @@ from mindsdb.utilities.config import Config
 
 from googleapiclient.discovery import build
 
-from mindsdb.integrations.utilities.handlers.auth_utilities import GoogleUserOAuth2Manager
+from mindsdb.integrations.utilities.handlers.auth_utilities.google import GoogleUserOAuth2Manager
 
 DEFAULT_SCOPES = [
-	'https://www.googleapis.com/auth/youtube',
-	'https://www.googleapis.com/auth/youtube.force-ssl',
-	'https://www.googleapis.com/auth/youtubepartner'
+    'https://www.googleapis.com/auth/youtube',
+    'https://www.googleapis.com/auth/youtube.force-ssl',
+    'https://www.googleapis.com/auth/youtubepartner'
 ]
 
 logger = log.getLogger(__name__)
@@ -81,7 +81,7 @@ class YoutubeHandler(APIHandler):
         """
         if self.is_connected is True:
             return self.connection
-        
+
         google_oauth2_manager = GoogleUserOAuth2Manager(self.handler_storage, self.scopes, self.credentials_file, self.credentials_url, self.connection_data.get('code'))
         creds = google_oauth2_manager.get_oauth2_credentials()
 

mindsdb/integrations/handlers/youtube_handler/youtube_tables.py

@@ -1,7 +1,6 @@
 from typing import List
 
 from mindsdb.integrations.libs.api_handler import APITable
-from mindsdb.integrations.utilities.sql_utils import extract_comparison_conditions
 from mindsdb.utilities import log
 
 from mindsdb_sql_parser import ast
@@ -61,21 +60,21 @@ class YoutubeCommentsTable(APITable):
 
         if not video_id and not channel_id:
             raise ValueError("Either video_id or channel_id has to be present in where clause.")
-            
+
         comments_df = self.get_comments(video_id=video_id, channel_id=channel_id)
 
         select_statement_executor = SELECTQueryExecutor(
-            comments_df, 
-            selected_columns, 
-            [where_condition for where_condition in where_conditions if where_condition[1] not in ['video_id', 'channel_id']], 
-            order_by_conditions, 
+            comments_df,
+            selected_columns,
+            [where_condition for where_condition in where_conditions if where_condition[1] not in ['video_id', 'channel_id']],
+            order_by_conditions,
             result_limit if query.limit else None
         )
 
         comments_df = select_statement_executor.execute_query()
 
         return comments_df
-    
+
     def insert(self, query: ast.Insert) -> None:
         """Inserts data into the YouTube POST /commentThreads API endpoint.
 
@@ -93,7 +92,7 @@ class YoutubeCommentsTable(APITable):
         ValueError
             If the query contains an unsupported condition
         """
-        
+
         insert_query_parser = INSERTQueryParser(query, self.get_columns())
 
         values_to_insert = insert_query_parser.parse_query()
@@ -101,13 +100,13 @@ class YoutubeCommentsTable(APITable):
         for value in values_to_insert:
             if not value.get('comment_id'):
                 if not value.get('comment'):
-                    raise ValueError(f"comment is mandatory for inserting a top-level comment.")
+                    raise ValueError("comment is mandatory for inserting a top-level comment.")
                 else:
                     self.insert_comment(video_id=value['video_id'], text=value['comment'])
 
             else:
                 if not value.get('reply'):
-                    raise ValueError(f"reply is mandatory for inserting a reply.")
+                    raise ValueError("reply is mandatory for inserting a reply.")
                 else:
                     self.insert_comment(comment_id=value['comment_id'], text=value['reply'])
 
@@ -178,7 +177,7 @@ class YoutubeCommentsTable(APITable):
                 replies = []
                 if 'replies' in comment:
                     for reply in comment["replies"]["comments"]:
-                        replies.append( 
+                        replies.append(
                             {
                                 "reply_author": reply["snippet"]["authorDisplayName"],
                                 "user_id": reply["snippet"]["authorChannelId"]["value"],
@@ -225,7 +224,7 @@ class YoutubeCommentsTable(APITable):
 
         youtube_comments_df = pd.json_normalize(data, 'replies', ['comment_id', 'channel_id', 'video_id', 'user_id', 'display_name', 'comment', "published_at", "updated_at"], record_prefix='replies.')
         youtube_comments_df = youtube_comments_df.rename(columns={'replies.user_id': 'reply_user_id', 'replies.reply_author': 'reply_author', 'replies.reply': 'reply'})
-        
+
         # check if DataFrame is empty
         if youtube_comments_df.empty:
             return youtube_comments_df
@@ -262,10 +261,10 @@ class YoutubeChannelsTable(APITable):
         channel_df = self.get_channel_details(channel_id)
 
         select_statement_executor = SELECTQueryExecutor(
-            channel_df, 
-            selected_columns, 
-            [where_condition for where_condition in where_conditions if where_condition[1] == 'channel_id'], 
-            order_by_conditions, 
+            channel_df,
+            selected_columns,
+            [where_condition for where_condition in where_conditions if where_condition[1] == 'channel_id'],
+            order_by_conditions,
             result_limit if query.limit else None
         )
 
@@ -325,7 +324,7 @@ class YoutubeVideosTable(APITable):
                     video_id = arg2
                 else:
                     raise NotImplementedError("Only '=' operator is supported for video_id column.")
-                
+
             elif arg1 == "channel_id":
                 if op == "=":
                     channel_id = arg2
@@ -334,24 +333,24 @@ class YoutubeVideosTable(APITable):
 
         if not video_id and not channel_id:
             raise ValueError("Either video_id or channel_id has to be present in where clause.")
-        
+
         if video_id:
             video_df = self.get_videos_by_video_ids([video_id])
         else:
             video_df = self.get_videos_by_channel_id(channel_id)
 
         select_statement_executor = SELECTQueryExecutor(
-            video_df, 
-            selected_columns, 
-            [where_condition for where_condition in where_conditions if where_condition[1] not in ['video_id', 'channel_id']], 
-            order_by_conditions, 
+            video_df,
+            selected_columns,
+            [where_condition for where_condition in where_conditions if where_condition[1] not in ['video_id', 'channel_id']],
+            order_by_conditions,
             result_limit if query.limit else None
         )
 
         video_df = select_statement_executor.execute_query()
 
         return video_df
-    
+
     def get_videos_by_channel_id(self, channel_id):
         video_ids = []
         resource = (
@@ -389,7 +388,7 @@ class YoutubeVideosTable(APITable):
         # loop over 50 video ids at a time
         # an invalid request error is caused otherwise
         for i in range(0, len(video_ids), 50):
-            resource = self.handler.connect().videos().list(part="statistics,snippet,contentDetails", id=",".join(video_ids[i:i+50])).execute()
+            resource = self.handler.connect().videos().list(part="statistics,snippet,contentDetails", id=",".join(video_ids[i:i + 50])).execute()
             for item in resource["items"]:
                 data.append(
                     {
@@ -418,15 +417,15 @@ class YoutubeVideosTable(APITable):
         except Exception as e:
             logger.error(f"Encountered an error while fetching transcripts for video ${video_id}: ${e}"),
             return "Transcript not available for this video"
-        
+
     def parse_duration(self, video_id, duration):
         try:
-            parsed_duration = re.search(f"PT(\d+H)?(\d+M)?(\d+S)", duration).groups()
+            parsed_duration = re.search(r"PT(\d+H)?(\d+M)?(\d+S)", duration).groups()
             duration_str = ""
             for d in parsed_duration:
                 if d:
                     duration_str += f"{d[:-1]}:"
-            
+
             return duration_str.strip(":")
         except Exception as e:
             logger.error(f"Encountered an error while parsing duration for video ${video_id}: ${e}"),

mindsdb/integrations/handlers/yugabyte_handler/tests/test_yugabyte_handler.py

@@ -35,7 +35,7 @@ class YugabyteHandlerTest(unittest.TestCase):
     def test_4_get_tables(self):
         tables = self.handler.get_tables()
         assert tables.type is not RESPONSE_TYPE.ERROR
- 
+
     def test_5_select_query(self):
         query = "SELECT * FROM PREM;"
         result = self.handler.native_query(query)
@@ -44,6 +44,6 @@ class YugabyteHandlerTest(unittest.TestCase):
     def test_6_check_connection(self):
         self.handler.check_connection()
 
-        
+
 if __name__ == '__main__':
-    unittest.main()
+    unittest.main()

mindsdb/integrations/handlers/yugabyte_handler/yugabyte_handler.py

@@ -1,10 +1,4 @@
 from collections import OrderedDict
-from typing import Optional
-
-from mindsdb_sql_parser import parse_sql
-from mindsdb_sql_parser.ast.base import ASTNode
-from mindsdb.utilities.render.sqlalchemy_render import SqlalchemyRender
-from pandas import DataFrame
 
 from mindsdb.integrations.handlers.postgres_handler.postgres_handler import (
     PostgresHandler,

mindsdb/integrations/libs/response.py

@@ -6,6 +6,7 @@ import pandas
 
 from mindsdb.utilities import log
 from mindsdb.api.executor.data_types.response_type import RESPONSE_TYPE
+from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import MYSQL_DATA_TYPE
 from mindsdb_sql_parser.ast import ASTNode
 
 
@@ -18,29 +19,32 @@ class _INFORMATION_SCHEMA_COLUMNS_NAMES:
     These column names match the standard INFORMATION_SCHEMA.COLUMNS structure
     used in SQL databases to describe table metadata.
     """
-    COLUMN_NAME: str = 'COLUMN_NAME'
-    DATA_TYPE: str = 'DATA_TYPE'
-    ORDINAL_POSITION: str = 'ORDINAL_POSITION'
-    COLUMN_DEFAULT: str = 'COLUMN_DEFAULT'
-    IS_NULLABLE: str = 'IS_NULLABLE'
-    CHARACTER_MAXIMUM_LENGTH: str = 'CHARACTER_MAXIMUM_LENGTH'
-    CHARACTER_OCTET_LENGTH: str = 'CHARACTER_OCTET_LENGTH'
-    NUMERIC_PRECISION: str = 'NUMERIC_PRECISION'
-    NUMERIC_SCALE: str = 'NUMERIC_SCALE'
-    DATETIME_PRECISION: str = 'DATETIME_PRECISION'
-    CHARACTER_SET_NAME: str = 'CHARACTER_SET_NAME'
-    COLLATION_NAME: str = 'COLLATION_NAME'
-    MYSQL_DATA_TYPE: str = 'MYSQL_DATA_TYPE'
+
+    COLUMN_NAME: str = "COLUMN_NAME"
+    DATA_TYPE: str = "DATA_TYPE"
+    ORDINAL_POSITION: str = "ORDINAL_POSITION"
+    COLUMN_DEFAULT: str = "COLUMN_DEFAULT"
+    IS_NULLABLE: str = "IS_NULLABLE"
+    CHARACTER_MAXIMUM_LENGTH: str = "CHARACTER_MAXIMUM_LENGTH"
+    CHARACTER_OCTET_LENGTH: str = "CHARACTER_OCTET_LENGTH"
+    NUMERIC_PRECISION: str = "NUMERIC_PRECISION"
+    NUMERIC_SCALE: str = "NUMERIC_SCALE"
+    DATETIME_PRECISION: str = "DATETIME_PRECISION"
+    CHARACTER_SET_NAME: str = "CHARACTER_SET_NAME"
+    COLLATION_NAME: str = "COLLATION_NAME"
+    MYSQL_DATA_TYPE: str = "MYSQL_DATA_TYPE"
 
 
 INF_SCHEMA_COLUMNS_NAMES = _INFORMATION_SCHEMA_COLUMNS_NAMES()
 INF_SCHEMA_COLUMNS_NAMES_SET = set(f.name for f in fields(INF_SCHEMA_COLUMNS_NAMES))
 
 
-
 class HandlerResponse:
-    def __init__(self, resp_type: RESPONSE_TYPE, data_frame: pandas.DataFrame = None, query: ASTNode = 0, error_code: int = 0,
-                 error_message: str | None = None, affected_rows: int | None = None) -> None:
+    def __init__(
+            self, resp_type: RESPONSE_TYPE, data_frame: pandas.DataFrame = None, query: ASTNode = 0,
+            error_code: int = 0, error_message: str | None = None, affected_rows: int | None = None,
+            mysql_types: list[MYSQL_DATA_TYPE] | None = None
+    ) -> None:
         self.resp_type = resp_type
         self.query = query
         self.data_frame = data_frame
@@ -49,6 +53,7 @@ class HandlerResponse:
         self.affected_rows = affected_rows
         if isinstance(self.affected_rows, int) is False or self.affected_rows < 0:
             self.affected_rows = 0
+        self.mysql_types = mysql_types
 
     @property
     def type(self):
@@ -66,7 +71,9 @@ class HandlerResponse:
                     f"Cannot convert {self.resp_type} to {RESPONSE_TYPE.COLUMNS_TABLE}, "
                     f"the error is: {self.error_message}"
                 )
-            raise ValueError(f"Cannot convert {self.resp_type} to {RESPONSE_TYPE.COLUMNS_TABLE}")
+            raise ValueError(
+                f"Cannot convert {self.resp_type} to {RESPONSE_TYPE.COLUMNS_TABLE}"
+            )
 
         self.data_frame.columns = [name.upper() for name in self.data_frame.columns]
         self.data_frame[INF_SCHEMA_COLUMNS_NAMES.MYSQL_DATA_TYPE] = self.data_frame[
@@ -77,24 +84,26 @@ class HandlerResponse:
         current_columns_set = set(self.data_frame.columns)
         if INF_SCHEMA_COLUMNS_NAMES_SET != current_columns_set:
             raise ValueError(
-                f'Columns set for INFORMATION_SCHEMA.COLUMNS is wrong: {list(current_columns_set)}'
+                f"Columns set for INFORMATION_SCHEMA.COLUMNS is wrong: {list(current_columns_set)}"
             )
         # endregion
 
-        self.data_frame = self.data_frame.astype({
-            INF_SCHEMA_COLUMNS_NAMES.COLUMN_NAME: 'string',
-            INF_SCHEMA_COLUMNS_NAMES.DATA_TYPE: 'string',
-            INF_SCHEMA_COLUMNS_NAMES.ORDINAL_POSITION: 'Int32',
-            INF_SCHEMA_COLUMNS_NAMES.COLUMN_DEFAULT: 'string',
-            INF_SCHEMA_COLUMNS_NAMES.IS_NULLABLE: 'string',
-            INF_SCHEMA_COLUMNS_NAMES.CHARACTER_MAXIMUM_LENGTH: 'Int32',
-            INF_SCHEMA_COLUMNS_NAMES.CHARACTER_OCTET_LENGTH: 'Int32',
-            INF_SCHEMA_COLUMNS_NAMES.NUMERIC_PRECISION: 'Int32',
-            INF_SCHEMA_COLUMNS_NAMES.NUMERIC_SCALE: 'Int32',
-            INF_SCHEMA_COLUMNS_NAMES.DATETIME_PRECISION: 'Int32',
-            INF_SCHEMA_COLUMNS_NAMES.CHARACTER_SET_NAME: 'string',
-            INF_SCHEMA_COLUMNS_NAMES.COLLATION_NAME: 'string',
-        })
+        self.data_frame = self.data_frame.astype(
+            {
+                INF_SCHEMA_COLUMNS_NAMES.COLUMN_NAME: "string",
+                INF_SCHEMA_COLUMNS_NAMES.DATA_TYPE: "string",
+                INF_SCHEMA_COLUMNS_NAMES.ORDINAL_POSITION: "Int32",
+                INF_SCHEMA_COLUMNS_NAMES.COLUMN_DEFAULT: "string",
+                INF_SCHEMA_COLUMNS_NAMES.IS_NULLABLE: "string",
+                INF_SCHEMA_COLUMNS_NAMES.CHARACTER_MAXIMUM_LENGTH: "Int32",
+                INF_SCHEMA_COLUMNS_NAMES.CHARACTER_OCTET_LENGTH: "Int32",
+                INF_SCHEMA_COLUMNS_NAMES.NUMERIC_PRECISION: "Int32",
+                INF_SCHEMA_COLUMNS_NAMES.NUMERIC_SCALE: "Int32",
+                INF_SCHEMA_COLUMNS_NAMES.DATETIME_PRECISION: "Int32",
+                INF_SCHEMA_COLUMNS_NAMES.CHARACTER_SET_NAME: "string",
+                INF_SCHEMA_COLUMNS_NAMES.COLLATION_NAME: "string",
+            }
+        )
         self.data_frame.replace([numpy.NaN, pandas.NA], None, inplace=True)
 
         self.resp_type = RESPONSE_TYPE.COLUMNS_TABLE
@@ -103,33 +112,39 @@ class HandlerResponse:
         try:
             data = None
             if self.data_frame is not None:
-                data = self.data_frame.to_json(orient="split", index=False, date_format="iso")
+                data = self.data_frame.to_json(
+                    orient="split", index=False, date_format="iso"
+                )
         except Exception as e:
             logger.error("%s.to_json: error - %s", self.__class__.__name__, e)
             data = None
-        return  {"type": self.resp_type,
-                 "query": self.query,
-                 "data_frame": data,
-                 "error_code": self.error_code,
-                 "error": self.error_message}
+        return {
+            "type": self.resp_type,
+            "query": self.query,
+            "data_frame": data,
+            "error_code": self.error_code,
+            "error": self.error_message,
+        }
 
     def __repr__(self):
-        return "%s: resp_type=%s, query=%s, data_frame=%s, err_code=%s, error=%s, affected_rows=%s" % (
-                self.__class__.__name__,
-                self.resp_type,
-                self.query,
-                self.data_frame,
-                self.error_code,
-                self.error_message,
-                self.affected_rows
-            )
+        return "%s: resp_type=%s, query=%s, data_frame=\n%s\nerr_code=%s, error=%s, affected_rows=%s" % (
+            self.__class__.__name__,
+            self.resp_type,
+            self.query,
+            self.data_frame,
+            self.error_code,
+            self.error_message,
+            self.affected_rows,
+        )
 
 
 class HandlerStatusResponse:
-    def __init__(self, success: bool = True,
-                 error_message: str = None,
-                 redirect_url: str = None,
-                 copy_storage: str = None
+    def __init__(
+        self,
+        success: bool = True,
+        error_message: str = None,
+        redirect_url: str = None,
+        copy_storage: str = None,
     ) -> None:
         self.success = success
         self.error_message = error_message
@@ -139,7 +154,7 @@ class HandlerStatusResponse:
     def to_json(self):
         data = {"success": self.success, "error": self.error_message}
         if self.redirect_url is not None:
-            data['redirect_url'] = self.redirect_url
+            data["redirect_url"] = self.redirect_url
         return data
 
     def __repr__(self):

mindsdb/integrations/libs/vectordatabase_handler.py

@@ -553,3 +553,9 @@ class VectorStoreHandler(BaseHandler):
             df(pd.DataFrame): Hybrid search result, sorted by hybrid search rank
         '''
         raise NotImplementedError(f'Hybrid search not supported for VectorStoreHandler {self.name}')
+
+    def create_index(self, *args, **kwargs):
+        """
+        Create an index on the specified table.
+        """
+        raise NotImplementedError(f'create_index not supported for VectorStoreHandler {self.name}')

mindsdb/integrations/utilities/files/file_reader.py

@@ -9,8 +9,6 @@ from typing import List
 import filetype
 import pandas as pd
 from charset_normalizer import from_bytes
-from langchain_text_splitters import RecursiveCharacterTextSplitter
-import fitz  # pymupdf
 
 from mindsdb.utilities import log
 
@@ -298,6 +296,8 @@ class FileReader(FormatDetector):
 
     @staticmethod
     def read_txt(file_obj: BytesIO, name=None, **kwargs):
+        # the lib is heavy, so import it only when needed
+        from langchain_text_splitters import RecursiveCharacterTextSplitter
         file_obj = decode(file_obj)
 
         try:
@@ -326,6 +326,9 @@ class FileReader(FormatDetector):
 
     @staticmethod
     def read_pdf(file_obj: BytesIO, name=None, **kwargs):
+        # the libs are heavy, so import it only when needed
+        import fitz  # pymupdf
+        from langchain_text_splitters import RecursiveCharacterTextSplitter
 
         with fitz.open(stream=file_obj.read()) as pdf:  # open pdf
             text = chr(12).join([page.get_text() for page in pdf])

mindsdb/integrations/utilities/handler_utils.py

@@ -43,6 +43,11 @@ def get_api_key(
     if f"{api_name.lower()}_api_key" in create_args:
         return create_args[f"{api_name.lower()}_api_key"]
 
+    # 2.5 - Check in params dictionary if it exists (for agents)
+    if "params" in create_args and create_args["params"] is not None:
+        if f"{api_name.lower()}_api_key" in create_args["params"]:
+            return create_args["params"][f"{api_name.lower()}_api_key"]
+
     # 3
     if engine_storage is not None:
         connection_args = engine_storage.get_connection_args()
@@ -68,7 +73,14 @@ def get_api_key(
         return create_args['api_keys'][api_name]
 
     if strict:
-        raise Exception(
-            f"Missing API key '{api_name.lower()}_api_key'. Either re-create this ML_ENGINE specifying the '{api_name.lower()}_api_key' parameter, or re-create this model and pass the API key with `USING` syntax."
-        )  # noqa
+        provider_upper = api_name.upper()
+        api_key_env_var = f"{provider_upper}_API_KEY"
+        api_key_arg = f"{api_name.lower()}_api_key"
+        error_message = (
+            f"API key for {api_name} not found. Please provide it using one of the following methods:\n"
+            f"1. Set the {api_key_env_var} environment variable\n"
+            f"2. Provide it as '{api_key_arg}' parameter when creating an agent using the CREATE AGENT syntax\n"
+            f"   Example: CREATE AGENT my_agent USING model='gpt-4', provider='{api_name}', {api_key_arg}='your-api-key';\n"
+        )
+        raise Exception(error_message)
     return None

mindsdb/integrations/utilities/handlers/api_utilities/__init__.py

@@ -1 +0,0 @@
-from .microsoft import MSGraphAPIBaseClient

mindsdb/integrations/utilities/handlers/auth_utilities/__init__.py

@@ -1,2 +0,0 @@
-from .google import GoogleUserOAuth2Manager, GoogleServiceAccountOAuth2Manager
-from .microsoft import MSGraphAPIApplicationPermissionsManager, MSGraphAPIDelegatedPermissionsManager

mindsdb/integrations/utilities/utils.py

@@ -8,7 +8,7 @@ def format_exception_error(exception):
         exception_type, _exception_object, exception_traceback = sys.exc_info()
         filename = exception_traceback.tb_frame.f_code.co_filename
         line_number = exception_traceback.tb_lineno
-        error_message = f'{exception_type.__name__}: {exception}, raised at: {filename}#{line_number}'
+        error_message = f"{exception_type.__name__}: {exception}, raised at: {filename}#{line_number}"
     except Exception:
         error_message = str(exception)
     return error_message
@@ -26,7 +26,7 @@ def dict_to_yaml(d, indent=0):
 
 
 # Mocks won't always have 'name' attribute.
-def get_class_name(instance: Any, default: str = 'unknown'):
-    if hasattr(instance.__class__, 'name'):
+def get_class_name(instance: Any, default: str = "unknown"):
+    if hasattr(instance.__class__, "name"):
         return instance.__class__.name
     return default