MindsDB 25.2.3.0__py3-none-any.whl → 25.3.1.0__py3-none-any.whl

mindsdb/integrations/handlers/gmail_handler/gmail_handler.py

@@ -1,7 +1,4 @@
 import json
-from shutil import copyfile
-
-import requests
 
 from mindsdb.integrations.libs.response import (
     HandlerStatusResponse as StatusResponse,
@@ -15,31 +12,28 @@ from mindsdb.utilities import log
 from mindsdb_sql_parser import parse_sql
 from mindsdb.utilities.config import Config
 
-import os
 import time
 from typing import List
 import pandas as pd
 
-from google.auth.transport.requests import Request
-from google.oauth2.credentials import Credentials
-from google_auth_oauthlib.flow import Flow
 from googleapiclient.discovery import build
 from googleapiclient.errors import HttpError
 from email.message import EmailMessage
 
 from base64 import urlsafe_b64encode, urlsafe_b64decode
 
-from .utils import AuthException, google_auth_flow, save_creds_to_file
+from mindsdb.integrations.utilities.handlers.auth_utilities import GoogleUserOAuth2Manager
+from mindsdb.integrations.utilities.handlers.auth_utilities.exceptions import AuthException
 
-DEFAULT_SCOPES = ['https://www.googleapis.com/auth/gmail.compose',
-                  'https://www.googleapis.com/auth/gmail.readonly',
-                  'https://www.googleapis.com/auth/gmail.modify']
+DEFAULT_SCOPES = [
+    'https://www.googleapis.com/auth/gmail.compose',
+    'https://www.googleapis.com/auth/gmail.readonly',
+    'https://www.googleapis.com/auth/gmail.modify'
+]
 
 logger = log.getLogger(__name__)
 
 
-
-
 class EmailsTable(APITable):
     """Implementation for the emails table for Gmail"""
 
@@ -283,6 +277,14 @@ class GmailHandler(APIHandler):
         super().__init__(name)
         self.connection_args = kwargs.get('connection_data', {})
 
+        self.token_file = None
+        self.max_page_size = 500
+        self.max_batch_size = 100
+        self.service = None
+        self.is_connected = False
+
+        self.handler_storage = kwargs['handler_storage']
+
         self.credentials_url = self.connection_args.get('credentials_url', None)
         self.credentials_file = self.connection_args.get('credentials_file', None)
         if self.connection_args.get('credentials'):
@@ -298,63 +300,11 @@ class GmailHandler(APIHandler):
                 self.credentials_url = secret_url
 
         self.scopes = self.connection_args.get('scopes', DEFAULT_SCOPES)
-        self.token_file = None
-        self.max_page_size = 500
-        self.max_batch_size = 100
-        self.service = None
-        self.is_connected = False
-
-        self.handler_storage = kwargs['handler_storage']
 
         emails = EmailsTable(self)
         self.emails = emails
         self._register_table('emails', emails)
 
-    def _download_secret_file(self, secret_file):
-        # Giving more priority to the S3 file
-        if self.credentials_url:
-            response = requests.get(self.credentials_url)
-            if response.status_code == 200:
-                with open(secret_file, 'w') as creds:
-                    creds.write(response.text)
-                return True
-            else:
-                logger.error("Failed to get credentials from S3", response.status_code)
-
-        if self.credentials_file and os.path.isfile(self.credentials_file):
-            copyfile(self.credentials_file, secret_file)
-            return True
-        return False
-
-    def create_connection(self):
-        creds = None
-
-        # Get the current dir, we'll check for Token & Creds files in this dir
-        curr_dir = self.handler_storage.folder_get('config')
-
-        creds_file = os.path.join(curr_dir, 'creds.json')
-        secret_file = os.path.join(curr_dir, 'secret.json')
-
-        if os.path.isfile(creds_file):
-            creds = Credentials.from_authorized_user_file(creds_file, self.scopes)
-
-        if not creds or not creds.valid:
-            if creds and creds.expired and creds.refresh_token:
-                creds.refresh(Request())
-
-            if self._download_secret_file(secret_file):
-                # save to storage
-                self.handler_storage.folder_sync('config')
-            else:
-                raise ValueError('No valid Gmail Credentials filepath or S3 url found.')
-
-            creds = google_auth_flow(secret_file, self.scopes, self.connection_args.get('code'))
-
-            save_creds_to_file(creds, creds_file)
-            self.handler_storage.folder_sync('config')
-
-        return build('gmail', 'v1', credentials=creds)
-
     def connect(self):
         """Authenticate with the Gmail API using the credentials file.
 
@@ -366,7 +316,10 @@ class GmailHandler(APIHandler):
         if self.is_connected and self.service is not None:
             return self.service
 
-        self.service = self.create_connection()
+        google_oauth2_manager = GoogleUserOAuth2Manager(self.handler_storage, self.scopes, self.credentials_file, self.credentials_url, self.connection_args.get('code'))
+        creds = google_oauth2_manager.get_oauth2_credentials()
+
+        self.service = build('gmail', 'v1', credentials=creds)
 
         self.is_connected = True
         return self.service

mindsdb/integrations/handlers/gmail_handler/requirements.txt

@@ -1,2 +1 @@
 google-api-python-client
-google-auth-httplib2

mindsdb/integrations/handlers/google_calendar_handler/connection_args.py

@@ -4,9 +4,24 @@ from mindsdb.integrations.libs.const import HANDLER_CONNECTION_ARG_TYPE as ARG_T
 
 
 connection_args = OrderedDict(
+    credentials_url={
+        'type': ARG_TYPE.STR,
+        'description': 'URL to Service Account Keys',
+        'label': 'URL to Service Account Keys',
+    },
+    credentials_file={
+        'type': ARG_TYPE.STR,
+        'description': 'Location of Service Account Keys',
+        'label': 'Path to Service Account Keys',
+    },
     credentials={
         'type': ARG_TYPE.PATH,
         'description': 'Service Account Keys',
         'label': 'Upload Service Account Keys',
     },
+    code={
+        'type': ARG_TYPE.STR,
+        'description': 'Code After Authorisation',
+        'label': 'Code After Authorisation',
+    },
 )

mindsdb/integrations/handlers/google_calendar_handler/google_calendar_handler.py

@@ -1,8 +1,4 @@
-import os
-
 import pandas as pd
-from google.auth.transport.requests import Request
-from google.oauth2.credentials import Credentials
 from googleapiclient.discovery import build
 
 from mindsdb.api.executor.data_types.response_type import RESPONSE_TYPE
@@ -11,13 +7,21 @@ from mindsdb.integrations.libs.response import (
     HandlerStatusResponse as StatusResponse,
     HandlerResponse as Response,
 )
+from mindsdb.utilities.config import Config
 from mindsdb.utilities import log
-logger = log.getLogger(__name__)
-
-from mindsdb.integrations.handlers.gmail_handler.utils import AuthException, google_auth_flow, save_creds_to_file
+from mindsdb.integrations.utilities.handlers.auth_utilities import GoogleUserOAuth2Manager
+from mindsdb.integrations.utilities.handlers.auth_utilities.exceptions import AuthException
 
 from .google_calendar_tables import GoogleCalendarEventsTable
 
+DEFAULT_SCOPES = [
+    'https://www.googleapis.com/auth/calendar',
+    'https://www.googleapis.com/auth/calendar.events',
+    'https://www.googleapis.com/auth/calendar.readonly'
+]
+
+logger = log.getLogger(__name__)
+
 
 class GoogleCalendarHandler(APIHandler):
     """
@@ -35,20 +39,29 @@ class GoogleCalendarHandler(APIHandler):
             events (GoogleCalendarEventsTable): The `GoogleCalendarEventsTable` object for interacting with the events table.
         """
         super().__init__(name)
+        self.connection_data = kwargs.get('connection_data', {})
 
-        self.token = None
         self.service = None
-        self.connection_data = kwargs.get('connection_data', {})
-        self.credentials_file = self.connection_data['credentials']
-        self.scopes = [
-            'https://www.googleapis.com/auth/calendar',
-            'https://www.googleapis.com/auth/calendar.events',
-            'https://www.googleapis.com/auth/calendar.readonly'
-        ]
         self.is_connected = False
 
         self.handler_storage = kwargs['handler_storage']
 
+        self.credentials_url = self.connection_data.get('credentials_url', None)
+        self.credentials_file = self.connection_data.get('credentials_file', None)
+        if self.connection_data.get('credentials'):
+            self.credentials_file = self.connection_data.pop('credentials')
+        if not self.credentials_file and not self.credentials_url:
+            # try to get from config
+            gcalendar_config = Config().get('handlers', {}).get('youtube', {})
+            secret_file = gcalendar_config.get('credentials_file')
+            secret_url = gcalendar_config.get('credentials_url')
+            if secret_file:
+                self.credentials_file = secret_file
+            elif secret_url:
+                self.credentials_url = secret_url
+
+        self.scopes = self.connection_data.get('scopes', DEFAULT_SCOPES)
+
         events = GoogleCalendarEventsTable(self)
         self.events = events
         self._register_table('events', events)
@@ -64,32 +77,8 @@ class GoogleCalendarHandler(APIHandler):
         if self.is_connected is True:
             return self.service
 
-        secret_file = self.credentials_file
-
-        curr_dir = self.handler_storage.folder_get('config')
-
-        creds_file = None
-        try:
-            creds_file = os.path.join(curr_dir, 'secret.json')
-        except Exception:
-            pass
-
-        creds = None
-        if os.path.isfile(creds_file):
-            creds = Credentials.from_authorized_user_file(creds_file, self.scopes)
-
-        if not creds or not creds.valid:
-            if creds and creds.expired and creds.refresh_token:
-                creds.refresh(Request())
-
-                save_creds_to_file(creds, creds_file)
-                self.handler_storage.folder_sync('config')
-
-            else:
-                creds = google_auth_flow(secret_file, self.scopes, self.connection_data.get('code'))
-
-                save_creds_to_file(creds, creds_file)
-                self.handler_storage.folder_sync('config')
+        google_oauth2_manager = GoogleUserOAuth2Manager(self.handler_storage, self.scopes, self.credentials_file, self.credentials_url, self.connection_data.get('code'))
+        creds = google_oauth2_manager.get_oauth2_credentials()
 
         self.service = build('calendar', 'v3', credentials=creds)
         return self.service
@@ -106,6 +95,7 @@ class GoogleCalendarHandler(APIHandler):
         try:
             self.connect()
             response.success = True
+            response.copy_storage = True
 
         except AuthException as error:
             response.error_message = str(error)

mindsdb/integrations/handlers/google_calendar_handler/requirements.txt

@@ -1,3 +1 @@
 google-api-python-client
-google-auth-httplib2
--r mindsdb/integrations/handlers/gmail_handler/requirements.txt

mindsdb/integrations/handlers/langchain_embedding_handler/requirements.txt

@@ -1,2 +1 @@
-openai==1.55.3
 tiktoken

mindsdb/integrations/handlers/langchain_handler/requirements.txt

@@ -1,4 +1,3 @@
-openai==1.55.3
 wikipedia==1.4.0
 tiktoken
 anthropic>=0.26.1

mindsdb/integrations/handlers/llama_index_handler/requirements.txt

@@ -1,5 +1,4 @@
 llama-index==0.10.13
-openai == 1.24.0
 pydantic-settings >= 2.1.0
 llama-index-readers-web
 llama-index-embeddings-openai

mindsdb/integrations/handlers/openai_handler/constants.py

@@ -8,7 +8,9 @@ CHAT_MODELS = (
     'gpt-4-32k',
     'gpt-4-1106-preview',
     'gpt-4-0125-preview',
-    'gpt-4o'
+    'gpt-4o',
+    'o3-mini',
+    'o1-mini'
 )
 COMPLETION_MODELS = ('babbage-002', 'davinci-002')
 FINETUNING_MODELS = ('gpt-3.5-turbo', 'babbage-002', 'davinci-002', 'gpt-4')

mindsdb/integrations/handlers/openai_handler/requirements.txt

@@ -1,2 +1 @@
-openai<2.0.0,>=1.54.0
 tiktoken

mindsdb/integrations/handlers/rag_handler/requirements.txt

@@ -1,6 +1,5 @@
 -r mindsdb/integrations/handlers/chromadb_handler/requirements.txt
 faiss-cpu
-openai==1.55.3
 html2text
 writerai~=1.1.0
 sentence-transformers # needed for HuggingFaceEmbeddings from langchain-community

mindsdb/integrations/handlers/ray_serve_handler/ray_serve_handler.py

@@ -1,9 +1,11 @@
+import io
 import json
 
 import requests
 from typing import Dict, Optional
 
 import pandas as pd
+import pyarrow.parquet as pq
 
 from mindsdb.integrations.libs.base import BaseMLEngine
 
@@ -37,9 +39,17 @@ class RayServeHandler(BaseMLEngine):
         args['target'] = target
         self.model_storage.json_set('args', args)
         try:
-            resp = requests.post(args['train_url'],
-                                 json={'df': df.to_json(orient='records'), 'target': target},
-                                 headers={'content-type': 'application/json; format=pandas-records'})
+            if args.get('is_parquet', False):
+                buffer = io.BytesIO()
+                df.to_parquet(buffer)
+                resp = requests.post(args['train_url'],
+                                     files={"df": ("df", buffer.getvalue(), "application/octet-stream")},
+                                     data={"args": json.dumps(args), "target": target},
+                                     )
+            else:
+                resp = requests.post(args['train_url'],
+                                     json={'df': df.to_json(orient='records'), 'target': target, 'args': args},
+                                     headers={'content-type': 'application/json; format=pandas-records'})
         except requests.exceptions.InvalidSchema:
             raise Exception("Error: The URL provided for the training endpoint is invalid.")
 
@@ -59,14 +69,29 @@ class RayServeHandler(BaseMLEngine):
         args = {**(self.model_storage.json_get('args')), **args}  # merge incoming args
         pred_args = args.get('predict_params', {})
         args = {**args, **pred_args}  # merge pred_args
-        resp = requests.post(args['predict_url'],
-                             json={'df': df.to_json(orient='records'), 'pred_args': pred_args},
-                             headers={'content-type': 'application/json; format=pandas-records'})
-
+        if args.get('is_parquet', False):
+            buffer = io.BytesIO()
+            df.attrs['pred_args'] = pred_args
+            df.to_parquet(buffer)
+            resp = requests.post(args['predict_url'],
+                                 files={"df": ("df", buffer.getvalue(), "application/octet-stream")},
+                                 data={"pred_args": json.dumps(pred_args)},
+                                 )
+        else:
+            resp = requests.post(args['predict_url'],
+                                 json={'df': df.to_json(orient='records'), 'pred_args': pred_args},
+                                 headers={'content-type': 'application/json; format=pandas-records'})
         try:
-            response = resp.json()
+            if args.get('is_parquet', False):
+                buffer = io.BytesIO(resp.content)
+                table = pq.read_table(buffer)
+                response = table.to_pandas()
+            else:
+                response = resp.json()
         except json.JSONDecodeError:
             error = resp.text
+        except Exception:
+            error = 'Could not decode parquet.'
         else:
             if 'prediction' in response:
                 target = args['target']

mindsdb/integrations/handlers/web_handler/urlcrawl_helpers.py

@@ -220,8 +220,6 @@ def get_all_website_links_recursively(url, reviewed_urls, limit=None, crawl_dept
     if limit is not None:
         if len(reviewed_urls) >= limit:
             return reviewed_urls
-    if crawl_depth == current_depth:
-        return reviewed_urls
 
     if not filters:
         matches_filter = True
@@ -241,6 +239,9 @@ def get_all_website_links_recursively(url, reviewed_urls, limit=None, crawl_dept
                 "error": str(error_message),
             }
 
+    if crawl_depth is not None and crawl_depth == current_depth:
+        return reviewed_urls
+
     to_rev_url_list = []
 
     # create a list of new urls to review that don't exist in the already reviewed ones

mindsdb/integrations/handlers/web_handler/web_handler.py

@@ -1,62 +1,71 @@
+from typing import List
+
 import pandas as pd
 from mindsdb.integrations.libs.response import HandlerStatusResponse
-from mindsdb_sql_parser import ast
-from mindsdb.integrations.libs.api_handler import APIHandler, APITable
-from mindsdb.utilities.config import Config
-from mindsdb.integrations.utilities.sql_utils import extract_comparison_conditions, project_dataframe
+from mindsdb.utilities.config import config
 from mindsdb.utilities.security import validate_urls
 from .urlcrawl_helpers import get_all_websites
 
+from mindsdb.integrations.libs.api_handler import APIResource, APIHandler
+from mindsdb.integrations.utilities.sql_utils import (FilterCondition, FilterOperator)
 
-class CrawlerTable(APITable):
 
-    def __init__(self, handler: APIHandler):
-        super().__init__(handler)
-        self.config = Config()
+class CrawlerTable(APIResource):
 
-    def select(self, query: ast.Select) -> pd.DataFrame:
+    def list(
+            self,
+            conditions: List[FilterCondition] = None,
+            limit: int = None,
+            **kwargs
+    ) -> pd.DataFrame:
         """
         Selects data from the provided websites
 
-        Args:
-            query (ast.Select): Given SQL SELECT query
-
         Returns:
             dataframe: Dataframe containing the crawled data
 
         Raises:
             NotImplementedError: If the query is not supported
         """
-        conditions = extract_comparison_conditions(query.where)
         urls = []
-        for operator, arg1, arg2 in conditions:
-            if operator == 'or':
-                raise NotImplementedError('OR is not supported')
-            if arg1 == 'url':
-                if operator in ['=', 'in']:
-                    urls = [str(arg2)] if isinstance(arg2, str) else arg2
-                else:
-                    raise NotImplementedError('Invalid URL format. Please provide a single URL like url = "example.com" or'
-                                              'multiple URLs using the format url IN ("url1", "url2", ...)')
+        crawl_depth = None
+        per_url_limit = None
+        for condition in conditions:
+            if condition.column == 'url':
+                if condition.op == FilterOperator.IN:
+                    urls = condition.value
+                elif condition.op == FilterOperator.EQUAL:
+                    urls = [condition.value]
+                condition.applied = True
+            if condition.column == 'crawl_depth' and condition.op == FilterOperator.EQUAL:
+                crawl_depth = condition.value
+                condition.applied = True
+            if condition.column == 'per_url_limit' and condition.op == FilterOperator.EQUAL:
+                per_url_limit = condition.value
+                condition.applied = True
 
         if len(urls) == 0:
             raise NotImplementedError(
-                'You must specify what url you want to crawl, for example: SELECT * FROM crawl WHERE url = "someurl"')
+                'You must specify what url you want to crawl, for example: SELECT * FROM web.crawler WHERE url = "someurl"')
 
-        allowed_urls = self.config.get('web_crawling_allowed_sites', [])
+        allowed_urls = config.get('web_crawling_allowed_sites', [])
         if allowed_urls and not validate_urls(urls, allowed_urls):
             raise ValueError(f"The provided URL is not allowed for web crawling. Please use any of {', '.join(allowed_urls)}.")
 
-        if query.limit is None:
-            raise NotImplementedError('You must specify a LIMIT clause which defines the number of pages to crawl')
-
-        limit = query.limit.value
-
-        result = get_all_websites(urls, limit, html=False)
-        if len(result) > limit:
+        if limit is None and per_url_limit is None and crawl_depth is None:
+            per_url_limit = 1
+        if per_url_limit is not None:
+            # crawl every url separately
+            results = []
+            for url in urls:
+                results.append(get_all_websites([url], per_url_limit, crawl_depth=crawl_depth))
+            result = pd.concat(results)
+        else:
+            result = get_all_websites(urls, limit, crawl_depth=crawl_depth)
+
+        if limit is not None and len(result) > limit:
             result = result[:limit]
-        # filter targets
-        result = project_dataframe(result, query.targets, self.get_columns())
+
         return result
 
     def get_columns(self):

mindsdb/integrations/handlers/youtube_handler/__init__.py

@@ -5,6 +5,7 @@ from .__about__ import __version__ as version, __description__ as description
 
 try:
     from .youtube_handler import YoutubeHandler as Handler
+    from .connection_args import connection_args
     import_error = None
 except Exception as e:
     Handler = None
@@ -24,4 +25,5 @@ __all__ = [
     "description",
     "import_error",
     "icon_path",
+    "connection_args",
 ]

mindsdb/integrations/handlers/youtube_handler/connection_args.py

@@ -0,0 +1,32 @@
+from collections import OrderedDict
+
+from mindsdb.integrations.libs.const import HANDLER_CONNECTION_ARG_TYPE as ARG_TYPE
+
+
+connection_args = OrderedDict(
+    youtube_api_token={
+        'type': ARG_TYPE.STR,
+        'description': 'Youtube API Token',
+        'label': 'Youtube API Token',
+    },
+    credentials_url={
+        'type': ARG_TYPE.STR,
+        'description': 'URL to Service Account Keys',
+        'label': 'URL to Service Account Keys',
+    },
+    credentials_file={
+        'type': ARG_TYPE.STR,
+        'description': 'Location of Service Account Keys',
+        'label': 'Path to Service Account Keys',
+    },
+    credentials={
+        'type': ARG_TYPE.PATH,
+        'description': 'Service Account Keys',
+        'label': 'Upload Service Account Keys',
+    },
+    code={
+        'type': ARG_TYPE.STR,
+        'description': 'Code After Authorisation',
+        'label': 'Code After Authorisation',
+    },
+)

mindsdb/integrations/handlers/youtube_handler/youtube_handler.py

@@ -10,9 +10,7 @@ from mindsdb.integrations.libs.response import (
 from mindsdb.utilities import log
 from mindsdb_sql_parser import parse_sql
 
-from collections import OrderedDict
 from mindsdb.utilities.config import Config
-from mindsdb.integrations.libs.const import HANDLER_CONNECTION_ARG_TYPE as ARG_TYPE
 
 from googleapiclient.discovery import build
 
@@ -38,12 +36,10 @@ class YoutubeHandler(APIHandler):
             name of a handler instance
         """
         super().__init__(name)
-
-        connection_data = kwargs.get("connection_data", {})
+        self.connection_data = kwargs.get("connection_data", {})
+        self.kwargs = kwargs
 
         self.parser = parse_sql
-        self.connection_data = connection_data
-        self.kwargs = kwargs
         self.connection = None
         self.is_connected = False
 
@@ -104,7 +100,6 @@ class YoutubeHandler(APIHandler):
             Status confirmation
         """
         response = StatusResponse(False)
-        need_to_close = self.is_connected is False
 
         try:
             self.connect()
@@ -131,34 +126,3 @@ class YoutubeHandler(APIHandler):
         """
         ast = parse_sql(query)
         return self.query(ast)
-
-
-connection_args = OrderedDict(
-    youtube_access_token={
-        "type": ARG_TYPE.STR,
-        "description": "API Key",
-        "label": "API Key",
-    },
-    credentials_url={
-        'type': ARG_TYPE.STR,
-        'description': 'URL to OAuth2 Credentials',
-        'label': 'URL to OAuth2 Credentials',
-    },
-    credentials_file={
-        'type': ARG_TYPE.STR,
-        'description': 'Location of OAuth2 Credentials',
-        'label': 'Location of OAuth2 Credentials',
-    },
-    credentials={
-        'type': ARG_TYPE.PATH,
-        'description': 'OAuth2 Credentials',
-        'label': 'Upload OAuth2 Credentials',
-    },
-    code={
-        'type': ARG_TYPE.STR,
-        'description': 'Authentication Code',
-        'label': 'Authentication Code',
-    }
-)
-
-connection_args_example = OrderedDict(youtube_api_token="<your-youtube-api-token>")

mindsdb/integrations/libs/llm/utils.py

@@ -16,6 +16,7 @@ from mindsdb.integrations.libs.llm.config import (
     NvidiaNIMConfig,
     MindsdbConfig,
 )
+from mindsdb.utilities.config import config
 from langchain_text_splitters import Language, RecursiveCharacterTextSplitter
 
 
@@ -115,6 +116,11 @@ def get_llm_config(provider: str, args: Dict) -> BaseLLMConfig:
     """
     temperature = min(1.0, max(0.0, args.get("temperature", 0.0)))
     if provider == "openai":
+
+        if any(x in args.get("model_name", "") for x in ['o1', 'o3']):
+            # for o1 and 03, 'temperature' does not support 0.0 with this model. Only the default (1) value is supported
+            temperature = 1
+
         return OpenAIConfig(
             model_name=args.get("model_name", DEFAULT_OPENAI_MODEL),
             temperature=temperature,
@@ -206,7 +212,7 @@ def get_llm_config(provider: str, args: Dict) -> BaseLLMConfig:
     if provider == "mindsdb":
         return MindsdbConfig(
             model_name=args["model_name"],
-            project_name=args.get("project_name", "mindsdb"),
+            project_name=args.get("project_name", config.get("default_project")),
         )
     if provider == "vllm":
         return OpenAIConfig(