MindsDB 25.4.5.0__py3-none-any.whl → 25.5.4.0__py3-none-any.whl

mindsdb/integrations/handlers/ms_teams_handler/ms_teams_tables.py

@@ -15,6 +15,7 @@ class TeamsTable(APIResource):
     """
     The table abstraction for the 'teams' resource of the Microsoft Graph API.
     """
+
     def list(
         self,
         conditions: List[FilterCondition] = None,
@@ -67,6 +68,7 @@ class ChannelsTable(APIResource):
     """
     The table abstraction for the 'channels' resource of the Microsoft Graph API.
     """
+
     def list(
         self,
         conditions: List[FilterCondition] = None,
@@ -146,6 +148,7 @@ class ChannelMessagesTable(APIResource):
     """
     The table abstraction for the 'channel messages' resource of the Microsoft Graph API.
     """
+
     def list(
         self,
         conditions: List[FilterCondition] = None,
@@ -206,7 +209,7 @@ class ChannelMessagesTable(APIResource):
 
         if not group_id or not channel_id:
             raise ValueError("The 'channelIdentity_teamId' and 'channelIdentity_channelId' columns are required.")
-        
+
         messages = client.get_channel_messages(group_id, channel_id, message_ids)
 
         messages_df = pd.json_normalize(messages, sep="_")
@@ -247,12 +250,13 @@ class ChannelMessagesTable(APIResource):
             "channelIdentity_teamId",
             "channelIdentity_channelId",
         ]
-    
+
 
 class ChatsTable(APIResource):
     """
     The table abstraction for the 'chats' resource of the Microsoft Graph API.
     """
+
     def list(
         self,
         conditions: List[FilterCondition] = None,
@@ -312,12 +316,13 @@ class ChatsTable(APIResource):
             "webUrl",
             "isHiddenForAllMembers"
         ]
-    
+
 
 class ChatMessagesTable(APIResource):
     """
     The table abstraction for the 'chat messages' resource of the Microsoft Graph API.
     """
+
     def list(
         self,
         conditions: List[FilterCondition] = None,
@@ -367,7 +372,7 @@ class ChatMessagesTable(APIResource):
 
         if not chat_id:
             raise ValueError("The 'chatId' column is required.")
-        
+
         messages = client.get_chat_messages(chat_id, message_ids)
 
         messages_df = pd.json_normalize(messages, sep="_")
@@ -405,4 +410,4 @@ class ChatMessagesTable(APIResource):
             "from_user_userIdentityType",
             "body_contentType",
             "body_content",
-        ]
+        ]

mindsdb/integrations/handlers/ms_teams_handler/requirements.txt

@@ -1,2 +1,4 @@
 botframework-connector
-botbuilder-schema
+botbuilder-schema
+msal
+-r mindsdb/integrations/utilities/handlers/auth_utilities/microsoft/requirements.txt

mindsdb/integrations/handlers/mssql_handler/mssql_handler.py

@@ -1,6 +1,10 @@
+from typing import Any
+import datetime
+
 import pymssql
 from pymssql import OperationalError
 import pandas as pd
+from pandas.api import types as pd_types
 
 from mindsdb_sql_parser import parse_sql
 from mindsdb_sql_parser.ast.base import ASTNode
@@ -46,10 +50,77 @@ def _map_type(mssql_type_text: str) -> MYSQL_DATA_TYPE:
         if internal_type_name in db_types_list:
             return mysql_data_type
 
-    logger.warning(f"MSSQL handler type mapping: unknown type: {internal_type_name}, use VARCHAR as fallback.")
+    logger.debug(f"MSSQL handler type mapping: unknown type: {internal_type_name}, use VARCHAR as fallback.")
     return MYSQL_DATA_TYPE.VARCHAR
 
 
+def _make_table_response(result: list[dict[str, Any]], cursor: pymssql.Cursor) -> Response:
+    """Build response from result and cursor.
+
+    Args:
+        result (list[dict[str, Any]]): result of the query.
+        cursor (pymssql.Cursor): cursor object.
+
+    Returns:
+        Response: response object.
+    """
+    description: list[tuple[Any]] = cursor.description
+    mysql_types: list[MYSQL_DATA_TYPE] = []
+
+    data_frame = pd.DataFrame(
+        result,
+        columns=[x[0] for x in cursor.description]
+    )
+
+    for column in description:
+        column_name = column[0]
+        column_type = column[1]
+        column_dtype = data_frame[column_name].dtype
+        match column_type:
+            case pymssql.NUMBER:
+                if pd_types.is_integer_dtype(column_dtype):
+                    mysql_types.append(MYSQL_DATA_TYPE.INT)
+                elif pd_types.is_float_dtype(column_dtype):
+                    mysql_types.append(MYSQL_DATA_TYPE.FLOAT)
+                elif pd_types.is_bool_dtype(column_dtype):
+                    # it is 'bit' type
+                    mysql_types.append(MYSQL_DATA_TYPE.TINYINT)
+                else:
+                    mysql_types.append(MYSQL_DATA_TYPE.DOUBLE)
+            case pymssql.DECIMAL:
+                mysql_types.append(MYSQL_DATA_TYPE.DECIMAL)
+            case pymssql.STRING:
+                mysql_types.append(MYSQL_DATA_TYPE.TEXT)
+            case pymssql.DATETIME:
+                mysql_types.append(MYSQL_DATA_TYPE.DATETIME)
+            case pymssql.BINARY:
+                # DATE and TIME types returned as 'BINARY' type, and dataframe type is 'object', so it is not possible
+                # to infer correct mysql type for them
+                if pd_types.is_datetime64_any_dtype(column_dtype):
+                    # pymssql return datetimes as 'binary' type
+                    # if timezone is present, then it is datetime.timezone
+                    series = data_frame[column_name]
+                    if (
+                        series.dt.tz is not None
+                        and isinstance(series.dt.tz, datetime.timezone)
+                        and series.dt.tz != datetime.timezone.utc
+                    ):
+                        series = series.dt.tz_convert('UTC')
+                        data_frame[column_name] = series.dt.tz_localize(None)
+                    mysql_types.append(MYSQL_DATA_TYPE.DATETIME)
+                else:
+                    mysql_types.append(MYSQL_DATA_TYPE.BINARY)
+            case _:
+                logger.warning(f"Unknown type: {column_type}, use TEXT as fallback.")
+                mysql_types.append(MYSQL_DATA_TYPE.TEXT)
+
+    return Response(
+        RESPONSE_TYPE.TABLE,
+        data_frame=data_frame,
+        mysql_types=mysql_types
+    )
+
+
 class SqlServerHandler(DatabaseHandler):
     """
     This handler handles connection and execution of the Microsoft SQL Server statements.
@@ -169,13 +240,7 @@ class SqlServerHandler(DatabaseHandler):
                 cur.execute(query)
                 if cur.description:
                     result = cur.fetchall()
-                    response = Response(
-                        RESPONSE_TYPE.TABLE,
-                        data_frame=pd.DataFrame(
-                            result,
-                            columns=[x[0] for x in cur.description]
-                        )
-                    )
+                    response = _make_table_response(result, cur)
                 else:
                     response = Response(RESPONSE_TYPE.OK, affected_rows=cur.rowcount)
                 connection.commit()

mindsdb/integrations/handlers/mysql_handler/__about__.py

@@ -1,9 +1,9 @@
-__title__ = 'MindsDB MySQL handler'
-__package_name__ = 'mindsdb_mysql_handler'
-__version__ = '0.0.1'
+__title__ = "MindsDB MySQL handler"
+__package_name__ = "mindsdb_mysql_handler"
+__version__ = "0.0.1"
 __description__ = "MindsDB handler for MySQL"
-__author__ = 'MindsDB Inc'
-__github__ = 'https://github.com/mindsdb/mindsdb'
-__pypi__ = 'https://pypi.org/project/mindsdb/'
-__license__ = 'MIT'
-__copyright__ = 'Copyright 2022- mindsdb'
+__author__ = "MindsDB Inc"
+__github__ = "https://github.com/mindsdb/mindsdb"
+__pypi__ = "https://pypi.org/project/mindsdb/"
+__license__ = "MIT"
+__copyright__ = "Copyright 2022- mindsdb"

mindsdb/integrations/handlers/mysql_handler/__init__.py

@@ -2,19 +2,29 @@ from mindsdb.integrations.libs.const import HANDLER_TYPE
 
 from .__about__ import __version__ as version, __description__ as description
 from .connection_args import connection_args, connection_args_example
+
 try:
     from .mysql_handler import MySQLHandler as Handler
+
     import_error = None
 except Exception as e:
     Handler = None
     import_error = e
 
-title = 'MySQL'
-name = 'mysql'
+title = "MySQL"
+name = "mysql"
 type = HANDLER_TYPE.DATA
-icon_path = 'icon.svg'
+icon_path = "icon.svg"
 
 __all__ = [
-    'Handler', 'version', 'name', 'type', 'title', 'description',
-    'connection_args', 'connection_args_example', 'import_error', 'icon_path'
+    "Handler",
+    "version",
+    "name",
+    "type",
+    "title",
+    "description",
+    "connection_args",
+    "connection_args_example",
+    "import_error",
+    "icon_path",
 ]

mindsdb/integrations/handlers/mysql_handler/connection_args.py

@@ -5,72 +5,68 @@ from mindsdb.integrations.libs.const import HANDLER_CONNECTION_ARG_TYPE as ARG_T
 
 connection_args = OrderedDict(
     url={
-        'type': ARG_TYPE.STR,
-        'description': 'The URI-Like connection string to the MySQL server. If provided, it will override the other connection arguments.',
-        'required': False,
-        'label': 'URL'
+        "type": ARG_TYPE.STR,
+        "description": "The URI-Like connection string to the MySQL server. If provided, it will override the other connection arguments.",
+        "required": False,
+        "label": "URL",
     },
     user={
-        'type': ARG_TYPE.STR,
-        'description': 'The user name used to authenticate with the MySQL server.',
-        'required': True,
-        'label': 'User'
+        "type": ARG_TYPE.STR,
+        "description": "The user name used to authenticate with the MySQL server.",
+        "required": True,
+        "label": "User",
     },
     password={
-        'type': ARG_TYPE.PWD,
-        'description': 'The password to authenticate the user with the MySQL server.',
-        'required': True,
-        'label': 'Password',
-        'secret': True
+        "type": ARG_TYPE.PWD,
+        "description": "The password to authenticate the user with the MySQL server.",
+        "required": True,
+        "label": "Password",
+        "secret": True,
     },
     database={
-        'type': ARG_TYPE.STR,
-        'description': 'The database name to use when connecting with the MySQL server.',
-        'required': True,
-        'label': 'Database'
+        "type": ARG_TYPE.STR,
+        "description": "The database name to use when connecting with the MySQL server.",
+        "required": True,
+        "label": "Database",
     },
     host={
-        'type': ARG_TYPE.STR,
-        'description': 'The host name or IP address of the MySQL server. NOTE: use \'127.0.0.1\' instead of \'localhost\' to connect to local server.',
-        'required': True,
-        'label': 'Host'
+        "type": ARG_TYPE.STR,
+        "description": "The host name or IP address of the MySQL server. NOTE: use '127.0.0.1' instead of 'localhost' to connect to local server.",
+        "required": True,
+        "label": "Host",
     },
     port={
-        'type': ARG_TYPE.INT,
-        'description': 'The TCP/IP port of the MySQL server. Must be an integer.',
-        'required': True,
-        'label': 'Port'
+        "type": ARG_TYPE.INT,
+        "description": "The TCP/IP port of the MySQL server. Must be an integer.",
+        "required": True,
+        "label": "Port",
     },
     ssl={
-        'type': ARG_TYPE.BOOL,
-        'description': 'Set it to True to enable ssl.',
-        'required': False,
-        'label': 'ssl'
+        "type": ARG_TYPE.BOOL,
+        "description": "Set it to True to enable ssl.",
+        "required": False,
+        "label": "ssl",
     },
     ssl_ca={
-        'type': ARG_TYPE.PATH,
-        'description': 'Path or URL of the Certificate Authority (CA) certificate file',
-        'required': False,
-        'label': 'ssl_ca'
+        "type": ARG_TYPE.PATH,
+        "description": "Path or URL of the Certificate Authority (CA) certificate file",
+        "required": False,
+        "label": "ssl_ca",
     },
     ssl_cert={
-        'type': ARG_TYPE.PATH,
-        'description': 'Path name or URL of the server public key certificate file',
-        'required': False,
-        'label': 'ssl_cert'
+        "type": ARG_TYPE.PATH,
+        "description": "Path name or URL of the server public key certificate file",
+        "required": False,
+        "label": "ssl_cert",
     },
     ssl_key={
-        'type': ARG_TYPE.PATH,
-        'description': 'The path name or URL of the server private key file',
-        'required': False,
-        'label': 'ssl_key',
-    }
+        "type": ARG_TYPE.PATH,
+        "description": "The path name or URL of the server private key file",
+        "required": False,
+        "label": "ssl_key",
+    },
 )
 
 connection_args_example = OrderedDict(
-    host='127.0.0.1',
-    port=3306,
-    user='root',
-    password='password',
-    database='database'
+    host="127.0.0.1", port=3306, user="root", password="password", database="database"
 )

mindsdb/integrations/handlers/mysql_handler/mysql_handler.py

@@ -10,16 +10,17 @@ from mindsdb.integrations.libs.base import DatabaseHandler
 from mindsdb.integrations.libs.response import (
     HandlerStatusResponse as StatusResponse,
     HandlerResponse as Response,
-    RESPONSE_TYPE
+    RESPONSE_TYPE,
 )
 from mindsdb.integrations.handlers.mysql_handler.settings import ConnectionConfig
 from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import MYSQL_DATA_TYPE
+from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import C_TYPES, DATA_C_TYPE_MAP
 
 logger = log.getLogger(__name__)
 
 
 def _map_type(mysql_type_text: str) -> MYSQL_DATA_TYPE:
-    """ Map MySQL text types names to MySQL types as enum.
+    """Map MySQL text types names to MySQL types as enum.
 
     Args:
         mysql_type_text (str): The name of the MySQL type to map.
@@ -30,23 +31,95 @@ def _map_type(mysql_type_text: str) -> MYSQL_DATA_TYPE:
     try:
         return MYSQL_DATA_TYPE(mysql_type_text.upper())
     except Exception:
-        logger.warning(f'MySQL handler: unknown type: {mysql_type_text}, use TEXT as fallback.')
+        logger.warning(
+            f"MySQL handler: unknown type: {mysql_type_text}, use TEXT as fallback."
+        )
         return MYSQL_DATA_TYPE.TEXT
 
 
+def _make_table_response(result: list[dict], cursor: mysql.connector.cursor.MySQLCursor) -> Response:
+    """Build response from result and cursor.
+
+    Args:
+        result (list[dict]): result of the query.
+        cursor (mysql.connector.cursor.MySQLCursor): cursor object.
+
+    Returns:
+        Response: response object.
+    """
+    description = cursor.description
+    reverse_c_type_map = {v.code: k for k, v in DATA_C_TYPE_MAP.items() if v.code != C_TYPES.MYSQL_TYPE_BLOB}
+    mysql_types: list[MYSQL_DATA_TYPE] = []
+    for col in description:
+        type_int = col[1]
+        if isinstance(type_int, int) is False:
+            mysql_types.append(MYSQL_DATA_TYPE.TEXT)
+            continue
+
+        if type_int == C_TYPES.MYSQL_TYPE_TINY:
+            # There are 3 types that returns as TINYINT: TINYINT, BOOL, BOOLEAN.
+            mysql_types.append(MYSQL_DATA_TYPE.TINYINT)
+            continue
+
+        if type_int in reverse_c_type_map:
+            mysql_types.append(reverse_c_type_map[type_int])
+            continue
+
+        if type_int != C_TYPES.MYSQL_TYPE_BLOB:
+            raise ValueError(f'Unknown MySQL type id={type_int} in column {col[0]}')
+
+        # region determine text/blob type by flags
+        # Unfortunately, there is no way to determine particular type of text/blob column by flags.
+        # Subtype have to be determined by 8-s element of description tuple, but mysql.conector
+        # return the same value for all text types (TINYTEXT, TEXT, MEDIUMTEXT, LONGTEXT), and for
+        # all blob types (TINYBLOB, BLOB, MEDIUMBLOB, LONGBLOB).
+        if col[7] == 16:  # and col[8] == 45
+            mysql_types.append(MYSQL_DATA_TYPE.TEXT)
+        elif col[7] == 144:  # and col[8] == 63
+            mysql_types.append(MYSQL_DATA_TYPE.BLOB)
+        else:
+            logger.debug(f'MySQL handler: unknown type code {col[7]}, use TEXT as fallback.')
+            mysql_types.append(MYSQL_DATA_TYPE.TEXT)
+        # endregion
+
+    # region cast int and bool to nullable types
+    serieses = []
+    for i, mysql_type in enumerate(mysql_types):
+        expected_dtype = None
+        column_name = description[i][0]
+        if mysql_type in (
+            MYSQL_DATA_TYPE.SMALLINT, MYSQL_DATA_TYPE.INT, MYSQL_DATA_TYPE.MEDIUMINT,
+            MYSQL_DATA_TYPE.BIGINT, MYSQL_DATA_TYPE.TINYINT
+        ):
+            expected_dtype = 'Int64'
+        elif mysql_type in (MYSQL_DATA_TYPE.BOOL, MYSQL_DATA_TYPE.BOOLEAN):
+            expected_dtype = 'boolean'
+        serieses.append(pd.Series([row[column_name] for row in result], dtype=expected_dtype, name=description[i][0]))
+    df = pd.concat(serieses, axis=1, copy=False)
+    # endregion
+
+    response = Response(
+        RESPONSE_TYPE.TABLE,
+        df,
+        affected_rows=cursor.rowcount,
+        mysql_types=mysql_types
+    )
+    return response
+
+
 class MySQLHandler(DatabaseHandler):
     """
     This handler handles connection and execution of the MySQL statements.
     """
 
-    name = 'mysql'
+    name = "mysql"
 
     def __init__(self, name, **kwargs):
         super().__init__(name)
         self.parser = parse_sql
-        self.dialect = 'mysql'
-        self.connection_data = kwargs.get('connection_data', {})
-        self.database = self.connection_data.get('database')
+        self.dialect = "mysql"
+        self.connection_data = kwargs.get("connection_data", {})
+        self.database = self.connection_data.get("database")
 
         self.connection = None
 
@@ -91,28 +164,28 @@ class MySQLHandler(DatabaseHandler):
         if self.is_connected and self.connection.is_connected():
             return self.connection
         config = self._unpack_config()
-        if 'conn_attrs' in self.connection_data:
-            config['conn_attrs'] = self.connection_data['conn_attrs']
+        if "conn_attrs" in self.connection_data:
+            config["conn_attrs"] = self.connection_data["conn_attrs"]
 
-        if 'connection_timeout' not in config:
-            config['connection_timeout'] = 10
+        if "connection_timeout" not in config:
+            config["connection_timeout"] = 10
 
-        ssl = self.connection_data.get('ssl')
+        ssl = self.connection_data.get("ssl")
         if ssl is True:
-            ssl_ca = self.connection_data.get('ssl_ca')
-            ssl_cert = self.connection_data.get('ssl_cert')
-            ssl_key = self.connection_data.get('ssl_key')
-            config['client_flags'] = [mysql.connector.constants.ClientFlag.SSL]
+            ssl_ca = self.connection_data.get("ssl_ca")
+            ssl_cert = self.connection_data.get("ssl_cert")
+            ssl_key = self.connection_data.get("ssl_key")
+            config["client_flags"] = [mysql.connector.constants.ClientFlag.SSL]
             if ssl_ca is not None:
                 config["ssl_ca"] = ssl_ca
             if ssl_cert is not None:
                 config["ssl_cert"] = ssl_cert
             if ssl_key is not None:
                 config["ssl_key"] = ssl_key
-        if 'collation' not in config:
-            config['collation'] = 'utf8mb4_general_ci'
-        if 'use_pure' not in config:
-            config['use_pure'] = True
+        if "collation" not in config:
+            config["collation"] = "utf8mb4_general_ci"
+        if "use_pure" not in config:
+            config["use_pure"] = True
         try:
             connection = mysql.connector.connect(**config)
             connection.autocommit = True
@@ -146,7 +219,9 @@ class MySQLHandler(DatabaseHandler):
             connection = self.connect()
             result.success = connection.is_connected()
         except mysql.connector.Error as e:
-            logger.error(f'Error connecting to MySQL {self.connection_data["database"]}, {e}!')
+            logger.error(
+                f'Error connecting to MySQL {self.connection_data["database"]}, {e}!'
+            )
             result.error_message = str(e)
 
         if result.success and need_to_close:
@@ -173,22 +248,14 @@ class MySQLHandler(DatabaseHandler):
                 cur.execute(query)
                 if cur.with_rows:
                     result = cur.fetchall()
-                    response = Response(
-                        RESPONSE_TYPE.TABLE,
-                        pd.DataFrame(
-                            result,
-                            columns=[x[0] for x in cur.description]
-                        ),
-                        affected_rows=cur.rowcount
-                    )
+                    response = _make_table_response(result, cur)
                 else:
                     response = Response(RESPONSE_TYPE.OK, affected_rows=cur.rowcount)
         except mysql.connector.Error as e:
-            logger.error(f'Error running query: {query} on {self.connection_data["database"]}!')
-            response = Response(
-                RESPONSE_TYPE.ERROR,
-                error_message=str(e)
+            logger.error(
+                f'Error running query: {query} on {self.connection_data["database"]}!'
             )
+            response = Response(RESPONSE_TYPE.ERROR, error_message=str(e))
             if connection is not None and connection.is_connected():
                 connection.rollback()
 
@@ -201,7 +268,7 @@ class MySQLHandler(DatabaseHandler):
         """
         Retrieve the data from the SQL statement.
         """
-        renderer = SqlalchemyRender('mysql')
+        renderer = SqlalchemyRender("mysql")
         query_str = renderer.get_string(query, with_failback=True)
         return self.native_query(query_str)
 

mindsdb/integrations/handlers/mysql_handler/settings.py

@@ -14,29 +14,31 @@ class ConnectionConfig(BaseModel):
     @model_validator(mode="before")
     def check_db_params(cls, values):
         """Ensures either URL is provided or all individual parameters are provided."""
-        url = values.get('url')
-        host = values.get('host')
-        user = values.get('user')
-        password = values.get('password')
-        database = values.get('database')
+        url = values.get("url")
+        host = values.get("host")
+        user = values.get("user")
+        password = values.get("password")
+        database = values.get("database")
         if not url and not (host and user and password and database):
-            raise ValueError("Either a valid URL or required parameters (host, user, password, database) must be provided.")
+            raise ValueError(
+                "Either a valid URL or required parameters (host, user, password, database) must be provided."
+            )
 
         if url:
             parsed = urlparse(url)
-            values['host'] = parsed.hostname or host
-            values['port'] = parsed.port if parsed.port is not None else 3306
-            values['user'] = parsed.username or user
-            values['password'] = parsed.password or password
-            values['database'] = parsed.path[1:] if parsed.path else database
+            values["host"] = parsed.hostname or host
+            values["port"] = parsed.port if parsed.port is not None else 3306
+            values["user"] = parsed.username or user
+            values["password"] = parsed.password or password
+            values["database"] = parsed.path[1:] if parsed.path else database
 
             # mysql connector raise error if url is provided
-            values.pop('url', None)
+            values.pop("url", None)
 
             return values
 
         if not url:
-            for param in ['host', 'user', 'password', 'database']:
+            for param in ["host", "user", "password", "database"]:
                 if not values.get(param):
                     raise ValueError(f"{param} is required when URL is not provided.")
         return values

mindsdb/integrations/handlers/neuralforecast_handler/neuralforecast_handler.py

@@ -41,7 +41,7 @@ class NeuralForecastHandler(BaseMLEngine):
         time_settings = args["timeseries_settings"]
         using_args = args["using"]
         assert time_settings["is_timeseries"], "Specify time series settings in your query"
-        ###### store model args and time series settings in the model folder
+        # store model args and time series settings in the model folder
         model_args = {}
         model_args["target"] = target
         model_args["horizon"] = time_settings["horizon"]

mindsdb/integrations/handlers/newsapi_handler/newsapi_handler.py

@@ -194,7 +194,7 @@ class NewsAPIHandler(APIHandler):
             try:
                 result = self.api.get_everything(**params)
             except Exception as e:
-                raise RuntimeError(f"API call failed: {e}") 
+                raise RuntimeError(f"API call failed: {e}")
             articles = result["articles"]
             for article in articles:
                 article["source_id"] = article["source"]["id"]