MindsDB 25.5.3.0__py3-none-any.whl → 25.5.4.0__py3-none-any.whl

mindsdb/api/mysql/mysql_proxy/utilities/dump.py

@@ -0,0 +1,351 @@
+import json
+import datetime
+from typing import Any
+
+import numpy as np
+from numpy import dtype as np_dtype
+import pandas as pd
+from pandas.api import types as pd_types
+
+from mindsdb.api.executor.sql_query.result_set import ResultSet, get_mysql_data_type_from_series, Column
+from mindsdb.api.mysql.mysql_proxy.utilities.lightwood_dtype import dtype as lightwood_dtype
+from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import MYSQL_DATA_TYPE, DATA_C_TYPE_MAP, CTypeProperties
+from mindsdb.utilities import log
+
+logger = log.getLogger(__name__)
+
+
+def column_to_mysql_column_dict(column: Column, database_name: str | None = None) -> dict[str, str | int]:
+    """Convert Column object to dict with column properties.
+
+    Args:
+        column (Column): Column object to convert.
+        database_name (str | None): Name of the database.
+
+    Returns:
+        dict[str, str | int]: Dictionary with mysql column properties.
+    """
+    # region infer type. Should not happen, but what if it is dtype of lightwood type?
+    if isinstance(column.type, str):
+        try:
+            column.type = MYSQL_DATA_TYPE(column.type)
+        except ValueError:
+            if column.type == lightwood_dtype.date:
+                column.type = MYSQL_DATA_TYPE.DATE
+            elif column.type == lightwood_dtype.datetime:
+                column.type = MYSQL_DATA_TYPE.DATETIME
+            elif column.type == lightwood_dtype.float:
+                column.type = MYSQL_DATA_TYPE.FLOAT
+            elif column.type == lightwood_dtype.integer:
+                column.type = MYSQL_DATA_TYPE.INT
+            else:
+                column.type = MYSQL_DATA_TYPE.TEXT
+    elif isinstance(column.type, np_dtype):
+        if pd_types.is_integer_dtype(column.type):
+            column.type = MYSQL_DATA_TYPE.INT
+        elif pd_types.is_numeric_dtype(column.type):
+            column.type = MYSQL_DATA_TYPE.FLOAT
+        elif pd_types.is_datetime64_any_dtype(column.type):
+            column.type = MYSQL_DATA_TYPE.DATETIME
+        else:
+            column.type = MYSQL_DATA_TYPE.TEXT
+    # endregion
+
+    if isinstance(column.type, MYSQL_DATA_TYPE) is False:
+        logger.warning(f'Unexpected column type: {column.type}. Use TEXT as fallback.')
+        column.type = MYSQL_DATA_TYPE.TEXT
+
+    type_properties: CTypeProperties = DATA_C_TYPE_MAP[column.type]
+
+    result = {
+        "database": column.database or database_name,
+        #  TODO add 'original_table'
+        "table_name": column.table_name,
+        "name": column.name,
+        "alias": column.alias or column.name,
+        "size": type_properties.size,
+        "flags": type_properties.flags,
+        "type": type_properties.code,
+    }
+    return result
+
+
+def _dump_bool(var: Any) -> int | None:
+    """Dumps a boolean value to an integer, as in MySQL boolean type is tinyint with values 0 and 1.
+    NOTE: None consider as True in dataframe with dtype=bool, we can't change it
+
+    Args:
+        var (Any): The boolean value to dump
+
+    Returns:
+        int | None: 1 or 0 or None
+    """
+    if pd.isna(var):
+        return None
+    return '1' if var else '0'
+
+
+def _dump_str(var: Any) -> str | None:
+    """Dumps a value to a string.
+
+    Args:
+        var (Any): The value to dump
+
+    Returns:
+        str | None: The string representation of the value or None if the value is None
+    """
+    if pd.isna(var):
+        return None
+    if isinstance(var, bytes):
+        try:
+            return var.decode('utf-8')
+        except Exception:
+            return str(var)[2:-1]
+    if isinstance(var, dict):
+        try:
+            return json.dumps(var)
+        except Exception:
+            return str(var)
+    return str(var)
+
+
+def _dump_int_or_str(var: Any) -> str | None:
+    """Dumps a value to a string.
+    If the value is numeric - then cast it to int to avoid float representation.
+
+    Args:
+        var (Any): The value to dump.
+
+    Returns:
+        str | None: The string representation of the value or None if the value is None
+    """
+    if pd.isna(var):
+        return None
+    try:
+        return str(int(var))
+    except ValueError:
+        return str(var)
+
+
+def _dump_date(var: datetime.date | str | None) -> str | None:
+    """Dumps a date value to a string.
+
+    Args:
+        var (datetime.date | str | None): The date value to dump
+
+    Returns:
+        str | None: The string representation of the date value or None if the value is None
+    """
+    if isinstance(var, (datetime.date, pd.Timestamp)):  # it is also True for datetime.datetime
+        return var.strftime("%Y-%m-%d")
+    elif isinstance(var, str):
+        return var
+    elif pd.isna(var):
+        return None
+    logger.warning(f'Unexpected value type for DATE: {type(var)}, {var}')
+    return _dump_str(var)
+
+
+def _dump_datetime(var: datetime.datetime | str | None) -> str | None:
+    """Dumps a datetime value to a string.
+    # NOTE mysql may display only %Y-%m-%d %H:%M:%S format for datetime column
+
+    Args:
+        var (datetime.datetime | str | None): The datetime value to dump
+
+    Returns:
+        str | None: The string representation of the datetime value or None if the value is None
+    """
+    if isinstance(var, datetime.date):  # it is also datetime.datetime
+        if hasattr(var, 'tzinfo') and var.tzinfo is not None:
+            return var.astimezone(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
+        return var.strftime("%Y-%m-%d %H:%M:%S")
+    elif isinstance(var, pd.Timestamp):
+        if var.tzinfo is not None:
+            return var.tz_convert('UTC').strftime("%Y-%m-%d %H:%M:%S")
+        return var.strftime("%Y-%m-%d %H:%M:%S")
+    elif isinstance(var, str):
+        return var
+    elif pd.isna(var):
+        return None
+    logger.warning(f'Unexpected value type for DATETIME: {type(var)}, {var}')
+    return _dump_str(var)
+
+
+def _dump_time(var: datetime.time | str | None) -> str | None:
+    """Dumps a time value to a string.
+
+    Args:
+        var (datetime.time | str | None): The time value to dump
+
+    Returns:
+        str | None: The string representation of the time value or None if the value is None
+    """
+    if isinstance(var, datetime.time):
+        if var.tzinfo is not None:
+            # NOTE strftime does not support timezone, so we need to convert to UTC
+            offset_seconds = var.tzinfo.utcoffset(None).total_seconds()
+            time_seconds = var.hour * 3600 + var.minute * 60 + var.second
+            utc_seconds = (time_seconds - offset_seconds) % (24 * 3600)
+            hours = int(utc_seconds // 3600)
+            minutes = int((utc_seconds % 3600) // 60)
+            seconds = int(utc_seconds % 60)
+            var = datetime.time(hours, minutes, seconds, var.microsecond)
+        return var.strftime("%H:%M:%S")
+    elif isinstance(var, datetime.datetime):
+        if var.tzinfo is not None:
+            return var.astimezone(datetime.timezone.utc).strftime("%H:%M:%S")
+        return var.strftime("%H:%M:%S")
+    elif isinstance(var, pd.Timestamp):
+        if var.tzinfo is not None:
+            return var.tz_convert('UTC').strftime("%H:%M:%S")
+        return var.strftime("%H:%M:%S")
+    elif isinstance(var, str):
+        return var
+    elif pd.isna(var):
+        return None
+    logger.warning(f'Unexpected value type for TIME: {type(var)}, {var}')
+    return _dump_str(var)
+
+
+def _handle_series_as_date(series: pd.Series) -> pd.Series:
+    """Convert values in a series to a string representation of a date.
+    NOTE: MySQL require exactly %Y-%m-%d for DATE type.
+
+    Args:
+        series (pd.Series): The series to handle
+
+    Returns:
+        pd.Series: The series with the date values as strings
+    """
+    if pd_types.is_datetime64_any_dtype(series.dtype):
+        return series.dt.strftime('%Y-%m-%d')
+    elif pd_types.is_object_dtype(series.dtype):
+        return series.apply(_dump_date)
+    logger.info(f'Unexpected dtype: {series.dtype} for column with type DATE')
+    return series.apply(_dump_str)
+
+
+def _handle_series_as_datetime(series: pd.Series) -> pd.Series:
+    """Convert values in a series to a string representation of a datetime.
+    NOTE: MySQL's DATETIME type require exactly %Y-%m-%d %H:%M:%S format.
+
+    Args:
+        series (pd.Series): The series to handle
+
+    Returns:
+        pd.Series: The series with the datetime values as strings
+    """
+    if pd_types.is_datetime64_any_dtype(series.dtype):
+        return series.dt.strftime('%Y-%m-%d %H:%M:%S')
+    elif pd_types.is_object_dtype(series.dtype):
+        return series.apply(_dump_datetime)
+    logger.info(f'Unexpected dtype: {series.dtype} for column with type DATETIME')
+    return series.apply(_dump_str)
+
+
+def _handle_series_as_time(series: pd.Series) -> pd.Series:
+    """Convert values in a series to a string representation of a time.
+    NOTE: MySQL's TIME type require exactly %H:%M:%S format.
+
+    Args:
+        series (pd.Series): The series to handle
+
+    Returns:
+        pd.Series: The series with the time values as strings
+    """
+    if pd_types.is_timedelta64_ns_dtype(series.dtype):
+        base_time = pd.Timestamp('2000-01-01')
+        series = ((base_time + series).dt.strftime('%H:%M:%S'))
+    elif pd_types.is_datetime64_dtype(series.dtype):
+        series = series.dt.strftime('%H:%M:%S')
+    elif pd_types.is_object_dtype(series.dtype):
+        series = series.apply(_dump_time)
+    else:
+        logger.info(f'Unexpected dtype: {series.dtype} for column with type TIME')
+        series = series.apply(_dump_str)
+    return series
+
+
+def _handle_series_as_int(series: pd.Series) -> pd.Series:
+    """Dump series to str(int) (or just str, of can't case to int). This need because of DataFrame store imput int as
+    float if dtype is object: pd.DataFrame([None, 1], dtype='object') -> [NaN, 1.0]
+
+    Args:
+        series (pd.Series): The series to handle
+
+    Returns:
+        pd.Series: The series with the int values as strings
+    """
+    if pd_types.is_integer_dtype(series.dtype):
+        if series.dtype == 'Int64':
+            # NOTE: 'apply' converts values to python floats
+            return series.astype(object).apply(_dump_str)
+        return series.apply(_dump_str)
+    return series.apply(_dump_int_or_str)
+
+
+def dump_result_set_to_mysql(result_set: ResultSet, infer_column_size: bool = False) -> tuple[pd.DataFrame, list[dict[str, str | int]]]:
+    """
+    Dumps the ResultSet to a format that can be used to send as MySQL response packet.
+    NOTE: This method modifies the original DataFrame and columns.
+
+    Args:
+        result_set (ResultSet): result set to dump
+        infer_column_size (bool): If True, infer the 'size' attribute of the column from the data.
+                                  Exact size is not necessary, approximate is enough.
+
+    Returns:
+        tuple[pd.DataFrame, list[dict[str, str | int]]]: A tuple containing the modified DataFrame and a list
+                                                            of MySQL column dictionaries. The dataframe values are
+                                                            str or None, dtype=object
+    """
+    df = result_set.get_raw_df()
+
+    for i, column in enumerate(result_set.columns):
+        series = df[i]
+        if isinstance(column.type, MYSQL_DATA_TYPE) is False:
+            column.type = get_mysql_data_type_from_series(series)
+
+        column_type: MYSQL_DATA_TYPE = column.type
+
+        match column_type:
+            case MYSQL_DATA_TYPE.BOOL | MYSQL_DATA_TYPE.BOOLEAN:
+                series = series.apply(_dump_bool)
+            case MYSQL_DATA_TYPE.DATE:
+                series = _handle_series_as_date(series)
+            case MYSQL_DATA_TYPE.DATETIME:
+                series = _handle_series_as_datetime(series)
+            case MYSQL_DATA_TYPE.TIME:
+                series = _handle_series_as_time(series)
+            case (
+                MYSQL_DATA_TYPE.INT | MYSQL_DATA_TYPE.TINYINT | MYSQL_DATA_TYPE.SMALLINT
+                | MYSQL_DATA_TYPE.MEDIUMINT | MYSQL_DATA_TYPE.BIGINT | MYSQL_DATA_TYPE.YEAR
+            ):
+                series = _handle_series_as_int(series)
+            case _:
+                series = series.apply(_dump_str)
+
+        # inplace modification of dt types raise SettingWithCopyWarning, so do regular replace
+        # we may split this operation for dt and other types for optimisation
+        df[i] = series.replace([np.NaN, pd.NA, pd.NaT], None)
+
+    columns_dicts = [
+        column_to_mysql_column_dict(column)
+        for column in result_set.columns
+    ]
+
+    if infer_column_size and any(column_info.get('size') is None for column_info in columns_dicts):
+        if len(df) == 0:
+            for column_info in columns_dicts:
+                if column_info['size'] is None:
+                    column_info['size'] = 1
+        else:
+            sample = df.head(100)
+            for i, column_info in enumerate(columns_dicts):
+                try:
+                    column_info['size'] = sample[sample.columns[i]].astype(str).str.len().max()
+                except Exception:
+                    column_info['size'] = 1
+
+    return df, columns_dicts

mindsdb/api/postgres/postgres_proxy/executor/executor.py

@@ -180,9 +180,9 @@ class Executor:
 
             self.params = [
                 Column(
+                    name=p.value,
                     alias=p.value,
                     type="str",
-                    name=p.value,
                 )
                 for p in params
             ]

mindsdb/api/postgres/postgres_proxy/postgres_proxy.py

@@ -276,8 +276,7 @@ class PostgresProxyHandler(socketserver.StreamRequestHandler):
             resp = SQLAnswer(
                 resp_type=RESPONSE_TYPE.TABLE,
                 state_track=executor.state_track,
-                columns=executor.to_postgres_columns(executor.columns),
-                data=executor.data,
+                result_set=executor.data,
                 status=executor.server_status
             )
         return resp
@@ -351,8 +350,8 @@ class PostgresProxyHandler(socketserver.StreamRequestHandler):
         return strip_null_byte(sql).strip(';')
 
     def return_table(self, sql_answer: SQLAnswer, row_descs=True):
-        fields = self.to_postgres_fields(sql_answer.columns)
-        rows = self.to_postgres_rows(sql_answer.data)
+        fields = self.to_postgres_fields(sql_answer.result_set.columns)
+        rows = self.to_postgres_rows(sql_answer.result_set)
         if row_descs:
             self.send(RowDescriptions(fields=fields))
         self.send(DataRow(rows=rows))
@@ -376,8 +375,8 @@ class PostgresProxyHandler(socketserver.StreamRequestHandler):
     def respond_from_sql_answer(self, sql, sql_answer: SQLAnswer, row_descs=True) -> bool:
         # TODO Add command complete passthrough for Complex Queries that exceed row limit in one go
         rows = 0
-        if sql_answer.data:
-            rows = len(sql_answer.data)
+        if sql_answer.result_set:
+            rows = len(sql_answer.result_set)
         if RESPONSE_TYPE.OK == sql_answer.type:
             return self.return_ok(sql, rows=rows)
         elif RESPONSE_TYPE.TABLE == sql_answer.type:

mindsdb/integrations/handlers/altibase_handler/altibase_handler.py

@@ -1,11 +1,10 @@
-from typing import Any, Optional
+from typing import Optional
 
 import jaydebeapi as jdbcconnector
 from mindsdb_sql_parser import parse_sql
 from mindsdb_sql_parser.ast.base import ASTNode
 import pandas as pd
 import pyodbc
-import pyodbc
 import numpy as np
 
 from mindsdb.integrations.libs.base import DatabaseHandler
@@ -33,7 +32,7 @@ class AltibaseHandler(DatabaseHandler):
             connection_data (dict): parameters for connecting to the database
         """
         super().__init__(name)
-        
+
         self.parser = parse_sql
 
         self.connection_args = connection_data
@@ -87,7 +86,7 @@ class AltibaseHandler(DatabaseHandler):
             self.is_connected = True
         except Exception as e:
             logger.error(f"Error while connecting to {self.database}, {e}")
-        
+
         return self.connection
 
     def connect_with_jdbc(self):
@@ -103,11 +102,11 @@ class AltibaseHandler(DatabaseHandler):
         jdbc_url = f"jdbc:Altibase://{self.host}:{self.port}/{self.database}"
 
         try:
-            if self.user and self.password and jar_location: 
+            if self.user and self.password and jar_location:
                 connection = jdbcconnector.connect(jclassname=jdbc_class, url=jdbc_url, driver_args=[self.user, self.password], jars=str(jar_location).split(","))
-            elif self.user and self.password: 
+            elif self.user and self.password:
                 connection = jdbcconnector.connect(jclassname=jdbc_class, url=jdbc_url, driver_args=[self.user, self.password])
-            elif jar_location: 
+            elif jar_location:
                 connection = jdbcconnector.connect(jclassname=jdbc_class, url=jdbc_url, jars=jar_location.split(","))
             else:
                 connection = jdbcconnector.connect(jclassname=jdbc_class, url=jdbc_url)
@@ -117,7 +116,7 @@ class AltibaseHandler(DatabaseHandler):
         except Exception as e:
             logger.error(f"Error while connecting to {self.database}, {e}")
             raise e
-        
+
         return self.connection
 
     def disconnect(self):
@@ -131,7 +130,7 @@ class AltibaseHandler(DatabaseHandler):
             except Exception as e:
                 logger.error(f"Error while disconnecting to {self.database}, {e}")
                 return False
-        return True 
+        return True
 
     def check_connection(self) -> StatusResponse:
         """ Check connection to the handler
@@ -168,7 +167,7 @@ class AltibaseHandler(DatabaseHandler):
             try:
                 cur.execute(query)
                 if cur.description:
-                    result = cur.fetchall() 
+                    result = cur.fetchall()
 
                     if self.dsn:
                         if len(result) > 0:
@@ -218,13 +217,13 @@ class AltibaseHandler(DatabaseHandler):
             HandlerResponse
         """
         query = '''
-            SELECT 
-                TABLE_NAME, 
-                TABLE_ID, 
-                TABLE_TYPE 
-            FROM 
-                system_.sys_tables_ 
-            WHERE 
+            SELECT
+                TABLE_NAME,
+                TABLE_ID,
+                TABLE_TYPE
+            FROM
+                system_.sys_tables_
+            WHERE
                 user_id = USER_ID();
             '''
 
@@ -238,16 +237,16 @@ class AltibaseHandler(DatabaseHandler):
             HandlerResponse
         """
         query = f"""
-            SELECT 
-                COLUMN_NAME, 
-                DATA_TYPE 
-            FROM 
-                system_.sys_columns_ ct 
-            inner join 
-                system_.sys_tables_ tt 
-                on ct.table_id=tt.table_id 
-            where 
+            SELECT
+                COLUMN_NAME,
+                DATA_TYPE
+            FROM
+                system_.sys_columns_ ct
+            inner join
+                system_.sys_tables_ tt
+                on ct.table_id=tt.table_id
+            where
                 tt.table_name = '{table_name.capitalize()}';
             """
-        
+
         return self.native_query(query)

mindsdb/integrations/handlers/altibase_handler/connection_args.py

@@ -4,47 +4,47 @@ from mindsdb.integrations.libs.const import HANDLER_CONNECTION_ARG_TYPE as ARG_T
 
 
 connection_args = OrderedDict(
-    host = {
+    host={
         'type': ARG_TYPE.STR,
         'description': 'The host name or IP address of the Altibase server. NOTE: use \'127.0.0.1\' instead of \'localhost\' to connect to local server.'
     },
-    port = {
+    port={
         'type': ARG_TYPE.INT,
         'description': 'The TCP/IP port of the Altibase server. Must be an integer.'
     },
-    user = {
+    user={
         'type': ARG_TYPE.STR,
         'description': 'The user name used to authenticate with the Altibase server.'
     },
-    password = {
+    password={
         'type': ARG_TYPE.STR,
         'description': 'The password to authenticate the user with the Altibase server.',
         'secret': True
     },
-    database = {
+    database={
         'type': ARG_TYPE.STR,
         'description': 'The database name to use when connecting with the Altibase server.'
     },
-    jdbc_class = {
+    jdbc_class={
         'type': ARG_TYPE.STR,
         'description': 'The driver class of the Altibase JDBC driver'
     },
-    jar_location = {
+    jar_location={
         'type': ARG_TYPE.PATH,
         'description': 'The location of the Altibase JDBC driver jar file'
     },
-    dsn = {
+    dsn={
         'type': ARG_TYPE.STR,
         'description': 'Datasource name of the Altibase server. NOTE: use dsn if you want to use an ODBC connection.'
     }
 )
 
 connection_args_example = OrderedDict(
-    host = '127.0.0.1',
-    port = 20300,
-    user = 'sys',
-    password = 'manager',
-    database = 'mydb',
+    host='127.0.0.1',
+    port=20300,
+    user='sys',
+    password='manager',
+    database='mydb',
     jdbc_class='Altibase.jdbc.driver.AltibaseDriver',
     jar_location='/data/altibase_home/lib/Altibase.jar'
 )

mindsdb/integrations/handlers/altibase_handler/tests/test_altibase_handler.py

@@ -2,6 +2,7 @@ import unittest
 from mindsdb.integrations.handlers.altibase_handler.altibase_handler import AltibaseHandler
 from mindsdb.api.executor.data_types.response_type import RESPONSE_TYPE
 
+
 class AltibaseHandlerTest(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
@@ -18,7 +19,7 @@ class AltibaseHandlerTest(unittest.TestCase):
 
     def test_0_connect(self):
         assert self.handler.connect()
-    
+
     def test_1_drop_table(self):
         # Not supported 'IF EXISTS' syntax
         res = self.handler.query("DROP TABLE TEST_TABLE")
@@ -27,20 +28,20 @@ class AltibaseHandlerTest(unittest.TestCase):
     def test_2_create_table(self):
         res = self.handler.query(
             '''CREATE TABLE TEST_TABLE (
-                ID INT PRIMARY KEY, 
+                ID INT PRIMARY KEY,
                 NAME VARCHAR(14)
                 )'''
-            )
+        )
         assert res.type is RESPONSE_TYPE.OK
 
     def test_3_insert(self):
         res = self.handler.query(
-            """INSERT INTO TEST_TABLE 
-            VALUES 
+            """INSERT INTO TEST_TABLE
+            VALUES
                 (100,'ONE HUNDRED'),
                 (200,'TWO HUNDRED'),
                 (300,'THREE HUNDRED')"""
-            )
+        )
         assert res.type is RESPONSE_TYPE.OK
 
     def test_4_select(self):
@@ -60,8 +61,7 @@ class AltibaseHandlerTest(unittest.TestCase):
 
     def test_8_disconnect(self):
         assert self.handler.disconnect()
-    
 
-        
+
 if __name__ == '__main__':
     unittest.main()