MindsDB 25.5.3.0__py3-none-any.whl → 25.5.4.1__py3-none-any.whl

mindsdb/api/mysql/mysql_proxy/libs/constants/mysql.py

@@ -9,6 +9,7 @@
  *******************************************************
 """
 import enum
+from dataclasses import dataclass, field
 
 # CAPABILITIES
 # As defined in : https://dev.mysql.com/doc/dev/mysql-server/8.0.0/group__group__cs__capabilities__flags.html
@@ -128,10 +129,10 @@ class TYPES(object):
     MYSQL_TYPE_DATETIME2 = 18
     MYSQL_TYPE_TIME2 = 19
     MYSQL_TYPE_TYPED_ARRAY = 20
-    MYSQL_TYPE_VECTOR = 242,
-    MYSQL_TYPE_INVALID = 243,
-    MYSQL_TYPE_BOOL = 244,
-    MYSQL_TYPE_JSON = 245,
+    MYSQL_TYPE_VECTOR = 242
+    MYSQL_TYPE_INVALID = 243
+    MYSQL_TYPE_BOOL = 244
+    MYSQL_TYPE_JSON = 245
     MYSQL_TYPE_NEWDECIMAL = 246
     MYSQL_TYPE_ENUM = 247
     MYSQL_TYPE_SET = 248
@@ -312,40 +313,6 @@ MYSQL_DATA_TYPE_COLUMNS_DEFAULT = {
 }
 
 
-# Map between data types and C types
-# https://dev.mysql.com/doc/c-api/8.0/en/c-api-prepared-statement-type-codes.html
-DATA_C_TYPE_MAP = {
-    MYSQL_DATA_TYPE.TINYINT: C_TYPES.MYSQL_TYPE_TINY,
-    MYSQL_DATA_TYPE.SMALLINT: C_TYPES.MYSQL_TYPE_SHORT,
-    MYSQL_DATA_TYPE.MEDIUMINT: C_TYPES.MYSQL_TYPE_INT24,
-    MYSQL_DATA_TYPE.INT: C_TYPES.MYSQL_TYPE_LONG,
-    MYSQL_DATA_TYPE.BIGINT: C_TYPES.MYSQL_TYPE_LONGLONG,
-    MYSQL_DATA_TYPE.FLOAT: C_TYPES.MYSQL_TYPE_FLOAT,
-    MYSQL_DATA_TYPE.DOUBLE: C_TYPES.MYSQL_TYPE_DOUBLE,
-    MYSQL_DATA_TYPE.DECIMAL: C_TYPES.MYSQL_TYPE_NEWDECIMAL,
-    MYSQL_DATA_TYPE.YEAR: C_TYPES.MYSQL_TYPE_SHORT,
-    MYSQL_DATA_TYPE.TIME: C_TYPES.MYSQL_TYPE_TIME,
-    MYSQL_DATA_TYPE.DATE: C_TYPES.MYSQL_TYPE_DATE,
-    MYSQL_DATA_TYPE.DATETIME: C_TYPES.MYSQL_TYPE_DATETIME,
-    MYSQL_DATA_TYPE.TIMESTAMP: C_TYPES.MYSQL_TYPE_TIMESTAMP,
-    MYSQL_DATA_TYPE.CHAR: C_TYPES.MYSQL_TYPE_STRING,
-    MYSQL_DATA_TYPE.BINARY: C_TYPES.MYSQL_TYPE_STRING,
-    MYSQL_DATA_TYPE.VARCHAR: C_TYPES.MYSQL_TYPE_VAR_STRING,
-    MYSQL_DATA_TYPE.VARBINARY: C_TYPES.MYSQL_TYPE_VAR_STRING,
-    MYSQL_DATA_TYPE.TINYBLOB: C_TYPES.MYSQL_TYPE_TINY_BLOB,
-    MYSQL_DATA_TYPE.TINYTEXT: C_TYPES.MYSQL_TYPE_TINY_BLOB,
-    MYSQL_DATA_TYPE.BLOB: C_TYPES.MYSQL_TYPE_BLOB,
-    MYSQL_DATA_TYPE.TEXT: C_TYPES.MYSQL_TYPE_BLOB,
-    MYSQL_DATA_TYPE.MEDIUMBLOB: C_TYPES.MYSQL_TYPE_MEDIUM_BLOB,
-    MYSQL_DATA_TYPE.MEDIUMTEXT: C_TYPES.MYSQL_TYPE_MEDIUM_BLOB,
-    MYSQL_DATA_TYPE.LONGBLOB: C_TYPES.MYSQL_TYPE_LONG_BLOB,
-    MYSQL_DATA_TYPE.LONGTEXT: C_TYPES.MYSQL_TYPE_LONG_BLOB,
-    MYSQL_DATA_TYPE.BIT: C_TYPES.MYSQL_TYPE_BIT,
-    MYSQL_DATA_TYPE.BOOL: C_TYPES.MYSQL_TYPE_TINY,
-    MYSQL_DATA_TYPE.BOOLEAN: C_TYPES.MYSQL_TYPE_TINY
-}
-
-
 class FIELD_FLAG(object):
     __slots__ = ()
     NOT_NULL = 1                # field cannot be null
@@ -368,6 +335,55 @@ class FIELD_FLAG(object):
 FIELD_FLAG = FIELD_FLAG()
 
 
+@dataclass(frozen=True)
+class CTypeProperties:
+    """Properties that describe int-representation of mysql column.
+
+    Attributes:
+        code (int): Code of the mysql type.
+        size (int | None): Size of the column. If not specified, then size is variable (text/blob types).
+        flags (list[int]): Flags of the mysql type.
+    """
+    code: int
+    size: int | None = None
+    flags: list[int] = field(default_factory=list)
+
+
+# Map between data types and C types
+# Fields size and flags been taken from tcp dump of mysql-server response
+# https://dev.mysql.com/doc/c-api/8.0/en/c-api-prepared-statement-type-codes.html
+DATA_C_TYPE_MAP = {
+    MYSQL_DATA_TYPE.TINYINT: CTypeProperties(C_TYPES.MYSQL_TYPE_TINY, 4),
+    MYSQL_DATA_TYPE.SMALLINT: CTypeProperties(C_TYPES.MYSQL_TYPE_SHORT, 6),
+    MYSQL_DATA_TYPE.MEDIUMINT: CTypeProperties(C_TYPES.MYSQL_TYPE_INT24, 9),
+    MYSQL_DATA_TYPE.INT: CTypeProperties(C_TYPES.MYSQL_TYPE_LONG, 11),
+    MYSQL_DATA_TYPE.BIGINT: CTypeProperties(C_TYPES.MYSQL_TYPE_LONGLONG, 20),
+    MYSQL_DATA_TYPE.FLOAT: CTypeProperties(C_TYPES.MYSQL_TYPE_FLOAT, 12),
+    MYSQL_DATA_TYPE.DOUBLE: CTypeProperties(C_TYPES.MYSQL_TYPE_DOUBLE, 22),
+    MYSQL_DATA_TYPE.DECIMAL: CTypeProperties(C_TYPES.MYSQL_TYPE_NEWDECIMAL),
+    MYSQL_DATA_TYPE.YEAR: CTypeProperties(C_TYPES.MYSQL_TYPE_YEAR, 4, [FIELD_FLAG.UNSIGNED, FIELD_FLAG.ZEROFILL_FLAG]),
+    MYSQL_DATA_TYPE.TIME: CTypeProperties(C_TYPES.MYSQL_TYPE_TIME, 10, [FIELD_FLAG.BINARY_COLLATION]),
+    MYSQL_DATA_TYPE.DATE: CTypeProperties(C_TYPES.MYSQL_TYPE_DATE, 10, [FIELD_FLAG.BINARY_COLLATION]),
+    MYSQL_DATA_TYPE.DATETIME: CTypeProperties(C_TYPES.MYSQL_TYPE_DATETIME, 19, [FIELD_FLAG.BINARY_COLLATION]),
+    MYSQL_DATA_TYPE.TIMESTAMP: CTypeProperties(C_TYPES.MYSQL_TYPE_TIMESTAMP, 19, [FIELD_FLAG.BINARY_COLLATION, FIELD_FLAG.TIMESTAMP]),
+    MYSQL_DATA_TYPE.CHAR: CTypeProperties(C_TYPES.MYSQL_TYPE_STRING),
+    MYSQL_DATA_TYPE.BINARY: CTypeProperties(C_TYPES.MYSQL_TYPE_STRING, flags=[FIELD_FLAG.BINARY_COLLATION]),
+    MYSQL_DATA_TYPE.VARCHAR: CTypeProperties(C_TYPES.MYSQL_TYPE_VAR_STRING),
+    MYSQL_DATA_TYPE.VARBINARY: CTypeProperties(C_TYPES.MYSQL_TYPE_VAR_STRING, flags=[FIELD_FLAG.BINARY_COLLATION]),
+    MYSQL_DATA_TYPE.TINYBLOB: CTypeProperties(C_TYPES.MYSQL_TYPE_BLOB, flags=[FIELD_FLAG.BLOB, FIELD_FLAG.BINARY_COLLATION]),
+    MYSQL_DATA_TYPE.TINYTEXT: CTypeProperties(C_TYPES.MYSQL_TYPE_BLOB, flags=[FIELD_FLAG.BLOB]),
+    MYSQL_DATA_TYPE.BLOB: CTypeProperties(C_TYPES.MYSQL_TYPE_BLOB, flags=[FIELD_FLAG.BLOB, FIELD_FLAG.BINARY_COLLATION]),
+    MYSQL_DATA_TYPE.TEXT: CTypeProperties(C_TYPES.MYSQL_TYPE_BLOB, flags=[FIELD_FLAG.BLOB]),
+    MYSQL_DATA_TYPE.MEDIUMBLOB: CTypeProperties(C_TYPES.MYSQL_TYPE_BLOB, flags=[FIELD_FLAG.BLOB, FIELD_FLAG.BINARY_COLLATION]),
+    MYSQL_DATA_TYPE.MEDIUMTEXT: CTypeProperties(C_TYPES.MYSQL_TYPE_BLOB, flags=[FIELD_FLAG.BLOB]),
+    MYSQL_DATA_TYPE.LONGBLOB: CTypeProperties(C_TYPES.MYSQL_TYPE_BLOB, flags=[FIELD_FLAG.BLOB, FIELD_FLAG.BINARY_COLLATION]),
+    MYSQL_DATA_TYPE.LONGTEXT: CTypeProperties(C_TYPES.MYSQL_TYPE_BLOB, flags=[FIELD_FLAG.BLOB]),
+    MYSQL_DATA_TYPE.BIT: CTypeProperties(C_TYPES.MYSQL_TYPE_BIT, 8, [FIELD_FLAG.UNSIGNED]),
+    MYSQL_DATA_TYPE.BOOL: CTypeProperties(C_TYPES.MYSQL_TYPE_TINY, 1),
+    MYSQL_DATA_TYPE.BOOLEAN: CTypeProperties(C_TYPES.MYSQL_TYPE_TINY, 1)
+}
+
+
 # HANDSHAKE
 
 DEFAULT_COALLITION_ID = 83

mindsdb/api/mysql/mysql_proxy/mysql_proxy.py

@@ -21,15 +21,10 @@ import sys
 import tempfile
 import traceback
 from functools import partial
-from typing import Dict, List, Optional
+from typing import List
 from dataclasses import dataclass
 
-from numpy import dtype as np_dtype
-from pandas.api import types as pd_types
-
-from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import NULL_VALUE
 from mindsdb.api.mysql.mysql_proxy.data_types.mysql_datum import Datum
-
 import mindsdb.utilities.hooks as hooks
 import mindsdb.utilities.profiler as profiler
 from mindsdb.utilities.sql import clear_sql
@@ -61,14 +56,14 @@ from mindsdb.api.mysql.mysql_proxy.external_libs.mysql_scramble import (
     scramble as scramble_func,
 )
 from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import (
-    CAPABILITIES,
+    DEFAULT_AUTH_METHOD,
     CHARSET_NUMBERS,
+    SERVER_STATUS,
+    CAPABILITIES,
+    NULL_VALUE,
     COMMANDS,
-    DEFAULT_AUTH_METHOD,
     ERR,
-    SERVER_STATUS,
-    TYPES,
-    getConstName,
+    getConstName
 )
 from mindsdb.api.executor.data_types.answer import ExecuteAnswer
 from mindsdb.api.executor.data_types.response_type import RESPONSE_TYPE
@@ -79,12 +74,14 @@ from mindsdb.api.mysql.mysql_proxy.utilities import (
 from mindsdb.api.executor import exceptions as exec_exc
 
 from mindsdb.api.common.check_auth import check_auth
-from mindsdb.api.mysql.mysql_proxy.utilities.lightwood_dtype import dtype
+from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import MYSQL_DATA_TYPE
+from mindsdb.api.executor.sql_query.result_set import Column, ResultSet
 from mindsdb.utilities import log
 from mindsdb.utilities.config import config
 from mindsdb.utilities.context import context as ctx
 from mindsdb.utilities.otel import increment_otel_query_request_counter
 from mindsdb.utilities.wizards import make_ssl_cert
+from mindsdb.api.mysql.mysql_proxy.utilities.dump import dump_result_set_to_mysql, column_to_mysql_column_dict
 
 logger = log.getLogger(__name__)
 
@@ -96,18 +93,43 @@ def empty_fn():
 @dataclass
 class SQLAnswer:
     resp_type: RESPONSE_TYPE = RESPONSE_TYPE.OK
-    columns: Optional[List[Dict]] = None
-    data: Optional[List[Dict]] = None   # resultSet ?
-    status: Optional[int] = None
-    state_track: Optional[List[List]] = None
-    error_code: Optional[int] = None
-    error_message: Optional[str] = None
-    affected_rows: Optional[int] = None
+    result_set: ResultSet | None = None
+    status: int | None = None
+    state_track: List[List] | None = None
+    error_code: int | None = None
+    error_message: str | None = None
+    affected_rows: int | None = None
+    mysql_types: list[MYSQL_DATA_TYPE] | None = None
 
     @property
     def type(self):
         return self.resp_type
 
+    def dump_http_response(self) -> dict:
+        if self.resp_type == RESPONSE_TYPE.OK:
+            return {
+                "type": self.resp_type,
+                "affected_rows": self.affected_rows,
+            }
+        elif self.resp_type in (RESPONSE_TYPE.TABLE, RESPONSE_TYPE.COLUMNS_TABLE):
+            data = self.result_set.to_lists(json_types=True)
+            return {
+                "type": RESPONSE_TYPE.TABLE,
+                "data": data,
+                "column_names": [
+                    column.alias or column.name
+                    for column in self.result_set.columns
+                ],
+            }
+        elif self.resp_type == RESPONSE_TYPE.ERROR:
+            return {
+                "type": RESPONSE_TYPE.ERROR,
+                "error_code": self.error_code or 0,
+                "error_message": self.error_message,
+            }
+        else:
+            raise ValueError(f"Unsupported response type for dump HTTP response: {self.resp_type}")
+
 
 class MysqlProxy(SocketServer.BaseRequestHandler):
     """
@@ -310,14 +332,14 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
         self.session.unregister_stmt(stmt_id)
 
     def send_query_answer(self, answer: SQLAnswer):
-        if answer.type == RESPONSE_TYPE.TABLE:
+        if answer.type in (RESPONSE_TYPE.TABLE, RESPONSE_TYPE.COLUMNS_TABLE):
             packages = []
 
-            if len(answer.data) > 1000:
+            if len(answer.result_set) > 1000:
                 # for big responses leverage pandas map function to convert data to packages
-                self.send_table_packets(columns=answer.columns, data=answer.data)
+                self.send_table_packets(result_set=answer.result_set)
             else:
-                packages += self.get_table_packets(columns=answer.columns, data=answer.data.to_lists())
+                packages += self.get_table_packets(result_set=answer.result_set)
 
             if answer.status is not None:
                 packages.append(self.last_packet(status=answer.status))
@@ -331,7 +353,7 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
                 ErrPacket, err_code=answer.error_code, msg=answer.error_message
             ).send()
 
-    def _get_column_defenition_packets(self, columns, data=None, columns_len=None):
+    def _get_column_defenition_packets(self, columns: dict, data=None):
         if data is None:
             data = []
         packets = []
@@ -346,18 +368,16 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
             column_name = column.get("name", "column_name")
             column_alias = column.get("alias", column_name)
             flags = column.get("flags", 0)
-            if columns_len is not None:
-                length = columns_len[i]
-            else:
-                if len(data) == 0:
-                    length = 0xFFFF
-                else:
-                    length = 1
-                    for row in data:
-                        if isinstance(row, dict):
-                            length = max(len(str(row[column_alias])), length)
-                        else:
-                            length = max(len(str(row[i])), length)
+            if isinstance(flags, list):
+                flags = sum(flags)
+            if column.get('size') is None:
+                length = 1
+                for row in data:
+                    if isinstance(row, dict):
+                        length = max(len(str(row[column_alias])), length)
+                    else:
+                        length = max(len(str(row[i])), length)
+                column['size'] = 1
 
             packets.append(
                 self.packet(
@@ -369,16 +389,19 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
                     column_name=column_name,
                     column_type=column["type"],
                     charset=column.get("charset", CHARSET_NUMBERS["utf8_unicode_ci"]),
-                    max_length=length,
+                    max_length=column["size"],
                     flags=flags,
                 )
             )
         return packets
 
-    def get_table_packets(self, columns, data, status=0):
+    def get_table_packets(self, result_set: ResultSet, status=0):
+        data_frame, columns_dict = dump_result_set_to_mysql(result_set)
+        data = data_frame.to_dict('split')['data']
+
         # TODO remove columns order
-        packets = [self.packet(ColumnCountPacket, count=len(columns))]
-        packets.extend(self._get_column_defenition_packets(columns, data))
+        packets = [self.packet(ColumnCountPacket, count=len(columns_dict))]
+        packets.extend(self._get_column_defenition_packets(columns_dict, data))
 
         if self.client_capabilities.DEPRECATE_EOF is False:
             packets.append(self.packet(EofPacket, status=status))
@@ -386,9 +409,9 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
         packets += [self.packet(ResultsetRowPacket, data=x) for x in data]
         return packets
 
-    def send_table_packets(self, columns, data, status=0):
+    def send_table_packets(self, result_set: ResultSet, status: int = 0):
+        df, columns_dicts = dump_result_set_to_mysql(result_set, infer_column_size=True)
         # text protocol, convert all to string and serialize as packages
-        df = data.get_raw_df()
 
         def apply_f(v):
             if v is None:
@@ -397,23 +420,10 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
                 v = str(v)
             return Datum.serialize_str(v)
 
-        # get column max size
-        # column_len is used by mysql client to determine width of columns, so it is not mandatory
-        # to get exactly max value. We can approximate them by sample.
-        columns_len = None
-        if len(df) > 0:
-            sample = df.head(100)
-            columns_len = []
-            for column in sample.columns:
-                try:
-                    columns_len.append(sample[column].astype(str).str.len().max())
-                except Exception:
-                    columns_len.append(1)
-
         # columns packages
-        packets = [self.packet(ColumnCountPacket, count=len(columns))]
+        packets = [self.packet(ColumnCountPacket, count=len(columns_dicts))]
 
-        packets.extend(self._get_column_defenition_packets(columns, columns_len=columns_len))
+        packets.extend(self._get_column_defenition_packets(columns_dicts))
 
         if self.client_capabilities.DEPRECATE_EOF is False:
             packets.append(self.packet(EofPacket, status=status))
@@ -486,57 +496,12 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
 
         return {"is_cloud": False}
 
-    def to_mysql_columns(self, columns_list):
-        """Converts raw columns data into convinient format(list of lists) for the futher usage.
-        Plus, it is also converts column types into internal ones."""
-
-        result = []
-
-        database = (
-            None if self.session.database == "" else self.session.database.lower()
-        )
-        for column_record in columns_list:
-
-            field_type = column_record.type
-
-            column_type = TYPES.MYSQL_TYPE_VAR_STRING
-            # is already in mysql protocol type?
-            if isinstance(field_type, int):
-                column_type = field_type
-            # pandas checks
-            elif isinstance(field_type, np_dtype):
-                if pd_types.is_integer_dtype(field_type):
-                    column_type = TYPES.MYSQL_TYPE_LONG
-                elif pd_types.is_numeric_dtype(field_type):
-                    column_type = TYPES.MYSQL_TYPE_DOUBLE
-                elif pd_types.is_datetime64_any_dtype(field_type):
-                    column_type = TYPES.MYSQL_TYPE_DATETIME
-            # lightwood checks
-            elif field_type == dtype.date:
-                column_type = TYPES.MYSQL_TYPE_DATE
-            elif field_type == dtype.datetime:
-                column_type = TYPES.MYSQL_TYPE_DATETIME
-            elif field_type == dtype.float:
-                column_type = TYPES.MYSQL_TYPE_DOUBLE
-            elif field_type == dtype.integer:
-                column_type = TYPES.MYSQL_TYPE_LONG
-
-            result.append(
-                {
-                    "database": column_record.database or database,
-                    #  TODO add 'original_table'
-                    "table_name": column_record.table_name,
-                    "name": column_record.name,
-                    "alias": column_record.alias or column_record.name,
-                    # NOTE all work with text-type, but if/when wanted change types to real,
-                    # it will need to check all types casts in BinaryResultsetRowPacket
-                    "type": column_type,
-                }
-            )
-        return result
+    def to_mysql_columns(self, columns_list: list[Column]) -> list[dict[str, str | int]]:
+        database_name = None if self.session.database == "" else self.session.database.lower()
+        return [column_to_mysql_column_dict(column, database_name=database_name) for column in columns_list]
 
     @profiler.profile()
-    def process_query(self, sql):
+    def process_query(self, sql) -> SQLAnswer:
         executor = Executor(session=self.session, sqlserver=self)
         executor.query_execute(sql)
         executor_answer = executor.executor_answer
@@ -551,10 +516,10 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
             resp = SQLAnswer(
                 resp_type=RESPONSE_TYPE.TABLE,
                 state_track=executor_answer.state_track,
-                columns=self.to_mysql_columns(executor_answer.data.columns),
-                data=executor_answer.data,
+                result_set=executor_answer.data,
                 status=executor.server_status,
-                affected_rows=executor_answer.affected_rows
+                affected_rows=executor_answer.affected_rows,
+                mysql_types=executor_answer.data.mysql_types
             )
 
         # Increment the counter and include metadata in attributes
@@ -609,23 +574,25 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
             return self.send_query_answer(resp)
 
         # TODO prepared_stmt['type'] == 'lock' is not used but it works
-        columns_def = self.to_mysql_columns(executor_answer.data.columns)
-        packages = [self.packet(ColumnCountPacket, count=len(columns_def))]
+        result_set = executor_answer.data
+        data_frame, columns_dict = dump_result_set_to_mysql(result_set)
+        data = data_frame.to_dict('split')['data']
 
-        packages.extend(self._get_column_defenition_packets(columns_def))
+        packages = [self.packet(ColumnCountPacket, count=len(columns_dict))]
+        packages.extend(self._get_column_defenition_packets(columns_dict))
 
         if self.client_capabilities.DEPRECATE_EOF is False:
             packages.append(self.packet(EofPacket, status=0x0062))
 
         # send all
-        for row in executor_answer.data.to_lists():
+        for row in data:
             packages.append(
-                self.packet(BinaryResultsetRowPacket, data=row, columns=columns_def)
+                self.packet(BinaryResultsetRowPacket, data=row, columns=columns_dict)
             )
 
         server_status = executor.server_status or 0x0002
         packages.append(self.last_packet(status=server_status))
-        prepared_stmt["fetched"] += len(executor_answer.data)
+        prepared_stmt["fetched"] += len(data)
 
         return self.send_package_group(packages)