MindsDB 1.2.9__py3-none-any.whl → 24.12.4.0__py3-none-any.whl

mindsdb/api/mysql/mysql_proxy/classes/sql_statement_parser.py

@@ -0,0 +1,151 @@
+import re
+
+from pyparsing import (
+    CaselessKeyword,
+    originalTextFor,
+    ParseException,
+    ParserElement,
+    QuotedString,
+    ZeroOrMore,
+    nestedExpr,
+    restOfLine,
+    StringEnd,
+    Literal,
+    SkipTo
+)
+from mindsdb.api.mysql.mysql_proxy.utilities import exceptions as exc
+
+RE_INT = re.compile(r'^[-+]?([1-9]\d*|0)$')
+RE_FLOAT = re.compile(r'^[-+]?([1-9]\d*\.\d*|0\.|0\.\d*)$')
+
+
+class SqlStatementParser():
+    def __init__(self, text):
+        self._original_sql = text
+        self._sql = SqlStatementParser.clear_sql(text)
+        self._keyword = SqlStatementParser.get_keyword(self._sql)
+        self._struct = None
+
+    @property
+    def keyword(self):
+        return self._keyword
+
+    @property
+    def sql(self):
+        return self._sql
+
+    @property
+    def struct(self):
+        return self._struct
+
+    @staticmethod
+    def clear_sql(sql: str) -> str:
+        ''' remove comments from sql
+            TODO current implementation is not remove /**/ from mid of string:
+            select a, /*comment*/ from b
+        '''
+        # remove /*comment*/
+        ParserElement.defaultWhitespaceChars = (" \t")
+        comment = nestedExpr('/*', '*/').suppress()
+        starting = ZeroOrMore(comment.suppress())
+        ending = ZeroOrMore(comment | ';').suppress() + StringEnd()
+        expr = starting + SkipTo(ending) + ending
+        sql = expr.transformString(sql)
+
+        # remove -- and # comments
+        oracleSqlComment = Literal("--") + restOfLine
+        mySqlComment = Literal("#") + restOfLine
+
+        expr = (
+            originalTextFor(QuotedString("'"))
+            | originalTextFor(QuotedString('"'))
+            | originalTextFor(QuotedString('`'))
+            | (oracleSqlComment | mySqlComment).suppress()
+        )
+
+        sql = expr.transformString(sql)
+        sql = sql.strip(' \n\t')
+
+        return sql
+
+    @staticmethod
+    def get_keyword(sql):
+        ''' Return keyword of sql statement. Should be one of:
+
+            start (transaction)
+            set (autocommit, names etc)
+            use
+            show
+            delete
+            insert
+            update
+            alter
+            select
+            rollback
+            commit
+            explain
+            drop
+            retrain
+            describe
+            begin
+
+            create_predictor
+            create_table
+            create_datasource
+            create_database
+        '''
+        START, SET, USE, SHOW, DELETE, INSERT, UPDATE, ALTER, SELECT, ROLLBACK, COMMIT, EXPLAIN, CREATE, AI, TABLE, PREDICTOR, VIEW, DATASOURCE, DROP, RETRAIN, DESCRIBE, DATABASE, BEGIN = map(
+            CaselessKeyword,
+            "START SET USE SHOW DELETE INSERT UPDATE ALTER SELECT ROLLBACK COMMIT EXPLAIN CREATE AI TABLE PREDICTOR VIEW DATASOURCE DROP RETRAIN DESCRIBE DATABASE BEGIN".split()
+        )
+        CREATE_PREDICTOR = CREATE + PREDICTOR
+        CREATE_VIEW = CREATE + VIEW
+        CREATE_DATASOURCE = CREATE + DATASOURCE
+        CREATE_DATABASE = CREATE + DATABASE
+        CREATE_TABLE = CREATE + TABLE
+
+        expr = (
+            START | SET | USE
+            | SHOW | DELETE | INSERT
+            | UPDATE | ALTER | SELECT
+            | ROLLBACK | COMMIT | EXPLAIN
+            | CREATE_PREDICTOR
+            | CREATE_VIEW | DROP | RETRAIN
+            | CREATE_DATASOURCE | DESCRIBE
+            | CREATE_DATABASE | CREATE_TABLE | BEGIN
+        )('keyword')
+
+        r = expr.parseString(sql)
+
+        keyword = '_'.join(r.get('keyword', [])).lower()
+
+        if keyword == 0:
+            raise exc.ErSqlSyntaxError('Cant get keyword from statement')
+
+        return keyword
+
+    def ends_with(self, text):
+        ''' Check if sql ends with 'text'. Not case sensitive.
+        '''
+        test_sql = ' '.join(self._sql.split()).lower()
+        return test_sql.endswith(text.lower())
+
+    def cut_from_tail(self, text):
+        ''' Removes 'text' from end of sql. Not case sensitive.
+        '''
+        text_arr = text.split(' ')
+
+        ending = CaselessKeyword(text_arr[0])
+        for x in text_arr[1:]:
+            ending = ending + CaselessKeyword(x)
+        ending = ending + StringEnd()
+
+        expr = (originalTextFor(SkipTo(ending)))('original') + (originalTextFor(ending))('ending')
+
+        try:
+            r = expr.parseString(self._sql)
+        except ParseException:
+            return False
+
+        self._sql = r.asDict()['original'].strip()
+        return True

mindsdb/api/mysql/mysql_proxy/data_types/mysql_datum.py

@@ -0,0 +1,212 @@
+"""
+*******************************************************
+ * Copyright (C) 2017 MindsDB Inc. <copyright@mindsdb.com>
+ *
+ * This file is part of MindsDB Server.
+ *
+ * MindsDB Server can not be copied and/or distributed without the express
+ * permission of MindsDB Inc
+ *******************************************************
+"""
+import struct
+
+from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import (
+    DEFAULT_CAPABILITIES,
+    NULL_VALUE,
+    ONE_BYTE_ENC,
+    THREE_BYTE_ENC,
+    TWO_BYTE_ENC,
+)
+from mindsdb.utilities import log
+
+logger = log.getLogger(__name__)
+
+NULL_VALUE_INT = ord(NULL_VALUE)
+
+
+class Datum:
+    __slots__ = ['value', 'var_type', 'var_len']
+
+    def __init__(self, var_type, value=None, var_len=None):
+        # TODO other types: float, timestamp
+        self.value = b""
+
+        if var_len is None:
+            idx = var_type.find('<')
+            var_len = var_type[idx + 1: -1]
+            var_type = var_type[: idx]
+        self.var_type = var_type
+        self.var_len = var_len
+
+        if value is not None:
+            self.set(value)
+
+    def set(self, value):
+        self.value = value
+
+    def setFromBuff(self, buff):
+        if self.var_len == "lenenc":
+            start = 1
+            ln_enc = buff[0]
+            if int(ln_enc) <= ONE_BYTE_ENC[0]:
+                start = 0
+                end = 1
+            elif int(ln_enc) == TWO_BYTE_ENC[0]:
+                end = 3
+            elif int(ln_enc) == THREE_BYTE_ENC[0]:
+                end = 4
+            elif ln_enc:
+                end = 9
+
+            num_str = buff[start:end]
+            if end > 9:
+                logger.error("Cant decode integer greater than 8 bytes")
+                return buff[end - 1 :]  # noqa: E203
+
+            for j in range(8 - (end - start)):
+                num_str += b"\0"
+
+            if self.var_type == "int":
+                self.value = struct.unpack("i", num_str)
+                return buff[end:]
+
+            if self.var_type in ["byte", "string"]:
+                length = struct.unpack("Q", num_str)[0]
+                self.value = buff[end : (length + end)]  # noqa: E203
+                return buff[(length + end) :]  # noqa: E203
+
+        if self.var_len == "EOF":
+            length = len(buff)
+            self.var_len = str(length)
+            self.value = buff
+            return ""
+        else:
+            length = self.var_len
+
+        if self.var_type == "string" and self.var_len == "NUL":
+            for j, x in enumerate(buff):
+                if int(x) == 0:
+                    length = j + 1
+                    break
+
+        length = int(length)
+        if self.var_type in ["byte", "string"]:
+            end = length
+            self.value = buff[:end]
+        else:  # if its an integer
+            end = length
+            num_str = buff[:end]
+            if end > 8:
+                logger.error("cant decode integer greater than 8 bytes")
+                return buff[end:]
+            for j in range(8 - end):
+                num_str += b"\0"
+            self.value = struct.unpack("Q", num_str)[0]
+        if str(self.var_len) == "NUL":
+            self.value = self.value[:-1]
+        return buff[end:]
+
+    @classmethod
+    def serialize_int(cls, value):
+        if value is None:
+            return NULL_VALUE
+
+        byte_count = -(value.bit_length() // (-8))
+
+        if byte_count == 0:
+            return b"\0"
+        if value < NULL_VALUE_INT:
+            return struct.pack("B", value)
+        if value >= NULL_VALUE_INT and byte_count <= 2:
+            return TWO_BYTE_ENC + struct.pack("H", value)
+        if byte_count <= 3:
+            return THREE_BYTE_ENC + struct.pack("i", value)[:3]
+        if byte_count <= 8:
+            return THREE_BYTE_ENC + struct.pack("Q", value)
+
+    def toStringPacket(self):
+        return self.get_serializer()(self.value)
+
+    def get_serializer(self):
+        if self.var_type == "string":
+            if self.var_len == "lenenc":
+                if isinstance(self.value, bytes):
+                    return self.serialize_bytes
+                return self.serialize_str
+            if self.var_len == "EOF":
+                return self.serialize_str_eof
+            if self.var_len == "NUL":
+                return lambda v: bytes(v, "utf-8") + struct.pack("b", 0)
+            if self.var_len == "packet":
+                return lambda v: v.get_packet_string()
+            else:
+                return lambda v: struct.pack(self.var_len + "s", bytes(v, "utf-8"))[
+                    :int(self.var_len)
+                ]
+
+        if self.var_type == "int":
+            if self.var_len == "lenenc":
+                return self.serialize_int
+            else:
+                return lambda v: struct.pack("Q", v)[:int(self.var_len)]
+
+    @classmethod
+    def serialize_str_eof(cls, value):
+        length = len(value)
+        var_len = length
+        if length == 0:
+            return b""
+        else:
+            return struct.pack(
+                "{len}s".format(len=var_len), bytes(value, "utf-8")
+            )[:length]
+
+    # def serialize_obj(self, value):
+    #     return self.serialize_str(str(value))
+
+    @classmethod
+    def serialize_str(cls, value):
+        return cls.serialize_bytes(value.encode("utf8"))
+
+    @classmethod
+    def serialize_bytes(cls, value):
+
+        val_len = len(value)
+
+        if val_len == 0:
+            return b"\0"
+
+        if val_len < NULL_VALUE_INT:
+            return struct.pack("B", val_len) + value
+
+        byte_count = -(val_len.bit_length() // (-8))
+        if byte_count <= 2:
+            return (
+                TWO_BYTE_ENC
+                + struct.pack("H", val_len)
+                + value
+            )
+        if byte_count <= 3:
+            return (
+                THREE_BYTE_ENC
+                + struct.pack("i", val_len)[:3]
+                + value
+            )
+        if byte_count <= 8:
+            return (
+                THREE_BYTE_ENC
+                + struct.pack("Q", val_len)
+                + value
+            )
+
+
+def test():
+    import pprint
+
+    u = Datum("int<8>", DEFAULT_CAPABILITIES >> 16)
+    pprint.pprint(u.toStringPacket())
+
+
+# only run the test if this file is called from debugger
+if __name__ == "__main__":
+    test()

mindsdb/api/mysql/mysql_proxy/data_types/mysql_packet.py

@@ -0,0 +1,166 @@
+"""
+*******************************************************
+ * Copyright (C) 2017 MindsDB Inc. <copyright@mindsdb.com>
+ *
+ * This file is part of MindsDB Server.
+ *
+ * MindsDB Server can not be copied and/or distributed without the express
+ * permission of MindsDB Inc
+ *******************************************************
+"""
+
+import struct
+
+from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import MAX_PACKET_SIZE
+from mindsdb.utilities import log
+
+logger = log.getLogger(__name__)
+
+
+class Packet:
+    def __init__(
+        self,
+        length=0,
+        body="",
+        packet_string=None,
+        socket=None,
+        session=None,
+        proxy=None,
+        **kwargs,
+    ):
+        self.mysql_socket = socket
+        self.session = session
+        self.proxy = proxy
+        self._kwargs = kwargs
+
+        self.setup()
+        if packet_string is not None:
+            self.load_from_packet_string(packet_string)
+        else:
+            self.load_from_params(length, session.packet_sequence_number, body)
+
+    def setup(self, length=0, seq=0, body=None):
+        ...
+
+    def load_from_params(self, length, seq, body):
+        self._length = length
+        self._seq = seq
+        self._body = body
+
+    def setBody(self, body_string):
+        self._body = body_string
+        self._length = len(body_string)
+
+    def load_from_packet_string(self, packet_string):
+        len_header = struct.unpack("i", packet_string[:3] + b"\x00")[0]
+        count_header = int(packet_string[3])
+        body = packet_string[4:]
+        self.load_from_params(length=len_header, seq=count_header, body=body)
+
+    def get_packet_string(self):
+        body = self.body
+        len_header = struct.pack("<i", self.length)[:3]  # keep it 3 bytes
+        count_header = struct.pack("B", self.seq)
+        packet = len_header + count_header + body
+        return packet
+
+    def get(self):
+        self.session.logging.debug(f"Get packet: {self.__class__.__name__}")
+
+        len_header = MAX_PACKET_SIZE
+        body = b""
+        count_header = 1
+        while len_header == MAX_PACKET_SIZE:
+            packet_string = self.mysql_socket.recv(4)
+            if len(packet_string) < 4:
+                self.session.logging.debug(
+                    f"Packet with less than 4 bytes in length: {packet_string}"
+                )
+                return False
+                break
+            len_header = struct.unpack("i", packet_string[:3] + b"\x00")[0]
+            count_header = int(packet_string[3])
+            if len_header == 0:
+                break
+            body += self.mysql_socket.recv(len_header)
+        self.session.logging.debug(f"Got packet: {str(body)}")
+        self.session.packet_sequence_number = (int(count_header) + 1) % 256
+        self.setup(len(body), count_header, body)
+        return True
+
+    def send(self):
+        string = self.get_packet_string()
+        self.session.logging.debug(f"Sending packet: {self.__class__.__name__}")
+        self.session.logging.debug(string)
+        self.mysql_socket.sendall(string)
+
+    def accum(self):
+        string = self.get_packet_string()
+        return string
+
+    def pprintPacket(self, body=None):
+        if body is None:
+            body = self.body
+        logger.info(str(self))
+        for i, x in enumerate(body):
+            part = "[BODY]"
+            logger.info(
+                """{part}{i}:{h} ({inte}:{actual})""".format(
+                    part=part, i=i + 1, h=hex(ord(x)), inte=ord(x), actual=str(x)
+                )
+            )
+
+    def isEOF(self):
+        if self.length == 0:
+            return True
+        else:
+            return False
+
+    @property
+    def length(self):
+        # self._length = len(self.body)
+        return self._length
+
+    @property
+    def seq(self):
+        return self._seq
+
+    @property
+    def body(self):
+        return self._body
+
+    @staticmethod
+    def bodyStringToPackets(body_string):
+        """
+        The method takes a string and turns it into mysql_packets
+
+        :param body_string: text to turn into mysql_packets
+        :return: a list of mysql_packets
+
+        """
+        ret = []
+        body_len = len(body_string)
+        mod = body_len % MAX_PACKET_SIZE
+        num_packets = body_len // MAX_PACKET_SIZE + (1 if mod > 0 else 0)
+
+        for i in range(num_packets):
+            left_limit = i * MAX_PACKET_SIZE
+            right_limit = mod if i + 1 == num_packets else MAX_PACKET_SIZE * (i + 1)
+            body = body_string[left_limit:right_limit]
+            ret += [Packet(length=right_limit, seq=i + 1, body=body)]
+
+        return ret
+
+    def __str__(self):
+        return str({"body": self.body, "length": self.length, "seq": self.seq})
+
+
+def test():
+    import pprint
+
+    pprint.pprint(Packet.bodyStringToPackets("abdds")[0].get_packet_string())
+
+
+# only run the test if this file is called from debugger
+if __name__ == "__main__":
+    test()

mindsdb/api/mysql/mysql_proxy/data_types/mysql_packets/__init__.py

@@ -0,0 +1,15 @@
+from mindsdb.api.mysql.mysql_proxy.data_types.mysql_packets.err_packet import ErrPacket
+from mindsdb.api.mysql.mysql_proxy.data_types.mysql_packets.handshake_packet import HandshakePacket
+from mindsdb.api.mysql.mysql_proxy.data_types.mysql_packets.fast_auth_fail_packet import FastAuthFail
+from mindsdb.api.mysql.mysql_proxy.data_types.mysql_packets.password_answer import PasswordAnswer
+from mindsdb.api.mysql.mysql_proxy.data_types.mysql_packets.handshake_response_packet import HandshakeResponsePacket
+from mindsdb.api.mysql.mysql_proxy.data_types.mysql_packets.ok_packet import OkPacket
+from mindsdb.api.mysql.mysql_proxy.data_types.mysql_packets.switch_auth_packet import SwitchOutPacket
+from mindsdb.api.mysql.mysql_proxy.data_types.mysql_packets.switch_auth_response_packet import SwitchOutResponse
+from mindsdb.api.mysql.mysql_proxy.data_types.mysql_packets.command_packet import CommandPacket
+from mindsdb.api.mysql.mysql_proxy.data_types.mysql_packets.column_count_packet import ColumnCountPacket
+from mindsdb.api.mysql.mysql_proxy.data_types.mysql_packets.column_definition_packet import ColumnDefenitionPacket
+from mindsdb.api.mysql.mysql_proxy.data_types.mysql_packets.resultset_row_package import ResultsetRowPacket
+from mindsdb.api.mysql.mysql_proxy.data_types.mysql_packets.eof_packet import EofPacket
+from mindsdb.api.mysql.mysql_proxy.data_types.mysql_packets.stmt_prepare_header import STMTPrepareHeaderPacket
+from mindsdb.api.mysql.mysql_proxy.data_types.mysql_packets.binary_resultset_row_package import BinaryResultsetRowPacket

mindsdb/api/mysql/mysql_proxy/data_types/mysql_packets/binary_resultset_row_package.py

@@ -0,0 +1,140 @@
+"""
+*******************************************************
+ * Copyright (C) 2017 MindsDB Inc. <copyright@mindsdb.com>
+ *
+ * This file is part of MindsDB Server.
+ *
+ * MindsDB Server can not be copied and/or distributed without the express
+ * permission of MindsDB Inc
+ *******************************************************
+"""
+import datetime as dt
+import struct
+
+import pandas as pd
+
+from mindsdb.api.mysql.mysql_proxy.data_types.mysql_datum import Datum
+from mindsdb.api.mysql.mysql_proxy.data_types.mysql_packet import Packet
+from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import TYPES
+
+
+class BinaryResultsetRowPacket(Packet):
+    '''
+    Implementation based on:
+    https://mariadb.com/kb/en/resultset-row/#binary-resultset-row
+    https://dev.mysql.com/doc/internals/en/null-bitmap.html
+    '''
+
+    def setup(self):
+        data = self._kwargs.get('data', {})
+        columns = self._kwargs.get('columns', {})
+
+        self.value = [b'\x00']
+        nulls = [0]
+        for i, el in enumerate(data):
+            if i > 0 and (i + 2) % 8 == 0:
+                nulls.append(0)
+            if el is None:
+                if i < 6:
+                    nulls[-1] = nulls[-1] + (1 << ((i + 2) % 8))
+                else:
+                    nulls[-1] = nulls[-1] + (1 << ((i - 6) % 8))
+        self.value.append(bytes(nulls))
+
+        for i, col in enumerate(columns):
+            # NOTE at this moment all types sends as strings, and it works
+            val = data[i]
+            if val is None:
+                continue
+
+            enc = None
+            env_val = None
+            col_type = col['type']
+            if col_type == TYPES.MYSQL_TYPE_DOUBLE:
+                enc = '<d'
+                val = float(val)
+            elif col_type == TYPES.MYSQL_TYPE_LONGLONG:
+                enc = '<q'
+                val = int(val)
+            elif col_type == TYPES.MYSQL_TYPE_LONG:
+                enc = '<l'
+                val = int(val)
+            elif col_type == TYPES.MYSQL_TYPE_FLOAT:
+                enc = '<f'
+                val = float(val)
+            elif col_type == TYPES.MYSQL_TYPE_YEAR:
+                enc = '<h'
+                val = int(float(val))
+            elif col_type == TYPES.MYSQL_TYPE_DATE:
+                env_val = self.encode_date(val)
+            elif col_type == TYPES.MYSQL_TYPE_TIMESTAMP:
+                env_val = self.encode_date(val)
+            elif col_type == TYPES.MYSQL_TYPE_DATETIME:
+                env_val = self.encode_date(val)
+            elif col_type == TYPES.MYSQL_TYPE_TIME:
+                enc = ''
+            elif col_type == TYPES.MYSQL_TYPE_NEWDECIMAL:
+                enc = ''
+            else:
+                enc = 'string'
+
+            if enc == '':
+                raise Exception(f'Column with type {col_type} cant be encripted')
+
+            if enc == 'string':
+                if not isinstance(val, str):
+                    val = str(val)
+                self.value.append(Datum('string', val, 'lenenc').toStringPacket())
+            else:
+                if env_val is None:
+                    env_val = struct.pack(enc, val)
+                self.value.append(env_val)
+
+    def encode_date(self, val):
+        # date_type = None
+        # date_value = None
+
+        if isinstance(val, str):
+            try:
+                date_value = dt.datetime.strptime(val, '%Y-%m-%d')
+                date_type = 'date'
+            except ValueError:
+                try:
+                    date_value = dt.datetime.strptime(val, '%Y-%m-%dT%H:%M:%S')
+                    date_type = 'datetime'
+                except ValueError:
+                    date_value = dt.datetime.strptime(val, '%Y-%m-%dT%H:%M:%S.%f')
+                    date_type = 'datetime'
+        elif isinstance(val, pd.Timestamp):
+            date_value = val
+            date_type = 'datetime'
+
+        out = struct.pack('<H', date_value.year)
+        out += struct.pack('<B', date_value.month)
+        out += struct.pack('<B', date_value.day)
+
+        if date_type == 'datetime':
+            out += struct.pack('<B', date_value.hour)
+            out += struct.pack('<B', date_value.minute)
+            out += struct.pack('<B', date_value.second)
+            out += struct.pack('<L', date_value.microsecond)
+
+        len_bit = struct.pack('<B', len(out))
+        return len_bit + out
+
+    @property
+    def body(self):
+        string = b''.join(self.value)
+        self.setBody(string)
+        return self._body
+
+    @staticmethod
+    def test():
+        import pprint
+        pprint.pprint(
+            str(BinaryResultsetRowPacket().get_packet_string())
+        )
+
+
+if __name__ == "__main__":
+    BinaryResultsetRowPacket.test()