MindsDB 25.3.4.1__py3-none-any.whl → 25.3.4.2__py3-none-any.whl

mindsdb/__about__.py

@@ -1,6 +1,6 @@
 __title__ = 'MindsDB'
 __package_name__ = 'mindsdb'
-__version__ = '25.3.4.1'
+__version__ = '25.3.4.2'
 __description__ = "MindsDB's AI SQL Server enables developers to build AI tools that need access to real-time data to perform their tasks"
 __email__ = "jorge@mindsdb.com"
 __author__ = 'MindsDB Inc'

mindsdb/api/executor/datahub/datanodes/integration_datanode.py

@@ -64,6 +64,7 @@ class IntegrationDataNode(DataNode):
             df = response.data_frame
             # case independent
             columns = [str(c).lower() for c in df.columns]
+            df.columns = columns
 
             col_name = None
             # looking for specific column names
@@ -78,8 +79,10 @@ class IntegrationDataNode(DataNode):
             names = df[df.columns[col_name]]
 
             # type
-            if 'type' in columns:
-                types = df[df.columns[columns.index('type')]]
+            if 'mysql_data_type' in columns:
+                types = df['mysql_data_type']
+            elif 'type' in columns:
+                types = df['type']
             else:
                 types = [None] * len(names)
 

mindsdb/api/executor/datahub/datanodes/system_tables.py

@@ -1,14 +1,17 @@
+from typing import Optional, Literal
+from dataclasses import dataclass, astuple, fields
 
 import pandas as pd
 from mindsdb_sql_parser.ast.base import ASTNode
-from mindsdb.integrations.utilities.sql_utils import extract_comparison_conditions
 
+from mindsdb.utilities import log
+from mindsdb.utilities.config import config
+from mindsdb.integrations.utilities.sql_utils import extract_comparison_conditions
+from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import MYSQL_DATA_TYPE
 from mindsdb.api.executor.datahub.classes.tables_row import (
     TABLES_ROW_TYPE,
     TablesRow,
 )
-from mindsdb.utilities import log
-from mindsdb.utilities.config import config
 
 logger = log.getLogger(__name__)
 
@@ -162,132 +165,108 @@ class TablesTable(Table):
         return df
 
 
-class ColumnsTable(Table):
+@dataclass
+class ColumnsTableRow:
+    """Represents a row in the COLUMNS table.
+    Fields description: https://dev.mysql.com/doc/refman/8.4/en/information-schema-columns-table.html
+    NOTE: attrs order matter, don't change it.
+    """
+    TABLE_CATALOG: Literal['def'] = 'def'
+    TABLE_SCHEMA: Optional[str] = None
+    TABLE_NAME: Optional[str] = None
+    COLUMN_NAME: Optional[str] = None
+    ORDINAL_POSITION: int = 0
+    COLUMN_DEFAULT: Optional[str] = None
+    IS_NULLABLE: Literal['YES', 'NO'] = 'YES'
+    DATA_TYPE: str = MYSQL_DATA_TYPE.VARCHAR.value
+    CHARACTER_MAXIMUM_LENGTH: Optional[int] = None
+    CHARACTER_OCTET_LENGTH: Optional[int] = None
+    NUMERIC_PRECISION: Optional[int] = None
+    NUMERIC_SCALE: Optional[int] = None
+    DATETIME_PRECISION: Optional[int] = None
+    CHARACTER_SET_NAME: Optional[str] = None
+    COLLATION_NAME: Optional[str] = None
+    COLUMN_TYPE: Optional[str] = None
+    COLUMN_KEY: Optional[str] = None
+    EXTRA: Optional[str] = None
+    PRIVILEGES: str = 'select'
+    COLUMN_COMMENT: Optional[str] = None
+    GENERATION_EXPRESSION: Optional[str] = None
+
+    def __post_init__(self):
+        # region check mandatory fields
+        mandatory_fields = ['TABLE_SCHEMA', 'TABLE_NAME', 'COLUMN_NAME']
+        if any(getattr(self, field_name) is None for field_name in mandatory_fields):
+            raise ValueError('One of mandatory fields is missed when creating ColumnsTableRow')
+        # endregion
+
+        # region set default values depend on type
+        defaults = {
+            'COLUMN_TYPE': self.DATA_TYPE
+        }
+        if MYSQL_DATA_TYPE(self.DATA_TYPE) in (
+            MYSQL_DATA_TYPE.TIMESTAMP,
+            MYSQL_DATA_TYPE.DATETIME,
+            MYSQL_DATA_TYPE.DATE
+        ):
+            defaults = {
+                'DATETIME_PRECISION': 0,
+                'COLUMN_TYPE': self.DATA_TYPE
+            }
+        elif MYSQL_DATA_TYPE(self.DATA_TYPE) in (
+            MYSQL_DATA_TYPE.FLOAT,
+            MYSQL_DATA_TYPE.DOUBLE,
+            MYSQL_DATA_TYPE.DECIMAL
+        ):
+            defaults = {
+                'NUMERIC_PRECISION': 12,
+                'NUMERIC_SCALE': 0,
+                'COLUMN_TYPE': self.DATA_TYPE
+            }
+        elif MYSQL_DATA_TYPE(self.DATA_TYPE) in (
+            MYSQL_DATA_TYPE.TINYINT,
+            MYSQL_DATA_TYPE.SMALLINT,
+            MYSQL_DATA_TYPE.MEDIUMINT,
+            MYSQL_DATA_TYPE.INT,
+            MYSQL_DATA_TYPE.BIGINT
+        ):
+            defaults = {
+                'NUMERIC_PRECISION': 20,
+                'NUMERIC_SCALE': 0,
+                'COLUMN_TYPE': self.DATA_TYPE
+            }
+        elif MYSQL_DATA_TYPE(self.DATA_TYPE) is MYSQL_DATA_TYPE.VARCHAR:
+            defaults = {
+                'CHARACTER_MAXIMUM_LENGTH': 1024,
+                'CHARACTER_OCTET_LENGTH': 3072,
+                'CHARACTER_SET_NAME': 'utf8',
+                'COLLATION_NAME': 'utf8_bin',
+                'COLUMN_TYPE': 'varchar(1024)'
+            }
+        else:
+            # show as MYSQL_DATA_TYPE.TEXT:
+            defaults = {
+                'CHARACTER_MAXIMUM_LENGTH': 65535,      # from https://bugs.mysql.com/bug.php?id=90685
+                'CHARACTER_OCTET_LENGTH': 65535,        #
+                'CHARACTER_SET_NAME': 'utf8',
+                'COLLATION_NAME': 'utf8_bin',
+                'COLUMN_TYPE': 'text'
+            }
+
+        for key, value in defaults.items():
+            setattr(self, key, value)
 
+        self.DATA_TYPE = self.DATA_TYPE.lower()
+        self.COLUMN_TYPE = self.COLUMN_TYPE.lower()
+        # endregion
+
+
+class ColumnsTable(Table):
     name = 'COLUMNS'
-    columns = [
-        "TABLE_CATALOG",
-        "TABLE_SCHEMA",
-        "TABLE_NAME",
-        "COLUMN_NAME",
-        "ORDINAL_POSITION",
-        "COLUMN_DEFAULT",
-        "IS_NULLABLE",
-        "DATA_TYPE",
-        "CHARACTER_MAXIMUM_LENGTH",
-        "CHARACTER_OCTET_LENGTH",
-        "NUMERIC_PRECISION",
-        "NUMERIC_SCALE",
-        "DATETIME_PRECISION",
-        "CHARACTER_SET_NAME",
-        "COLLATION_NAME",
-        "COLUMN_TYPE",
-        "COLUMN_KEY",
-        "EXTRA",
-        "PRIVILEGES",
-        "COLUMN_COMMENT",
-        "GENERATION_EXPRESSION",
-    ]
+    columns = [field.name for field in fields(ColumnsTableRow)]
 
     @classmethod
     def get_data(cls, inf_schema=None, query: ASTNode = None, **kwargs):
-
-        # NOTE there is a lot of types in mysql, but listed below should be enough for our purposes
-        row_templates = {
-            "text": [
-                "def",
-                "SCHEMA_NAME",
-                "TABLE_NAME",
-                "COLUMN_NAME",
-                "COL_INDEX",
-                None,
-                "YES",
-                "varchar",
-                1024,
-                3072,
-                None,
-                None,
-                None,
-                "utf8",
-                "utf8_bin",
-                "varchar(1024)",
-                None,
-                None,
-                "select",
-                None,
-                None,
-            ],
-            "timestamp": [
-                "def",
-                "SCHEMA_NAME",
-                "TABLE_NAME",
-                "COLUMN_NAME",
-                "COL_INDEX",
-                "CURRENT_TIMESTAMP",
-                "YES",
-                "timestamp",
-                None,
-                None,
-                None,
-                None,
-                0,
-                None,
-                None,
-                "timestamp",
-                None,
-                None,
-                "select",
-                None,
-                None,
-            ],
-            "bigint": [
-                "def",
-                "SCHEMA_NAME",
-                "TABLE_NAME",
-                "COLUMN_NAME",
-                "COL_INDEX",
-                None,
-                "YES",
-                "bigint",
-                None,
-                None,
-                20,
-                0,
-                None,
-                None,
-                None,
-                "bigint unsigned",
-                None,
-                None,
-                "select",
-                None,
-                None,
-            ],
-            "float": [
-                "def",
-                "SCHEMA_NAME",
-                "TABLE_NAME",
-                "COLUMN_NAME",
-                "COL_INDEX",
-                None,
-                "YES",
-                "float",
-                None,
-                None,
-                12,
-                0,
-                None,
-                None,
-                None,
-                "float",
-                None,
-                None,
-                "select",
-                None,
-                None,
-            ],
-        }
-
         result = []
 
         databases, tables_names = _get_scope(query)
@@ -320,20 +299,34 @@ class ColumnsTable(Table):
                 for i, column in enumerate(table_columns):
                     column_name = column['name']
                     column_type = column.get('type', 'text')
-                    if column_type in ('double precision', 'real', 'numeric'):
-                        column_type = 'float'
-                    elif column_type in ('integer', 'smallint', 'int'):
-                        column_type = 'bigint'
-                    elif column_type in ('timestamp without time zone', 'timestamp with time zone', 'date'):
-                        column_type = 'timestamp'
-                    elif column_type not in row_templates:
-                        column_type = 'text'
-                    result_row = row_templates[column_type].copy()
-                    result_row[1] = db_name
-                    result_row[2] = table_name
-                    result_row[3] = column_name
-                    result_row[4] = i
-                    result.append(result_row)
+
+                    # region infer type
+                    if isinstance(column_type, MYSQL_DATA_TYPE) is False:
+                        if column_type in ('double precision', 'real', 'numeric', 'float'):
+                            column_type = MYSQL_DATA_TYPE.FLOAT
+                        elif column_type in ('integer', 'smallint', 'int', 'bigint'):
+                            column_type = MYSQL_DATA_TYPE.BIGINT
+                        elif column_type in (
+                            'timestamp without time zone',
+                            'timestamp with time zone',
+                            'date', 'timestamp'
+                        ):
+                            column_type = MYSQL_DATA_TYPE.DATETIME
+                        else:
+                            column_type = MYSQL_DATA_TYPE.VARCHAR
+                    # endregion
+
+                    column_row = astuple(
+                        ColumnsTableRow(
+                            TABLE_SCHEMA=db_name,
+                            TABLE_NAME=table_name,
+                            COLUMN_NAME=column_name,
+                            DATA_TYPE=column_type.value,
+                            ORDINAL_POSITION=i
+                        )
+                    )
+
+                    result.append(column_row)
 
         df = pd.DataFrame(result, columns=cls.columns)
         return df

mindsdb/api/mysql/mysql_proxy/libs/constants/mysql.py

@@ -8,6 +8,7 @@
  * permission of MindsDB Inc
  *******************************************************
 """
+import enum
 
 # CAPABILITIES
 # As defined in : https://dev.mysql.com/doc/dev/mysql-server/8.0.0/group__group__cs__capabilities__flags.html
@@ -101,6 +102,8 @@ COMMANDS = COMMANDS()
 
 
 # FIELD TYPES
+# https://dev.mysql.com/doc/dev/mysql-server/latest/field__types_8h_source.html
+# https://mariadb.com/kb/en/result-set-packets/
 class TYPES(object):
     __slots__ = ()
     MYSQL_TYPE_DECIMAL = 0
@@ -123,6 +126,11 @@ class TYPES(object):
     MYSQL_TYPE_TIMESTAMP2 = 17
     MYSQL_TYPE_DATETIME2 = 18
     MYSQL_TYPE_TIME2 = 19
+    MYSQL_TYPE_TYPED_ARRAY = 20
+    MYSQL_TYPE_VECTOR = 242,
+    MYSQL_TYPE_INVALID = 243,
+    MYSQL_TYPE_BOOL = 244,
+    MYSQL_TYPE_JSON = 245,
     MYSQL_TYPE_NEWDECIMAL = 246
     MYSQL_TYPE_ENUM = 247
     MYSQL_TYPE_SET = 248
@@ -135,9 +143,75 @@ class TYPES(object):
     MYSQL_TYPE_GEOMETRY = 255
 
 
+C_TYPES = TYPES()
 TYPES = TYPES()
 
 
+class MYSQL_DATA_TYPE(enum.Enum):
+    TINYINT = 'TINYINT'
+    SMALLINT = 'SMALLINT'
+    MEDIUMINT = 'MEDIUMINT'
+    INT = 'INT'
+    BIGINT = 'BIGINT'
+    FLOAT = 'FLOAT'
+    DOUBLE = 'DOUBLE'
+    DECIMAL = 'DECIMAL'
+    YEAR = 'YEAR'
+    TIME = 'TIME'
+    DATE = 'DATE'
+    DATETIME = 'DATETIME'
+    TIMESTAMP = 'TIMESTAMP'
+    CHAR = 'CHAR'
+    BINARY = 'BINARY'
+    VARCHAR = 'VARCHAR'
+    VARBINARY = 'VARBINARY'
+    TINYBLOB = 'TINYBLOB'
+    TINYTEXT = 'TINYTEXT'
+    BLOB = 'BLOB'
+    TEXT = 'TEXT'
+    MEDIUMBLOB = 'MEDIUMBLOB'
+    MEDIUMTEXT = 'MEDIUMTEXT'
+    LONGBLOB = 'LONGBLOB'
+    LONGTEXT = 'LONGTEXT'
+    BIT = 'BIT'
+    BOOL = 'BOOL'
+    BOOLEAN = 'BOOLEAN'
+
+
+# Map between data types and C types
+# https://dev.mysql.com/doc/c-api/8.0/en/c-api-prepared-statement-type-codes.html
+DATA_C_TYPE_MAP = {
+    MYSQL_DATA_TYPE.TINYINT: C_TYPES.MYSQL_TYPE_TINY,
+    MYSQL_DATA_TYPE.SMALLINT: C_TYPES.MYSQL_TYPE_SHORT,
+    MYSQL_DATA_TYPE.MEDIUMINT: C_TYPES.MYSQL_TYPE_INT24,
+    MYSQL_DATA_TYPE.INT: C_TYPES.MYSQL_TYPE_LONG,
+    MYSQL_DATA_TYPE.BIGINT: C_TYPES.MYSQL_TYPE_LONGLONG,
+    MYSQL_DATA_TYPE.FLOAT: C_TYPES.MYSQL_TYPE_FLOAT,
+    MYSQL_DATA_TYPE.DOUBLE: C_TYPES.MYSQL_TYPE_DOUBLE,
+    MYSQL_DATA_TYPE.DECIMAL: C_TYPES.MYSQL_TYPE_NEWDECIMAL,
+    MYSQL_DATA_TYPE.YEAR: C_TYPES.MYSQL_TYPE_SHORT,
+    MYSQL_DATA_TYPE.TIME: C_TYPES.MYSQL_TYPE_TIME,
+    MYSQL_DATA_TYPE.DATE: C_TYPES.MYSQL_TYPE_DATE,
+    MYSQL_DATA_TYPE.DATETIME: C_TYPES.MYSQL_TYPE_DATETIME,
+    MYSQL_DATA_TYPE.TIMESTAMP: C_TYPES.MYSQL_TYPE_TIMESTAMP,
+    MYSQL_DATA_TYPE.CHAR: C_TYPES.MYSQL_TYPE_STRING,
+    MYSQL_DATA_TYPE.BINARY: C_TYPES.MYSQL_TYPE_STRING,
+    MYSQL_DATA_TYPE.VARCHAR: C_TYPES.MYSQL_TYPE_VAR_STRING,
+    MYSQL_DATA_TYPE.VARBINARY: C_TYPES.MYSQL_TYPE_VAR_STRING,
+    MYSQL_DATA_TYPE.TINYBLOB: C_TYPES.MYSQL_TYPE_TINY_BLOB,
+    MYSQL_DATA_TYPE.TINYTEXT: C_TYPES.MYSQL_TYPE_TINY_BLOB,
+    MYSQL_DATA_TYPE.BLOB: C_TYPES.MYSQL_TYPE_BLOB,
+    MYSQL_DATA_TYPE.TEXT: C_TYPES.MYSQL_TYPE_BLOB,
+    MYSQL_DATA_TYPE.MEDIUMBLOB: C_TYPES.MYSQL_TYPE_MEDIUM_BLOB,
+    MYSQL_DATA_TYPE.MEDIUMTEXT: C_TYPES.MYSQL_TYPE_MEDIUM_BLOB,
+    MYSQL_DATA_TYPE.LONGBLOB: C_TYPES.MYSQL_TYPE_LONG_BLOB,
+    MYSQL_DATA_TYPE.LONGTEXT: C_TYPES.MYSQL_TYPE_LONG_BLOB,
+    MYSQL_DATA_TYPE.BIT: C_TYPES.MYSQL_TYPE_BIT,
+    MYSQL_DATA_TYPE.BOOL: C_TYPES.MYSQL_TYPE_TINY,
+    MYSQL_DATA_TYPE.BOOLEAN: C_TYPES.MYSQL_TYPE_TINY
+}
+
+
 class FIELD_FLAG(object):
     __slots__ = ()
     NOT_NULL = 1                # field cannot be null

mindsdb/integrations/handlers/confluence_handler/confluence_api_client.py

@@ -149,8 +149,20 @@ class ConfluenceAPIClient:
         results.extend(response["results"])
 
         while response["_links"].get("next"):
-            params["cursor"] = response["_links"].get("next")
-            response = self._make_request("GET", url, params)
+            next_url = response["_links"].get("next")
+            next_params = {}
+            if params:
+                next_params.update(params)
+            if "cursor=" in next_url:
+                # cursor= is 7 characters long
+                cursor_start = next_url.find("cursor=") + 7
+                cursor_value = next_url[cursor_start:]
+                if "&" in cursor_value:
+                    cursor_value = cursor_value.split("&")[0]
+                next_params["cursor"] = cursor_value
+                response = self._make_request("GET", url, next_params)
+            else:
+                response = self._make_request("GET", next_url)
             results.extend(response["results"])
 
         return results