MindsDB 25.4.3.2__py3-none-any.whl → 25.4.4.0__py3-none-any.whl

mindsdb/integrations/handlers/mssql_handler/mssql_handler.py

@@ -241,14 +241,23 @@ class SqlServerHandler(DatabaseHandler):
 
         query = f"""
             SELECT
-                column_name as "Field",
-                data_type as "Type"
+                COLUMN_NAME,
+                DATA_TYPE,
+                ORDINAL_POSITION,
+                COLUMN_DEFAULT,
+                IS_NULLABLE,
+                CHARACTER_MAXIMUM_LENGTH,
+                CHARACTER_OCTET_LENGTH,
+                NUMERIC_PRECISION,
+                NUMERIC_SCALE,
+                DATETIME_PRECISION,
+                CHARACTER_SET_NAME,
+                COLLATION_NAME
             FROM
                 information_schema.columns
             WHERE
                 table_name = '{table_name}'
         """
         result = self.native_query(query)
-        if result.resp_type is RESPONSE_TYPE.TABLE:
-            result.data_frame['mysql_data_type'] = result.data_frame['Type'].apply(_map_type)
+        result.to_columns_table_response(map_type_fn=_map_type)
         return result

mindsdb/integrations/handlers/mysql_handler/mysql_handler.py

@@ -231,14 +231,23 @@ class MySQLHandler(DatabaseHandler):
         """
         q = f"""
             select
-                COLUMN_NAME AS FIELD, DATA_TYPE AS TYPE
+                COLUMN_NAME,
+                DATA_TYPE,
+                ORDINAL_POSITION,
+                COLUMN_DEFAULT,
+                IS_NULLABLE,
+                CHARACTER_MAXIMUM_LENGTH,
+                CHARACTER_OCTET_LENGTH,
+                NUMERIC_PRECISION,
+                NUMERIC_SCALE,
+                DATETIME_PRECISION,
+                CHARACTER_SET_NAME,
+                COLLATION_NAME
             from
                 information_schema.columns
             where
-                table_name = '{table_name}'
+                table_name = '{table_name}';
         """
         result = self.native_query(q)
-        if result.resp_type is RESPONSE_TYPE.TABLE:
-            result.data_frame = result.data_frame.rename(columns={'FIELD': 'Field', 'TYPE': 'Type'})
-            result.data_frame['mysql_data_type'] = result.data_frame['Type'].apply(_map_type)
+        result.to_columns_table_response(map_type_fn=_map_type)
         return result

mindsdb/integrations/handlers/oracle_handler/oracle_handler.py

@@ -282,13 +282,23 @@ class OracleHandler(DatabaseHandler):
         """
         query = f"""
             SELECT
-                column_name AS field,
-                data_type AS type
+                COLUMN_NAME,
+                DATA_TYPE,
+                COLUMN_ID AS ORDINAL_POSITION,
+                DATA_DEFAULT AS COLUMN_DEFAULT,
+                CASE NULLABLE WHEN 'Y' THEN 'YES' ELSE 'NO' END AS IS_NULLABLE,
+                CHAR_LENGTH AS CHARACTER_MAXIMUM_LENGTH,
+                NULL AS CHARACTER_OCTET_LENGTH,
+                DATA_PRECISION AS NUMERIC_PRECISION,
+                DATA_SCALE AS NUMERIC_SCALE,
+                NULL AS DATETIME_PRECISION,
+                CHARACTER_SET_NAME,
+                NULL AS COLLATION_NAME
             FROM USER_TAB_COLUMNS
             WHERE table_name = '{table_name}'
+            ORDER BY TABLE_NAME, COLUMN_ID;
         """
         result = self.native_query(query)
         if result.resp_type is RESPONSE_TYPE.TABLE:
-            result.data_frame.columns = [name.lower() for name in result.data_frame.columns]
-            result.data_frame['mysql_data_type'] = result.data_frame['type'].apply(_map_type)
+            result.to_columns_table_response(map_type_fn=_map_type)
         return result

mindsdb/integrations/handlers/pgvector_handler/pgvector_handler.py

@@ -40,8 +40,31 @@ class PgVectorHandler(PostgresHandler, VectorStoreHandler):
         # we get these from the connection args on PostgresHandler parent
         self._is_sparse = self.connection_args.get('is_sparse', False)
         self._vector_size = self.connection_args.get('vector_size', None) 
-        if self._is_sparse and not self._vector_size:
-            raise ValueError("vector_size is required when is_sparse=True")
+
+        if self._is_sparse:
+            if not self._vector_size:
+                raise ValueError("vector_size is required when is_sparse=True")
+
+                # Use inner product for sparse vectors
+                distance_op = "<#>"
+
+        else:
+            distance_op = '<=>'
+            if 'distance' in self.connection_args:
+                distance_ops = {
+                    'l1': '<+>',
+                    'l2': '<->',
+                    'ip': '<#>',  # inner product
+                    'cosine': '<=>',
+                    'hamming': '<~>',
+                    'jaccard': '<%>'
+                }
+
+                distance_op = distance_ops.get(self.connection_args['distance'])
+                if distance_op is None:
+                    raise ValueError(f'Wrong distance type. Allowed options are {list(distance_ops.keys())}')
+
+        self.distance_op = distance_op
         self.connect()
 
     def _make_connection_args(self):
@@ -224,20 +247,16 @@ class PgVectorHandler(PostgresHandler, VectorStoreHandler):
                         from pgvector.utils import SparseVector
                         embedding = SparseVector(search_vector, self._vector_size)
                         search_vector = embedding.to_text()
-                    # Use inner product for sparse vectors
-                    distance_op = "<#>"
                 else:
                     # Convert list to vector string if needed
                     if isinstance(search_vector, list):
                         search_vector = f"[{','.join(str(x) for x in search_vector)}]"
-                    # Use cosine similarity for dense vectors
-                    distance_op = "<=>"
 
                 # Calculate distance as part of the query if needed
                 if has_distance:
-                    targets = f"{targets}, (embeddings {distance_op} '{search_vector}') as distance"
+                    targets = f"{targets}, (embeddings {self.distance_op} '{search_vector}') as distance"
                 
-                return f"SELECT {targets} FROM {table_name} {where_clause} ORDER BY embeddings {distance_op} '{search_vector}' ASC {limit_clause} {offset_clause} "
+                return f"SELECT {targets} FROM {table_name} {where_clause} ORDER BY embeddings {self.distance_op} '{search_vector}' ASC {limit_clause} {offset_clause} "
 
             else:
                 # if filter conditions, return rows that satisfy the conditions
@@ -418,18 +437,14 @@ class PgVectorHandler(PostgresHandler, VectorStoreHandler):
         """
         table_name = self._check_table(table_name)
 
-        data_dict = data.to_dict(orient="list")
-
-        if 'metadata' in data_dict:
-            data_dict['metadata'] = [json.dumps(i) for i in data_dict['metadata']]
-        transposed_data = list(zip(*data_dict.values()))
-
-        columns = ", ".join(data.keys())
-        values = ", ".join(["%s"] * len(data.keys()))
+        if 'metadata' in data.columns:
+             data['metadata'] = data['metadata'].apply(json.dumps)
 
-        insert_statement = f"INSERT INTO {table_name} ({columns}) VALUES ({values})"
-
-        self.raw_query(insert_statement, params=transposed_data)
+        resp = super().insert(table_name, data)
+        if resp.resp_type == RESPONSE_TYPE.ERROR:
+            raise RuntimeError(resp.error_message)
+        if resp.resp_type == RESPONSE_TYPE.TABLE:
+            return resp.data_frame
 
     def update(
         self, table_name: str, data: pd.DataFrame, key_columns: List[str] = None

mindsdb/integrations/handlers/postgres_handler/postgres_handler.py

@@ -1,7 +1,6 @@
 import time
 import json
 from typing import Optional
-import threading
 
 import pandas as pd
 import psycopg
@@ -43,7 +42,8 @@ def _map_type(internal_type_name: str) -> MYSQL_DATA_TYPE:
         ('real', 'money', 'float'): MYSQL_DATA_TYPE.FLOAT,
         ('numeric', 'decimal'): MYSQL_DATA_TYPE.DECIMAL,
         ('double precision',): MYSQL_DATA_TYPE.DOUBLE,
-        ('character varying', 'varchar', 'character', 'char', 'bpchar', 'bpchar', 'text'): MYSQL_DATA_TYPE.TEXT,
+        ('character varying', 'varchar'): MYSQL_DATA_TYPE.VARCHAR,
+        ('character', 'char', 'bpchar', 'bpchar', 'text'): MYSQL_DATA_TYPE.TEXT,
         ('timestamp', 'timestamp without time zone', 'timestamp with time zone'): MYSQL_DATA_TYPE.DATETIME,
         ('date', ): MYSQL_DATA_TYPE.DATE,
         ('time', 'time without time zone', 'time with time zone'): MYSQL_DATA_TYPE.TIME,
@@ -76,9 +76,7 @@ class PostgresHandler(DatabaseHandler):
 
         self.connection = None
         self.is_connected = False
-        self.thread_safe = True
-
-        self._insert_lock = threading.Lock()
+        self.thread_safe = False
 
     def __del__(self):
         if self.is_connected:
@@ -266,15 +264,13 @@ class PostgresHandler(DatabaseHandler):
 
         columns = df.columns
 
-        # postgres 'copy' is not thread safe. use lock to prevent concurrent execution
-        with self._insert_lock:
-            resp = self.get_columns(table_name)
+        resp = self.get_columns(table_name)
 
         # copy requires precise cases of names: get current column names from table and adapt input dataframe columns
         if resp.data_frame is not None and not resp.data_frame.empty:
             db_columns = {
                 c.lower(): c
-                for c in resp.data_frame['field']
+                for c in resp.data_frame['COLUMN_NAME']
             }
 
             # try to get case of existing column
@@ -288,11 +284,10 @@ class PostgresHandler(DatabaseHandler):
 
         with connection.cursor() as cur:
             try:
-                with self._insert_lock:
-                    with cur.copy(f'copy "{table_name}" ({",".join(columns)}) from STDIN WITH CSV') as copy:
-                        df.to_csv(copy, index=False, header=False)
+                with cur.copy(f'copy "{table_name}" ({",".join(columns)}) from STDIN WITH CSV') as copy:
+                    df.to_csv(copy, index=False, header=False)
 
-                    connection.commit()
+                connection.commit()
             except Exception as e:
                 logger.error(f'Error running insert to {table_name} on {self.database}, {e}!')
                 connection.rollback()
@@ -366,8 +361,18 @@ class PostgresHandler(DatabaseHandler):
             schema_name = 'current_schema()'
         query = f"""
             SELECT
-                column_name as "Field",
-                data_type as "Type"
+                COLUMN_NAME,
+                DATA_TYPE,
+                ORDINAL_POSITION,
+                COLUMN_DEFAULT,
+                IS_NULLABLE,
+                CHARACTER_MAXIMUM_LENGTH,
+                CHARACTER_OCTET_LENGTH,
+                NUMERIC_PRECISION,
+                NUMERIC_SCALE,
+                DATETIME_PRECISION,
+                CHARACTER_SET_NAME,
+                COLLATION_NAME
             FROM
                 information_schema.columns
             WHERE
@@ -376,9 +381,7 @@ class PostgresHandler(DatabaseHandler):
                 table_schema = {schema_name}
         """
         result = self.native_query(query)
-        if result.resp_type is RESPONSE_TYPE.TABLE:
-            result.data_frame.columns = [name.lower() for name in result.data_frame.columns]
-            result.data_frame['mysql_data_type'] = result.data_frame['type'].apply(_map_type)
+        result.to_columns_table_response(map_type_fn=_map_type)
         return result
 
     def subscribe(self, stop_event, callback, table_name, columns=None, **kwargs):

mindsdb/integrations/handlers/snowflake_handler/snowflake_handler.py

@@ -340,14 +340,24 @@ class SnowflakeHandler(DatabaseHandler):
             raise ValueError("Invalid table name provided.")
 
         query = f"""
-            SELECT COLUMN_NAME AS FIELD, DATA_TYPE AS TYPE
+            SELECT
+                COLUMN_NAME,
+                DATA_TYPE,
+                ORDINAL_POSITION,
+                COLUMN_DEFAULT,
+                IS_NULLABLE,
+                CHARACTER_MAXIMUM_LENGTH,
+                CHARACTER_OCTET_LENGTH,
+                NUMERIC_PRECISION,
+                NUMERIC_SCALE,
+                DATETIME_PRECISION,
+                CHARACTER_SET_NAME,
+                COLLATION_NAME
             FROM INFORMATION_SCHEMA.COLUMNS
             WHERE TABLE_NAME = '{table_name}'
               AND TABLE_SCHEMA = current_schema()
         """
         result = self.native_query(query)
-        if result.resp_type is RESPONSE_TYPE.TABLE:
-            result.data_frame = result.data_frame.rename(columns={'FIELD': 'Field', 'TYPE': 'Type'})
-            result.data_frame['mysql_data_type'] = result.data_frame['Type'].apply(_map_type)
+        result.to_columns_table_response(map_type_fn=_map_type)
 
         return result

mindsdb/integrations/handlers/web_handler/urlcrawl_helpers.py

@@ -151,7 +151,7 @@ def get_all_website_links(url) -> dict:
             # Parse HTML content with BeautifulSoup
             soup = BeautifulSoup(content_html, "html.parser")
             content_text = get_readable_text_from_soup(soup)
-            for a_tag in soup.findAll("a"):
+            for a_tag in soup.find_all("a"):
                 href = a_tag.attrs.get("href")
                 if href == "" or href is None:
                     continue

mindsdb/integrations/libs/response.py

@@ -1,5 +1,8 @@
-from typing import Optional
-from pandas import DataFrame
+from typing import Callable
+from dataclasses import dataclass, fields
+
+import numpy
+import pandas
 
 from mindsdb.utilities import log
 from mindsdb.api.executor.data_types.response_type import RESPONSE_TYPE
@@ -8,9 +11,36 @@ from mindsdb_sql_parser.ast import ASTNode
 
 logger = log.getLogger(__name__)
 
+
+@dataclass(frozen=True)
+class _INFORMATION_SCHEMA_COLUMNS_NAMES:
+    """Set of DataFrame columns that must be returned when calling `handler.get_columns(...)`.
+    These column names match the standard INFORMATION_SCHEMA.COLUMNS structure
+    used in SQL databases to describe table metadata.
+    """
+    COLUMN_NAME: str = 'COLUMN_NAME'
+    DATA_TYPE: str = 'DATA_TYPE'
+    ORDINAL_POSITION: str = 'ORDINAL_POSITION'
+    COLUMN_DEFAULT: str = 'COLUMN_DEFAULT'
+    IS_NULLABLE: str = 'IS_NULLABLE'
+    CHARACTER_MAXIMUM_LENGTH: str = 'CHARACTER_MAXIMUM_LENGTH'
+    CHARACTER_OCTET_LENGTH: str = 'CHARACTER_OCTET_LENGTH'
+    NUMERIC_PRECISION: str = 'NUMERIC_PRECISION'
+    NUMERIC_SCALE: str = 'NUMERIC_SCALE'
+    DATETIME_PRECISION: str = 'DATETIME_PRECISION'
+    CHARACTER_SET_NAME: str = 'CHARACTER_SET_NAME'
+    COLLATION_NAME: str = 'COLLATION_NAME'
+    MYSQL_DATA_TYPE: str = 'MYSQL_DATA_TYPE'
+
+
+INF_SCHEMA_COLUMNS_NAMES = _INFORMATION_SCHEMA_COLUMNS_NAMES()
+INF_SCHEMA_COLUMNS_NAMES_SET = set(f.name for f in fields(INF_SCHEMA_COLUMNS_NAMES))
+
+
+
 class HandlerResponse:
-    def __init__(self, resp_type: RESPONSE_TYPE, data_frame: DataFrame = None, query: ASTNode = 0, error_code: int = 0,
-                 error_message: Optional[str] = None, affected_rows: Optional[int] = None) -> None:
+    def __init__(self, resp_type: RESPONSE_TYPE, data_frame: pandas.DataFrame = None, query: ASTNode = 0, error_code: int = 0,
+                 error_message: str | None = None, affected_rows: int | None = None) -> None:
         self.resp_type = resp_type
         self.query = query
         self.data_frame = data_frame
@@ -24,6 +54,51 @@ class HandlerResponse:
     def type(self):
         return self.resp_type
 
+    def to_columns_table_response(self, map_type_fn: Callable) -> None:
+        """Transform the response to a `columns table` response.
+        NOTE: original dataframe will be mutated
+        """
+        if self.resp_type == RESPONSE_TYPE.COLUMNS_TABLE:
+            return
+        if self.resp_type != RESPONSE_TYPE.TABLE:
+            if self.resp_type == RESPONSE_TYPE.ERROR:
+                raise ValueError(
+                    f"Cannot convert {self.resp_type} to {RESPONSE_TYPE.COLUMNS_TABLE}, "
+                    f"the error is: {self.error_message}"
+                )
+            raise ValueError(f"Cannot convert {self.resp_type} to {RESPONSE_TYPE.COLUMNS_TABLE}")
+
+        self.data_frame.columns = [name.upper() for name in self.data_frame.columns]
+        self.data_frame[INF_SCHEMA_COLUMNS_NAMES.MYSQL_DATA_TYPE] = self.data_frame[
+            INF_SCHEMA_COLUMNS_NAMES.DATA_TYPE
+        ].apply(map_type_fn)
+
+        # region validate df
+        current_columns_set = set(self.data_frame.columns)
+        if INF_SCHEMA_COLUMNS_NAMES_SET != current_columns_set:
+            raise ValueError(
+                f'Columns set for INFORMATION_SCHEMA.COLUMNS is wrong: {list(current_columns_set)}'
+            )
+        # endregion
+
+        self.data_frame = self.data_frame.astype({
+            INF_SCHEMA_COLUMNS_NAMES.COLUMN_NAME: 'string',
+            INF_SCHEMA_COLUMNS_NAMES.DATA_TYPE: 'string',
+            INF_SCHEMA_COLUMNS_NAMES.ORDINAL_POSITION: 'Int32',
+            INF_SCHEMA_COLUMNS_NAMES.COLUMN_DEFAULT: 'string',
+            INF_SCHEMA_COLUMNS_NAMES.IS_NULLABLE: 'string',
+            INF_SCHEMA_COLUMNS_NAMES.CHARACTER_MAXIMUM_LENGTH: 'Int32',
+            INF_SCHEMA_COLUMNS_NAMES.CHARACTER_OCTET_LENGTH: 'Int32',
+            INF_SCHEMA_COLUMNS_NAMES.NUMERIC_PRECISION: 'Int32',
+            INF_SCHEMA_COLUMNS_NAMES.NUMERIC_SCALE: 'Int32',
+            INF_SCHEMA_COLUMNS_NAMES.DATETIME_PRECISION: 'Int32',
+            INF_SCHEMA_COLUMNS_NAMES.CHARACTER_SET_NAME: 'string',
+            INF_SCHEMA_COLUMNS_NAMES.COLLATION_NAME: 'string',
+        })
+        self.data_frame.replace([numpy.NaN, pandas.NA], None, inplace=True)
+
+        self.resp_type = RESPONSE_TYPE.COLUMNS_TABLE
+
     def to_json(self):
         try:
             data = None
@@ -49,6 +124,7 @@ class HandlerResponse:
                 self.affected_rows
             )
 
+
 class HandlerStatusResponse:
     def __init__(self, success: bool = True,
                  error_message: str = None,
@@ -70,31 +146,3 @@ class HandlerStatusResponse:
         return f"{self.__class__.__name__}: success={self.success},\
               error={self.error_message},\
               redirect_url={self.redirect_url}"
-
-
-class ExecutorResponse:
-    def __init__(self, resp_type: RESPONSE_TYPE, query: object, error_code: int = 0, error_message: str = None):
-        self.resp_type = resp_type
-        self.query = query
-
-        self.error_code = error_code
-        self.error_message = error_message
-
-    @property
-    def type(self):
-        return self.resp_type
-
-    def to_json(self):
-        return  {"type": self.resp_type,
-                 "query": self.query,
-                 "error_code": self.error_code,
-                 "error": self.error_message}
-
-    def __repr__(self):
-        return "%s: resp_type=%s, query=%s, err_code=%s, error=%s" % (
-                self.__class__.__name__,
-                self.resp_type,
-                self.query,
-                self.error_code,
-                self.error_message,
-            )