MindsDB 25.9.3rc1__py3-none-any.whl → 25.10.0rc1__py3-none-any.whl

mindsdb/integrations/handlers/oracle_handler/oracle_handler.py

@@ -1,11 +1,11 @@
-from typing import Text, Dict, Optional, Any
+from typing import Any, Dict, List, Optional, Text
 
 import oracledb
 import pandas as pd
 from oracledb import connect, Connection, DatabaseError, Cursor
 from mindsdb_sql_parser.ast.base import ASTNode
 
-from mindsdb.integrations.libs.base import DatabaseHandler
+from mindsdb.integrations.libs.base import MetaDatabaseHandler
 from mindsdb.integrations.libs.response import (
     HandlerStatusResponse as StatusResponse,
     HandlerResponse as Response,
@@ -13,6 +13,7 @@ from mindsdb.integrations.libs.response import (
 )
 from mindsdb.utilities import log
 from mindsdb.utilities.render.sqlalchemy_render import SqlalchemyRender
+import mindsdb.utilities.profiler as profiler
 from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import MYSQL_DATA_TYPE
 
 
@@ -40,7 +41,6 @@ def _map_type(internal_type_name: str) -> MYSQL_DATA_TYPE:
             "NATIONAL CHARACTER",
             "NATIONAL CHAR",
             "VARCHAR",
-            "NCHAR",
             "NATIONAL CHARACTER VARYING",
             "NATIONAL CHAR VARYING",
             "NCHAR VARYING",
@@ -53,8 +53,18 @@ def _map_type(internal_type_name: str) -> MYSQL_DATA_TYPE:
         ("BINARY_DOUBLE",): MYSQL_DATA_TYPE.DOUBLE,
         ("LONG",): MYSQL_DATA_TYPE.BIGINT,
         ("DATE",): MYSQL_DATA_TYPE.DATE,
-        ("HOUR", "MINUTE", "SECOND", "TIMEZONE_HOUR", "TIMEZONE_MINUTE"): MYSQL_DATA_TYPE.SMALLINT,
-        ("TIMESTAMP", "TIMESTAMP WITH TIME ZONE", "TIMESTAMP WITH LOCAL TIME ZONE"): MYSQL_DATA_TYPE.TIMESTAMP,
+        (
+            "HOUR",
+            "MINUTE",
+            "SECOND",
+            "TIMEZONE_HOUR",
+            "TIMEZONE_MINUTE",
+        ): MYSQL_DATA_TYPE.SMALLINT,
+        (
+            "TIMESTAMP",
+            "TIMESTAMP WITH TIME ZONE",
+            "TIMESTAMP WITH LOCAL TIME ZONE",
+        ): MYSQL_DATA_TYPE.TIMESTAMP,
         ("RAW", "LONG RAW", "BLOB", "BFILE"): MYSQL_DATA_TYPE.BINARY,
         ("ROWID", "UROWID"): MYSQL_DATA_TYPE.TEXT,
         ("CHAR", "NCHAR", "CLOB", "NCLOB", "CHARACTER"): MYSQL_DATA_TYPE.CHAR,
@@ -147,7 +157,7 @@ def _make_table_response(result: list[tuple[Any]], cursor: Cursor) -> Response:
     return Response(RESPONSE_TYPE.TABLE, data_frame=df, mysql_types=mysql_types)
 
 
-class OracleHandler(DatabaseHandler):
+class OracleHandler(MetaDatabaseHandler):
     """
     This handler handles connection and execution of SQL queries on Oracle.
     """
@@ -160,7 +170,7 @@ class OracleHandler(DatabaseHandler):
 
         Args:
             name (Text): The name of the handler instance.
-            connection_data (Dict): The connection data required to connect to Amazon DynamoDB.
+            connection_data (Dict): The connection data required to connect to OracleDB.
             kwargs: Arbitrary keyword arguments.
         """
         super().__init__(name)
@@ -188,7 +198,16 @@ class OracleHandler(DatabaseHandler):
             raise ValueError("Required parameters (user, password) must be provided.")
 
         if self.connection_data.get("thick_mode", False):
-            oracledb.init_oracle_client()
+            oracle_client_lib_dir = self.connection_data.get("oracle_client_lib_dir")
+            if isinstance(oracle_client_lib_dir, str) and oracle_client_lib_dir.strip():
+                try:
+                    oracledb.init_oracle_client(lib_dir=oracle_client_lib_dir)
+                except Exception as e:
+                    raise ValueError(f"Failed to initialize Oracle client: {e}")
+            else:
+                raise ValueError(
+                    "Parameter 'oracle_client_lib_dir' must be provided as a non-empty string when using thick_mode."
+                )
 
         config = {
             "user": self.connection_data["user"],
@@ -240,7 +259,7 @@ class OracleHandler(DatabaseHandler):
             raise
 
         except Exception as unknown_error:
-            logger.error(f"Unknown error when connecting to Elasticsearch: {unknown_error}")
+            logger.error(f"Unknown error when connecting to Oracle: {unknown_error}")
             raise
 
         self.is_connected = True
@@ -285,6 +304,7 @@ class OracleHandler(DatabaseHandler):
 
         return response
 
+    @profiler.profile()
     def native_query(self, query: Text) -> Response:
         """
         Executes a SQL query on the Oracle database and returns the result.
@@ -325,9 +345,69 @@ class OracleHandler(DatabaseHandler):
 
         if need_to_close is True:
             self.disconnect()
-
         return response
 
+    def query_stream(self, query: ASTNode, fetch_size: int = 1000):
+        """
+        Executes a SQL query represented by an ASTNode and retrieves the data in a streaming fashion.
+
+        Args:
+            query (ASTNode): An ASTNode representing the SQL query to be executed.
+            fetch_size (int): The number of rows to fetch in each batch.
+        Yields:
+            pd.DataFrame: A DataFrame containing a batch of rows from the query result.
+            Response: In case of an error, yields a Response object with the error details.
+        """
+        query_str = SqlalchemyRender("oracle").get_string(query, with_failback=True)
+        need_to_close = self.is_connected is False
+
+        connection = self.connect()
+        with connection.cursor() as cur:
+            try:
+                cur.execute(query_str)
+                while True:
+                    result = cur.fetchmany(fetch_size)
+                    if not result:
+                        break
+                    df = pd.DataFrame(result, columns=[col[0] for col in cur.description])
+                    yield df
+                connection.commit()
+            finally:
+                connect
+        if need_to_close is True:
+            self.disconnect()
+
+    def insert(self, table_name: str, df: pd.DataFrame) -> Response:
+        """
+        Inserts data from a DataFrame into a specified table in the Oracle database.
+
+        Args:
+            table_name (str): The name of the table where the data will be inserted.
+            df (pd.DataFrame): The DataFrame containing the data to be inserted.
+        Returns:
+            Response: A response object indicating the success or failure of the insert operation.
+        """
+        need_to_close = self.is_connected is False
+        connection = self.connect()
+        columns = list(df.columns)
+        placeholders = ", ".join([f":{i + 1}" for i in range(len(columns))])
+        insert_query = f"INSERT INTO {table_name} ({', '.join(columns)}) VALUES ({placeholders})"
+
+        with connection.cursor() as cur:
+            try:
+                cur.executemany(insert_query, df.values.tolist())
+                connection.commit()
+                rowcount = cur.rowcount
+            except DatabaseError as database_error:
+                logger.error(f"Error inserting data into table {table_name} on Oracle, {database_error}!")
+                connection.rollback()
+                raise
+        if need_to_close is True:
+            self.disconnect()
+
+        return Response(RESPONSE_TYPE.OK, affected_rows=rowcount)
+
+    @profiler.profile()
     def query(self, query: ASTNode) -> Response:
         """
         Executes a SQL query represented by an ASTNode and retrieves the data.
@@ -349,12 +429,29 @@ class OracleHandler(DatabaseHandler):
         Returns:
             Response: A response object containing the list of tables and views, formatted as per the `Response` class.
         """
-        # TODO: This query does not seem to be correct.
         query = """
-            SELECT table_name
-            FROM user_tables
-            ORDER BY 1
-        """
+            SELECT
+                owner AS table_schema,
+                table_name AS table_name,
+                'BASE TABLE' AS table_type
+            FROM all_tables t
+            JOIN all_users u ON t.owner = u.username
+            WHERE t.tablespace_name = 'USERS'
+
+            UNION ALL
+
+            SELECT
+                v.owner AS table_schema,
+                v.view_name AS table_name,
+                'VIEW' AS table_type
+            FROM all_views v
+            JOIN all_users u ON v.owner = u.username
+            WHERE v.owner IN (
+                SELECT DISTINCT owner
+                FROM all_tables
+                WHERE tablespace_name = 'USERS'
+            )
+            """
         return self.native_query(query)
 
     def get_columns(self, table_name: Text) -> Response:
@@ -385,9 +482,217 @@ class OracleHandler(DatabaseHandler):
                 NULL AS COLLATION_NAME
             FROM USER_TAB_COLUMNS
             WHERE table_name = '{table_name}'
-            ORDER BY TABLE_NAME, COLUMN_ID;
+            ORDER BY TABLE_NAME, COLUMN_ID
         """
         result = self.native_query(query)
         if result.resp_type is RESPONSE_TYPE.TABLE:
             result.to_columns_table_response(map_type_fn=_map_type)
         return result
+
+    def meta_get_tables(self, table_names: Optional[List[str]]) -> Response:
+        """
+        Retrieves metadata about all non-system tables and views in the current schema of the Oracle database.
+
+        Returns:
+            list[dict[str, Any]]: A list of dictionaries, each containing metadata about a table or view.
+        """
+        query = """
+            SELECT
+                o.object_name AS table_name,
+                USER AS table_schema,
+                o.object_type AS table_type,
+                c.comments AS table_description,
+                t.num_rows AS row_count
+            FROM
+                user_objects o
+            LEFT JOIN
+                user_tab_comments c ON o.object_name = c.table_name
+            LEFT JOIN
+                user_tables t ON o.object_name = t.table_name AND o.object_type = 'TABLE'
+            WHERE
+                o.object_type IN ('TABLE', 'VIEW')
+        """
+        if table_names is not None and len(table_names) > 0:
+            table_names = [f"'{t.upper()}'" for t in table_names]
+            query += f" AND o.object_name IN ({','.join(table_names)})"
+
+        query += " ORDER BY o.object_name"
+
+        result = self.native_query(query)
+        return result
+
+    def meta_get_columns(self, table_names: Optional[List[str]]) -> Response:
+        """Retrieves metadata about the columns of specified tables in the Oracle database.
+
+        Args:
+            table_names (list[str]): A list of table names for which to retrieve column metadata.
+
+        Returns:
+            list[dict[str, Any]]: A list of dictionaries, each containing metadata about a column.
+        """
+        query = """
+            SELECT
+                utc.table_name,
+                utc.column_name,
+                utc.data_type,
+                ucc.comments AS column_description,
+                utc.data_default AS column_default,
+                CASE
+                    WHEN utc.nullable = 'Y' THEN 1
+                    ELSE 0
+                END AS is_nullable
+            FROM
+                user_tab_columns utc
+            JOIN
+                user_tables ut ON utc.table_name = ut.table_name
+            LEFT JOIN
+                user_col_comments ucc ON utc.table_name = ucc.table_name AND utc.column_name = ucc.column_name
+        """
+        if table_names is not None and len(table_names) > 0:
+            table_names = [f"'{t.upper()}'" for t in table_names]
+            query += f" WHERE utc.table_name IN ({','.join(table_names)})"
+        query += " ORDER BY utc.table_name, utc.column_id"
+        result = self.native_query(query)
+        return result
+
+    def meta_get_column_statistics(self, table_names: Optional[List[str]]) -> Response:
+        """Retrieves statistics about the columns of specified tables in the Oracle database.
+
+        Args:
+            table_names (list[str]): A list of table names for which to retrieve column statistics.
+
+        Returns:
+            list[dict[str, Any]]: A list of dictionaries, each containing statistics about a column.
+        """
+        table_filter = ""
+        if table_names is not None and len(table_names) > 0:
+            quoted_names = [f"'{t.upper()}'" for t in table_names]
+            table_filter = f" WHERE cs.table_name IN ({','.join(quoted_names)})"
+
+        query = (
+            """
+            SELECT
+                cs.table_name AS TABLE_NAME,
+                cs.column_name AS COLUMN_NAME,
+                CASE
+                    WHEN cs.sample_size > 0 THEN ROUND((cs.num_nulls / cs.sample_size) * 100, 2)
+                    ELSE NULL
+                END AS NULL_PERCENTAGE,
+                cs.num_distinct AS DISTINCT_VALUES_COUNT,
+                NULL AS MOST_COMMON_VALUES,
+                NULL AS MOST_COMMON_FREQUENCIES,
+                cs.histogram AS HISTOGRAM_TYPE,
+                h.bounds AS HISTOGRAM_BOUNDS
+            FROM
+                user_tab_col_statistics cs
+            LEFT JOIN (
+                SELECT
+                    table_name,
+                    column_name,
+                    LISTAGG(endpoint_value, ', ') WITHIN GROUP (ORDER BY endpoint_number) AS bounds
+                FROM
+                    user_tab_histograms
+                GROUP BY
+                    table_name,
+                    column_name
+            ) h ON cs.table_name = h.table_name AND cs.column_name = h.column_name
+            """
+            + table_filter
+            + """
+            ORDER BY
+                cs.table_name,
+                cs.column_name
+            """
+        )
+
+        result = self.native_query(query)
+
+        if result.resp_type is RESPONSE_TYPE.TABLE and result.data_frame is not None:
+            df = result.data_frame
+
+            def extract_min_max(
+                histogram_str: str,
+            ) -> tuple[Optional[float], Optional[float]]:
+                if histogram_str and str(histogram_str).lower() not in ["nan", "none"]:
+                    values = str(histogram_str).split(",")
+                    if values:
+                        min_val = values[0].strip(" '\"")
+                        max_val = values[-1].strip(" '\"")
+                        return min_val, max_val
+                return None, None
+
+            min_max_values = df["HISTOGRAM_BOUNDS"].apply(extract_min_max)
+            df["MINIMUM_VALUE"] = min_max_values.apply(lambda x: x[0])
+            df["MAXIMUM_VALUE"] = min_max_values.apply(lambda x: x[1])
+            df.drop(columns=["HISTOGRAM_BOUNDS"], inplace=True)
+        return result
+
+    def meta_get_primary_keys(self, table_names: Optional[List[str]]) -> Response:
+        """
+        Retrieves the primary keys for the specified tables in the Oracle database.
+
+        Args:
+            table_names (list[str]): A list of table names for which to retrieve primary keys.
+
+        Returns:
+            list[dict[str, Any]]: A list of dictionaries, each containing information about a primary key.
+        """
+
+        query = """
+            SELECT
+                cols.table_name,
+                cols.column_name,
+                cols.position AS ordinal_position,
+                cons.constraint_name
+            FROM
+                all_constraints cons
+            JOIN
+                all_cons_columns cols ON cons.constraint_name = cols.constraint_name AND cons.owner = cols.owner
+            WHERE
+                cons.constraint_type = 'P'
+                AND cons.owner = SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA')
+        """
+        if table_names is not None and len(table_names) > 0:
+            quoted_names = [f"'{t.upper()}'" for t in table_names]
+            query += f" AND cols.table_name IN ({','.join(quoted_names)})"
+
+        query += " ORDER BY cols.table_name, cols.position"
+
+        result = self.native_query(query)
+        return result
+
+    def meta_get_foreign_keys(self, table_names: Optional[List[str]]) -> Response:
+        """
+        Retrieves the foreign keys for the specified tables in the Oracle database.
+
+        Args:
+            table_names (list[str]): A list of table names for which to retrieve foreign keys.
+
+        Returns:
+            list[dict[str, Any]]: A list of dictionaries, each containing information about a foreign key.
+        """
+
+        query = """
+        SELECT
+            pk_cols.table_name AS parent_table_name,
+            pk_cols.column_name AS parent_column_name,
+            fk_cols.table_name AS child_table_name,
+            fk_cols.column_name AS child_column_name,
+            fk_cons.constraint_name
+        FROM
+            all_constraints fk_cons
+        JOIN
+            all_cons_columns fk_cols ON fk_cons.owner = fk_cols.owner AND fk_cons.constraint_name = fk_cols.constraint_name
+        JOIN
+            all_cons_columns pk_cols ON fk_cons.owner = pk_cols.owner AND fk_cons.r_constraint_name = pk_cols.constraint_name
+        WHERE
+            fk_cons.constraint_type = 'R'
+            AND fk_cons.owner = SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA')
+        """
+        if table_names is not None and len(table_names) > 0:
+            quoted_names = [f"'{t.upper()}'" for t in table_names]
+            query += f" AND fk_cols.table_name IN ({','.join(quoted_names)})"
+
+        query += " ORDER BY fk_cols.table_name, fk_cols.position"
+        result = self.native_query(query)
+        return result

mindsdb/integrations/handlers/oracle_handler/requirements.txt

@@ -1 +1 @@
-oracledb==2.4.1
+oracledb==3.3.0

mindsdb/integrations/handlers/postgres_handler/postgres_handler.py

@@ -147,7 +147,7 @@ class PostgresHandler(MetaDatabaseHandler):
 
         self.connection = None
         self.is_connected = False
-        self.thread_safe = False
+        self.thread_safe = True
 
     def __del__(self):
         if self.is_connected:
@@ -314,7 +314,7 @@ class PostgresHandler(MetaDatabaseHandler):
                 if logger.isEnabledFor(logging.DEBUG):
                     log_message += f". Executed query:\n{query}"
                 logger.info(log_message)
-                response = Response(RESPONSE_TYPE.ERROR, error_code=0, error_message=str(e), is_acceptable_error=True)
+                response = Response(RESPONSE_TYPE.ERROR, error_code=0, error_message=str(e), is_expected_error=True)
                 connection.rollback()
             except Exception as e:
                 logger.error(f"Error running query:\n{query}\non {self.database}, {e}")