MindsDB 25.9.2.0a1__py3-none-any.whl → 25.10.0rc1__py3-none-any.whl

mindsdb/integrations/handlers/oracle_handler/oracle_handler.py

@@ -1,11 +1,11 @@
-from typing import Text, Dict, Optional, Any
+from typing import Any, Dict, List, Optional, Text
 
 import oracledb
 import pandas as pd
 from oracledb import connect, Connection, DatabaseError, Cursor
 from mindsdb_sql_parser.ast.base import ASTNode
 
-from mindsdb.integrations.libs.base import DatabaseHandler
+from mindsdb.integrations.libs.base import MetaDatabaseHandler
 from mindsdb.integrations.libs.response import (
     HandlerStatusResponse as StatusResponse,
     HandlerResponse as Response,
@@ -13,6 +13,7 @@ from mindsdb.integrations.libs.response import (
 )
 from mindsdb.utilities import log
 from mindsdb.utilities.render.sqlalchemy_render import SqlalchemyRender
+import mindsdb.utilities.profiler as profiler
 from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import MYSQL_DATA_TYPE
 
 
@@ -40,7 +41,6 @@ def _map_type(internal_type_name: str) -> MYSQL_DATA_TYPE:
             "NATIONAL CHARACTER",
             "NATIONAL CHAR",
             "VARCHAR",
-            "NCHAR",
             "NATIONAL CHARACTER VARYING",
             "NATIONAL CHAR VARYING",
             "NCHAR VARYING",
@@ -53,8 +53,18 @@ def _map_type(internal_type_name: str) -> MYSQL_DATA_TYPE:
         ("BINARY_DOUBLE",): MYSQL_DATA_TYPE.DOUBLE,
         ("LONG",): MYSQL_DATA_TYPE.BIGINT,
         ("DATE",): MYSQL_DATA_TYPE.DATE,
-        ("HOUR", "MINUTE", "SECOND", "TIMEZONE_HOUR", "TIMEZONE_MINUTE"): MYSQL_DATA_TYPE.SMALLINT,
-        ("TIMESTAMP", "TIMESTAMP WITH TIME ZONE", "TIMESTAMP WITH LOCAL TIME ZONE"): MYSQL_DATA_TYPE.TIMESTAMP,
+        (
+            "HOUR",
+            "MINUTE",
+            "SECOND",
+            "TIMEZONE_HOUR",
+            "TIMEZONE_MINUTE",
+        ): MYSQL_DATA_TYPE.SMALLINT,
+        (
+            "TIMESTAMP",
+            "TIMESTAMP WITH TIME ZONE",
+            "TIMESTAMP WITH LOCAL TIME ZONE",
+        ): MYSQL_DATA_TYPE.TIMESTAMP,
         ("RAW", "LONG RAW", "BLOB", "BFILE"): MYSQL_DATA_TYPE.BINARY,
         ("ROWID", "UROWID"): MYSQL_DATA_TYPE.TEXT,
         ("CHAR", "NCHAR", "CLOB", "NCLOB", "CHARACTER"): MYSQL_DATA_TYPE.CHAR,
@@ -147,7 +157,7 @@ def _make_table_response(result: list[tuple[Any]], cursor: Cursor) -> Response:
     return Response(RESPONSE_TYPE.TABLE, data_frame=df, mysql_types=mysql_types)
 
 
-class OracleHandler(DatabaseHandler):
+class OracleHandler(MetaDatabaseHandler):
     """
     This handler handles connection and execution of SQL queries on Oracle.
     """
@@ -160,7 +170,7 @@ class OracleHandler(DatabaseHandler):
 
         Args:
             name (Text): The name of the handler instance.
-            connection_data (Dict): The connection data required to connect to Amazon DynamoDB.
+            connection_data (Dict): The connection data required to connect to OracleDB.
             kwargs: Arbitrary keyword arguments.
         """
         super().__init__(name)
@@ -188,7 +198,16 @@ class OracleHandler(DatabaseHandler):
             raise ValueError("Required parameters (user, password) must be provided.")
 
         if self.connection_data.get("thick_mode", False):
-            oracledb.init_oracle_client()
+            oracle_client_lib_dir = self.connection_data.get("oracle_client_lib_dir")
+            if isinstance(oracle_client_lib_dir, str) and oracle_client_lib_dir.strip():
+                try:
+                    oracledb.init_oracle_client(lib_dir=oracle_client_lib_dir)
+                except Exception as e:
+                    raise ValueError(f"Failed to initialize Oracle client: {e}")
+            else:
+                raise ValueError(
+                    "Parameter 'oracle_client_lib_dir' must be provided as a non-empty string when using thick_mode."
+                )
 
         config = {
             "user": self.connection_data["user"],
@@ -240,7 +259,7 @@ class OracleHandler(DatabaseHandler):
             raise
 
         except Exception as unknown_error:
-            logger.error(f"Unknown error when connecting to Elasticsearch: {unknown_error}")
+            logger.error(f"Unknown error when connecting to Oracle: {unknown_error}")
             raise
 
         self.is_connected = True
@@ -285,6 +304,7 @@ class OracleHandler(DatabaseHandler):
 
         return response
 
+    @profiler.profile()
     def native_query(self, query: Text) -> Response:
         """
         Executes a SQL query on the Oracle database and returns the result.
@@ -325,9 +345,69 @@ class OracleHandler(DatabaseHandler):
 
         if need_to_close is True:
             self.disconnect()
-
         return response
 
+    def query_stream(self, query: ASTNode, fetch_size: int = 1000):
+        """
+        Executes a SQL query represented by an ASTNode and retrieves the data in a streaming fashion.
+
+        Args:
+            query (ASTNode): An ASTNode representing the SQL query to be executed.
+            fetch_size (int): The number of rows to fetch in each batch.
+        Yields:
+            pd.DataFrame: A DataFrame containing a batch of rows from the query result.
+            Response: In case of an error, yields a Response object with the error details.
+        """
+        query_str = SqlalchemyRender("oracle").get_string(query, with_failback=True)
+        need_to_close = self.is_connected is False
+
+        connection = self.connect()
+        with connection.cursor() as cur:
+            try:
+                cur.execute(query_str)
+                while True:
+                    result = cur.fetchmany(fetch_size)
+                    if not result:
+                        break
+                    df = pd.DataFrame(result, columns=[col[0] for col in cur.description])
+                    yield df
+                connection.commit()
+            finally:
+                connect
+        if need_to_close is True:
+            self.disconnect()
+
+    def insert(self, table_name: str, df: pd.DataFrame) -> Response:
+        """
+        Inserts data from a DataFrame into a specified table in the Oracle database.
+
+        Args:
+            table_name (str): The name of the table where the data will be inserted.
+            df (pd.DataFrame): The DataFrame containing the data to be inserted.
+        Returns:
+            Response: A response object indicating the success or failure of the insert operation.
+        """
+        need_to_close = self.is_connected is False
+        connection = self.connect()
+        columns = list(df.columns)
+        placeholders = ", ".join([f":{i + 1}" for i in range(len(columns))])
+        insert_query = f"INSERT INTO {table_name} ({', '.join(columns)}) VALUES ({placeholders})"
+
+        with connection.cursor() as cur:
+            try:
+                cur.executemany(insert_query, df.values.tolist())
+                connection.commit()
+                rowcount = cur.rowcount
+            except DatabaseError as database_error:
+                logger.error(f"Error inserting data into table {table_name} on Oracle, {database_error}!")
+                connection.rollback()
+                raise
+        if need_to_close is True:
+            self.disconnect()
+
+        return Response(RESPONSE_TYPE.OK, affected_rows=rowcount)
+
+    @profiler.profile()
     def query(self, query: ASTNode) -> Response:
         """
         Executes a SQL query represented by an ASTNode and retrieves the data.
@@ -349,12 +429,29 @@ class OracleHandler(DatabaseHandler):
         Returns:
             Response: A response object containing the list of tables and views, formatted as per the `Response` class.
         """
-        # TODO: This query does not seem to be correct.
         query = """
-            SELECT table_name
-            FROM user_tables
-            ORDER BY 1
-        """
+            SELECT
+                owner AS table_schema,
+                table_name AS table_name,
+                'BASE TABLE' AS table_type
+            FROM all_tables t
+            JOIN all_users u ON t.owner = u.username
+            WHERE t.tablespace_name = 'USERS'
+
+            UNION ALL
+
+            SELECT
+                v.owner AS table_schema,
+                v.view_name AS table_name,
+                'VIEW' AS table_type
+            FROM all_views v
+            JOIN all_users u ON v.owner = u.username
+            WHERE v.owner IN (
+                SELECT DISTINCT owner
+                FROM all_tables
+                WHERE tablespace_name = 'USERS'
+            )
+            """
         return self.native_query(query)
 
     def get_columns(self, table_name: Text) -> Response:
@@ -385,9 +482,217 @@ class OracleHandler(DatabaseHandler):
                 NULL AS COLLATION_NAME
             FROM USER_TAB_COLUMNS
             WHERE table_name = '{table_name}'
-            ORDER BY TABLE_NAME, COLUMN_ID;
+            ORDER BY TABLE_NAME, COLUMN_ID
         """
         result = self.native_query(query)
         if result.resp_type is RESPONSE_TYPE.TABLE:
             result.to_columns_table_response(map_type_fn=_map_type)
         return result
+
+    def meta_get_tables(self, table_names: Optional[List[str]]) -> Response:
+        """
+        Retrieves metadata about all non-system tables and views in the current schema of the Oracle database.
+
+        Returns:
+            list[dict[str, Any]]: A list of dictionaries, each containing metadata about a table or view.
+        """
+        query = """
+            SELECT
+                o.object_name AS table_name,
+                USER AS table_schema,
+                o.object_type AS table_type,
+                c.comments AS table_description,
+                t.num_rows AS row_count
+            FROM
+                user_objects o
+            LEFT JOIN
+                user_tab_comments c ON o.object_name = c.table_name
+            LEFT JOIN
+                user_tables t ON o.object_name = t.table_name AND o.object_type = 'TABLE'
+            WHERE
+                o.object_type IN ('TABLE', 'VIEW')
+        """
+        if table_names is not None and len(table_names) > 0:
+            table_names = [f"'{t.upper()}'" for t in table_names]
+            query += f" AND o.object_name IN ({','.join(table_names)})"
+
+        query += " ORDER BY o.object_name"
+
+        result = self.native_query(query)
+        return result
+
+    def meta_get_columns(self, table_names: Optional[List[str]]) -> Response:
+        """Retrieves metadata about the columns of specified tables in the Oracle database.
+
+        Args:
+            table_names (list[str]): A list of table names for which to retrieve column metadata.
+
+        Returns:
+            list[dict[str, Any]]: A list of dictionaries, each containing metadata about a column.
+        """
+        query = """
+            SELECT
+                utc.table_name,
+                utc.column_name,
+                utc.data_type,
+                ucc.comments AS column_description,
+                utc.data_default AS column_default,
+                CASE
+                    WHEN utc.nullable = 'Y' THEN 1
+                    ELSE 0
+                END AS is_nullable
+            FROM
+                user_tab_columns utc
+            JOIN
+                user_tables ut ON utc.table_name = ut.table_name
+            LEFT JOIN
+                user_col_comments ucc ON utc.table_name = ucc.table_name AND utc.column_name = ucc.column_name
+        """
+        if table_names is not None and len(table_names) > 0:
+            table_names = [f"'{t.upper()}'" for t in table_names]
+            query += f" WHERE utc.table_name IN ({','.join(table_names)})"
+        query += " ORDER BY utc.table_name, utc.column_id"
+        result = self.native_query(query)
+        return result
+
+    def meta_get_column_statistics(self, table_names: Optional[List[str]]) -> Response:
+        """Retrieves statistics about the columns of specified tables in the Oracle database.
+
+        Args:
+            table_names (list[str]): A list of table names for which to retrieve column statistics.
+
+        Returns:
+            list[dict[str, Any]]: A list of dictionaries, each containing statistics about a column.
+        """
+        table_filter = ""
+        if table_names is not None and len(table_names) > 0:
+            quoted_names = [f"'{t.upper()}'" for t in table_names]
+            table_filter = f" WHERE cs.table_name IN ({','.join(quoted_names)})"
+
+        query = (
+            """
+            SELECT
+                cs.table_name AS TABLE_NAME,
+                cs.column_name AS COLUMN_NAME,
+                CASE
+                    WHEN cs.sample_size > 0 THEN ROUND((cs.num_nulls / cs.sample_size) * 100, 2)
+                    ELSE NULL
+                END AS NULL_PERCENTAGE,
+                cs.num_distinct AS DISTINCT_VALUES_COUNT,
+                NULL AS MOST_COMMON_VALUES,
+                NULL AS MOST_COMMON_FREQUENCIES,
+                cs.histogram AS HISTOGRAM_TYPE,
+                h.bounds AS HISTOGRAM_BOUNDS
+            FROM
+                user_tab_col_statistics cs
+            LEFT JOIN (
+                SELECT
+                    table_name,
+                    column_name,
+                    LISTAGG(endpoint_value, ', ') WITHIN GROUP (ORDER BY endpoint_number) AS bounds
+                FROM
+                    user_tab_histograms
+                GROUP BY
+                    table_name,
+                    column_name
+            ) h ON cs.table_name = h.table_name AND cs.column_name = h.column_name
+            """
+            + table_filter
+            + """
+            ORDER BY
+                cs.table_name,
+                cs.column_name
+            """
+        )
+
+        result = self.native_query(query)
+
+        if result.resp_type is RESPONSE_TYPE.TABLE and result.data_frame is not None:
+            df = result.data_frame
+
+            def extract_min_max(
+                histogram_str: str,
+            ) -> tuple[Optional[float], Optional[float]]:
+                if histogram_str and str(histogram_str).lower() not in ["nan", "none"]:
+                    values = str(histogram_str).split(",")
+                    if values:
+                        min_val = values[0].strip(" '\"")
+                        max_val = values[-1].strip(" '\"")
+                        return min_val, max_val
+                return None, None
+
+            min_max_values = df["HISTOGRAM_BOUNDS"].apply(extract_min_max)
+            df["MINIMUM_VALUE"] = min_max_values.apply(lambda x: x[0])
+            df["MAXIMUM_VALUE"] = min_max_values.apply(lambda x: x[1])
+            df.drop(columns=["HISTOGRAM_BOUNDS"], inplace=True)
+        return result
+
+    def meta_get_primary_keys(self, table_names: Optional[List[str]]) -> Response:
+        """
+        Retrieves the primary keys for the specified tables in the Oracle database.
+
+        Args:
+            table_names (list[str]): A list of table names for which to retrieve primary keys.
+
+        Returns:
+            list[dict[str, Any]]: A list of dictionaries, each containing information about a primary key.
+        """
+
+        query = """
+            SELECT
+                cols.table_name,
+                cols.column_name,
+                cols.position AS ordinal_position,
+                cons.constraint_name
+            FROM
+                all_constraints cons
+            JOIN
+                all_cons_columns cols ON cons.constraint_name = cols.constraint_name AND cons.owner = cols.owner
+            WHERE
+                cons.constraint_type = 'P'
+                AND cons.owner = SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA')
+        """
+        if table_names is not None and len(table_names) > 0:
+            quoted_names = [f"'{t.upper()}'" for t in table_names]
+            query += f" AND cols.table_name IN ({','.join(quoted_names)})"
+
+        query += " ORDER BY cols.table_name, cols.position"
+
+        result = self.native_query(query)
+        return result
+
+    def meta_get_foreign_keys(self, table_names: Optional[List[str]]) -> Response:
+        """
+        Retrieves the foreign keys for the specified tables in the Oracle database.
+
+        Args:
+            table_names (list[str]): A list of table names for which to retrieve foreign keys.
+
+        Returns:
+            list[dict[str, Any]]: A list of dictionaries, each containing information about a foreign key.
+        """
+
+        query = """
+        SELECT
+            pk_cols.table_name AS parent_table_name,
+            pk_cols.column_name AS parent_column_name,
+            fk_cols.table_name AS child_table_name,
+            fk_cols.column_name AS child_column_name,
+            fk_cons.constraint_name
+        FROM
+            all_constraints fk_cons
+        JOIN
+            all_cons_columns fk_cols ON fk_cons.owner = fk_cols.owner AND fk_cons.constraint_name = fk_cols.constraint_name
+        JOIN
+            all_cons_columns pk_cols ON fk_cons.owner = pk_cols.owner AND fk_cons.r_constraint_name = pk_cols.constraint_name
+        WHERE
+            fk_cons.constraint_type = 'R'
+            AND fk_cons.owner = SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA')
+        """
+        if table_names is not None and len(table_names) > 0:
+            quoted_names = [f"'{t.upper()}'" for t in table_names]
+            query += f" AND fk_cols.table_name IN ({','.join(quoted_names)})"
+
+        query += " ORDER BY fk_cols.table_name, fk_cols.position"
+        result = self.native_query(query)
+        return result

mindsdb/integrations/handlers/oracle_handler/requirements.txt

@@ -1 +1 @@
-oracledb==2.4.1
+oracledb==3.3.0

mindsdb/integrations/handlers/postgres_handler/postgres_handler.py

@@ -1,5 +1,6 @@
 import time
 import json
+import logging
 from typing import Optional, Any
 
 import pandas as pd
@@ -146,7 +147,7 @@ class PostgresHandler(MetaDatabaseHandler):
 
         self.connection = None
         self.is_connected = False
-        self.thread_safe = False
+        self.thread_safe = True
 
     def __del__(self):
         if self.is_connected:
@@ -304,8 +305,19 @@ class PostgresHandler(MetaDatabaseHandler):
                     result = cur.fetchall()
                     response = _make_table_response(result, cur)
                 connection.commit()
+            except (psycopg.ProgrammingError, psycopg.DataError) as e:
+                # These is 'expected' exceptions, they should not be treated as mindsdb's errors
+                # ProgrammingError: table not found or already exists, syntax error, etc
+                # DataError: division by zero, numeric value out of range, etc.
+                # https://www.psycopg.org/psycopg3/docs/api/errors.html
+                log_message = "Database query failed with error, likely due to invalid SQL query"
+                if logger.isEnabledFor(logging.DEBUG):
+                    log_message += f". Executed query:\n{query}"
+                logger.info(log_message)
+                response = Response(RESPONSE_TYPE.ERROR, error_code=0, error_message=str(e), is_expected_error=True)
+                connection.rollback()
             except Exception as e:
-                logger.error(f"Error running query: {query} on {self.database}, {e}!")
+                logger.error(f"Error running query:\n{query}\non {self.database}, {e}")
                 response = Response(RESPONSE_TYPE.ERROR, error_code=0, error_message=str(e))
                 connection.rollback()
 

mindsdb/integrations/handlers/shopify_handler/shopify_handler.py

@@ -1,7 +1,17 @@
 import shopify
 import requests
 
-from mindsdb.integrations.handlers.shopify_handler.shopify_tables import ProductsTable, CustomersTable, OrdersTable, InventoryLevelTable, LocationTable, CustomerReviews, CarrierServiceTable, ShippingZoneTable, SalesChannelTable
+from mindsdb.integrations.handlers.shopify_handler.shopify_tables import (
+    ProductsTable,
+    CustomersTable,
+    OrdersTable,
+    InventoryLevelTable,
+    LocationTable,
+    CustomerReviews,
+    CarrierServiceTable,
+    ShippingZoneTable,
+    SalesChannelTable,
+)
 from mindsdb.integrations.libs.api_handler import APIHandler
 from mindsdb.integrations.libs.response import (
     HandlerStatusResponse as StatusResponse,
@@ -9,7 +19,11 @@ from mindsdb.integrations.libs.response import (
 
 from mindsdb.utilities import log
 from mindsdb_sql_parser import parse_sql
-from mindsdb.integrations.libs.api_handler_exceptions import InvalidNativeQuery, ConnectionFailed, MissingConnectionParams
+from mindsdb.integrations.libs.api_handler_exceptions import (
+    InvalidNativeQuery,
+    ConnectionFailed,
+    MissingConnectionParams,
+)
 
 logger = log.getLogger(__name__)
 
@@ -19,7 +33,7 @@ class ShopifyHandler(APIHandler):
     The Shopify handler implementation.
     """
 
-    name = 'shopify'
+    name = "shopify"
 
     def __init__(self, name: str, **kwargs):
         """
@@ -81,10 +95,12 @@ class ShopifyHandler(APIHandler):
         if self.kwargs.get("connection_data") is None:
             raise MissingConnectionParams("Incomplete parameters passed to Shopify Handler")
 
-        api_session = shopify.Session(self.connection_data['shop_url'], '2021-10', self.connection_data['access_token'])
+        api_session = shopify.Session(self.connection_data["shop_url"], "2021-10", self.connection_data["access_token"])
 
-        self.yotpo_app_key = self.connection_data['yotpo_app_key'] if 'yotpo_app_key' in self.connection_data else None
-        self.yotpo_access_token = self.connection_data['yotpo_access_token'] if 'yotpo_access_token' in self.connection_data else None
+        self.yotpo_app_key = self.connection_data["yotpo_app_key"] if "yotpo_app_key" in self.connection_data else None
+        self.yotpo_access_token = (
+            self.connection_data["yotpo_access_token"] if "yotpo_access_token" in self.connection_data else None
+        )
 
         self.connection = api_session
 
@@ -107,16 +123,13 @@ class ShopifyHandler(APIHandler):
             shopify.Shop.current()
             response.success = True
         except Exception as e:
-            logger.error('Error connecting to Shopify!')
-            raise ConnectionFailed("Conenction to Shopify failed.")
+            logger.error("Error connecting to Shopify!")
             response.error_message = str(e)
+            raise ConnectionFailed("Conenction to Shopify failed.")
 
         if self.yotpo_app_key is not None and self.yotpo_access_token is not None:
             url = f"https://api.yotpo.com/v1/apps/{self.yotpo_app_key}/reviews?count=1&utoken={self.yotpo_access_token}"
-            headers = {
-                "accept": "application/json",
-                "Content-Type": "application/json"
-            }
+            headers = {"accept": "application/json", "Content-Type": "application/json"}
             if requests.get(url, headers=headers).status_code == 200:
                 response.success = True
             else:

mindsdb/integrations/handlers/snowflake_handler/snowflake_handler.py

@@ -204,10 +204,11 @@ class SnowflakeHandler(MetaDatabaseHandler):
             "user": self.connection_data.get("user"),
             "password": self.connection_data.get("password"),
             "database": self.connection_data.get("database"),
+            "schema": self.connection_data.get("schema", "PUBLIC"),
         }
 
         # Optional connection parameters
-        optional_params = ["schema", "warehouse", "role"]
+        optional_params = ["warehouse", "role"]
         for param in optional_params:
             if param in self.connection_data:
                 config[param] = self.connection_data[param]

mindsdb/integrations/handlers/statsforecast_handler/requirements.txt

@@ -1,2 +1,3 @@
 statsforecast==1.6.0
 scipy==1.15.3
+numba >=0.55.0, <=0.61.2

mindsdb/integrations/handlers/statsforecast_handler/requirements_extra.txt

@@ -1,2 +1,3 @@
 statsforecast==1.6.0
 scipy==1.15.3
+numba >=0.55.0, <=0.61.2

mindsdb/integrations/handlers/web_handler/urlcrawl_helpers.py

@@ -170,9 +170,9 @@ def get_all_website_links(url, headers: dict = None) -> dict:
                 href = href.rstrip("/")
                 urls.add(href)
 
-    except Exception as e:
+    except Exception:
         error_message = traceback.format_exc().splitlines()[-1]
-        logger.error("An exception occurred: %s", str(e))
+        logger.exception("An exception occurred:")
         return {
             "url": url,
             "urls": urls,
@@ -238,9 +238,9 @@ def get_all_website_links_recursively(
     if url not in reviewed_urls and matches_filter:
         try:
             reviewed_urls[url] = get_all_website_links(url, headers=headers)
-        except Exception as e:
+        except Exception:
             error_message = traceback.format_exc().splitlines()[-1]
-            logger.error("An exception occurred: %s", str(e))
+            logger.exception("An exception occurred:")
             reviewed_urls[url] = {
                 "url": url,
                 "urls": [],