MindsDB 25.7.3.0__py3-none-any.whl → 25.8.2.0__py3-none-any.whl

mindsdb/api/executor/utilities/mysql_to_duckdb_functions.py

@@ -0,0 +1,264 @@
+from mindsdb_sql_parser.ast import Identifier, Function, Constant, BinaryOperation
+
+
+def adapt_char_fn(node: Function) -> Function | None:
+    """Replace MySQL's multy-arg CHAR call to chain of DuckDB's CHR calls
+
+    Example:
+        CHAR(77, 78, 79) => CHR(77) || CHR(78) || CHR(79)
+
+    Args:
+        node (Function): Function node to adapt
+
+    Returns:
+        Function | None: Adapted function node
+    """
+    if len(node.args) == 1:
+        node.op = "chr"
+        return node
+
+    acc = None
+    for arg in node.args:
+        fn = Function(op="chr", args=[arg])
+        if acc is None:
+            acc = fn
+            continue
+        acc = BinaryOperation("||", args=[acc, fn])
+
+    acc.parentheses = True
+    acc.alias = node.alias
+    return acc
+
+
+def adapt_locate_fn(node: Function) -> Function | None:
+    """Replace MySQL's LOCATE (or INSTR) call to DuckDB's STRPOS call
+
+    Example:
+        LOCATE('bar', 'foobarbar') => STRPOS('foobarbar', 'bar')
+        INSTR('foobarbar', 'bar') => STRPOS('foobarbar', 'bar')
+        LOCATE('bar', 'foobarbar', 3) => ValueError (there is no analogue in DuckDB)
+
+    Args:
+        node (Function): Function node to adapt
+
+    Returns:
+        Function | None: Adapted function node
+
+    Raises:
+        ValueError: If the function has 3 arguments
+    """
+    if len(node.args) == 3:
+        raise ValueError("MySQL LOCATE function with 3 arguments is not supported")
+    if node.op == "locate":
+        node.args = [node.args[1], node.args[0]]
+    elif node.op == "insrt":
+        node.args = [node.args[0], node.args[1]]
+    node.op = "strpos"
+
+
+def adapt_unhex_fn(node: Function) -> None:
+    """Check MySQL's UNHEX function call arguments to ensure they are strings,
+    because DuckDB's UNHEX accepts only string arguments, while MySQL's UNHEX can accept integer arguments.
+    NOTE: if return dataframe from duckdb then unhex values are array - this may be an issue
+
+    Args:
+        node (Function): Function node to adapt
+
+    Returns:
+        None
+
+    Raises:
+        ValueError: If the function argument is not a string
+    """
+    for arg in node.args:
+        if not isinstance(arg, (str, bytes)):
+            raise ValueError("MySQL UNHEX function argument must be a string")
+
+
+def adapt_format_fn(node: Function) -> None:
+    """Adapt MySQL's FORMAT function to DuckDB's FORMAT function
+
+    Example:
+        FORMAT(1234567.89, 0) => FORMAT('{:,.0f}', 1234567.89)
+        FORMAT(1234567.89, 2) => FORMAT('{:,.2f}', 1234567.89)
+        FORMAT(name, 2) => FORMAT('{:,.2f}', name)
+        FORMAT('{:.2f}', 1234567.89) => FORMAT('{:,.2f}', 1234567.89)  # no changes for original style
+
+    Args:
+        node (Function): Function node to adapt
+
+    Returns:
+        None
+
+    Raises:
+        ValueError: If MySQL's function has 3rd 'locale' argument, like FORMAT(12332.2, 2, 'de_DE')
+    """
+    match node.args[0], node.args[1]:
+        case Constant(value=(int() | float())), Constant(value=int()):
+            ...
+        case Identifier(), Constant(value=int()):
+            ...
+        case _:
+            return node
+
+    if len(node.args) > 2:
+        raise ValueError("'locale' argument of 'format' function is not supported")
+    decimal_places = node.args[1].value
+
+    if isinstance(node.args[0], Constant):
+        node.args[1].value = node.args[0].value
+        node.args[0].value = f"{{:,.{decimal_places}f}}"
+    else:
+        node.args[1] = node.args[0]
+        node.args[0] = Constant(f"{{:,.{decimal_places}f}}")
+
+
+def adapt_sha2_fn(node: Function) -> None:
+    """Adapt MySQL's SHA2 function to DuckDB's SHA256 function
+
+    Example:
+        SHA2('test', 256) => SHA256('test')
+
+    Args:
+        node (Function): Function node to adapt
+
+    Returns:
+        None
+
+    Raises:
+        ValueError: If the function has more than 1 argument or the argument is not 256
+    """
+    if len(node.args) > 1 and node.args[1].value != 256:
+        raise ValueError("Only sha256 is supported")
+    node.op = "sha256"
+    node.args = [node.args[0]]
+
+
+def adapt_length_fn(node: Function) -> None:
+    """Adapt MySQL's LENGTH function to DuckDB's STRLEN function
+    NOTE: duckdb also have LENGTH, therefore it can not be used
+
+    Example:
+        LENGTH('test') => STRLEN('test')
+
+    Args:
+        node (Function): Function node to adapt
+
+    Returns:
+        None
+    """
+    node.op = "strlen"
+
+
+def adapt_regexp_substr_fn(node: Function) -> None:
+    """Adapt MySQL's REGEXP_SUBSTR function to DuckDB's REGEXP_EXTRACT function
+
+    Example:
+        REGEXP_SUBSTR('foobarbar', 'bar', 1, 1) => REGEXP_EXTRACT('foobarbar', 'bar')
+
+    Args:
+        node (Function): Function node to adapt
+
+    Returns:
+        None
+
+    Raises:
+        ValueError: If the function has more than 2 arguments or 3rd or 4th argument is not 1
+    """
+    if (
+        len(node.args) == 3
+        and node.args[2].value != 1
+        or len(node.args) == 4
+        and (node.args[3].value != 1 or node.args[2].value != 1)
+        or len(node.args) > 4
+    ):
+        raise ValueError("Only 2 arguments are supported for REGEXP_SUBSTR function")
+    node.args = node.args[:2]
+    node.op = "regexp_extract"
+
+
+def adapt_substring_index_fn(node: Function) -> BinaryOperation | Function:
+    """Adapt MySQL's SUBSTRING_INDEX function to DuckDB's SPLIT_PART function
+
+    Example:
+        SUBSTRING_INDEX('a.b.c.d', '.', 1) => SPLIT_PART('a.b.c.d', '.', 1)
+        SUBSTRING_INDEX('a.b.c.d', '.', 2) => CONCAT_WS('.', SPLIT_PART('a.b.c.d', '.', 1), SPLIT_PART('a.b.c.d', '.', 2))
+
+    Args:
+        node (Function): Function node to adapt
+
+    Returns:
+        BinaryOperation | Function: Binary operation node or function node
+
+    Raises:
+        ValueError: If the function has more than 3 arguments or the 3rd argument is not 1
+    """
+    if len(node.args[1].value) > 1:
+        raise ValueError("Only one car in separator")
+
+    if node.args[2].value == 1:
+        node.op = "split_part"
+        return node
+
+    acc = [node.args[1]]
+    for i in range(node.args[2].value):
+        fn = Function(op="split_part", args=[node.args[0], node.args[1], Constant(i + 1)])
+        acc.append(fn)
+
+    acc = Function(op="concat_ws", args=acc)
+    acc.alias = node.alias
+    return acc
+
+
+def adapt_curtime_fn(node: Function) -> BinaryOperation:
+    """Adapt MySQL's CURTIME function to DuckDB's GET_CURRENT_TIME function.
+    To get the same type as MySQL's CURTIME function, we need to cast the result to time type.
+
+    Example:
+        CURTIME() => GET_CURRENT_TIME()::time
+
+    Args:
+        node (Function): Function node to adapt
+
+    Returns:
+        BinaryOperation: Binary operation node
+    """
+    return BinaryOperation("::", args=[Function(op="get_current_time", args=[]), Identifier("time")], alias=node.alias)
+
+
+def adapt_timestampdiff_fn(node: Function) -> None:
+    """Adapt MySQL's TIMESTAMPDIFF function to DuckDB's DATE_DIFF function
+    NOTE: Looks like cast string args to timestamp works in most cases, but there may be some exceptions.
+
+    Example:
+        TIMESTAMPDIFF(YEAR, '2000-02-01', '2003-05-01') => DATE_DIFF('year', timestamp '2000-02-01', timestamp '2003-05-01')
+
+    Args:
+        node (Function): Function node to adapt
+
+    Returns:
+        None
+    """
+    node.op = "date_diff"
+    node.args[0] = Constant(node.args[0].parts[0])
+    node.args[1] = BinaryOperation(" ", args=[Identifier("timestamp"), node.args[1]])
+    node.args[2] = BinaryOperation(" ", args=[Identifier("timestamp"), node.args[2]])
+
+
+def adapt_extract_fn(node: Function) -> None:
+    """Adapt MySQL's EXTRACT function to DuckDB's EXTRACT function
+    TODO: multi-part args, like YEAR_MONTH, is not supported yet
+    NOTE: Looks like adding 'timestamp' works in most cases, but there may be some exceptions.
+
+    Example:
+        EXTRACT(YEAR FROM '2000-02-01') => EXTRACT('year' from timestamp '2000-02-01')
+
+    Args:
+        node (Function): Function node to adapt
+
+    Returns:
+        None
+    """
+    node.args[0] = Constant(node.args[0].parts[0])
+    if not isinstance(node.from_arg, Identifier):
+        node.from_arg = BinaryOperation(" ", args=[Identifier("timestamp"), node.from_arg])

mindsdb/api/executor/utilities/sql.py

@@ -14,6 +14,19 @@ from mindsdb.utilities.exception import format_db_error_message
 from mindsdb.utilities.functions import resolve_table_identifier, resolve_model_identifier
 from mindsdb.utilities.json_encoder import CustomJSONEncoder
 from mindsdb.utilities.render.sqlalchemy_render import SqlalchemyRender
+from mindsdb.api.executor.utilities.mysql_to_duckdb_functions import (
+    adapt_char_fn,
+    adapt_locate_fn,
+    adapt_unhex_fn,
+    adapt_format_fn,
+    adapt_sha2_fn,
+    adapt_length_fn,
+    adapt_regexp_substr_fn,
+    adapt_substring_index_fn,
+    adapt_curtime_fn,
+    adapt_timestampdiff_fn,
+    adapt_extract_fn,
+)
 
 logger = log.getLogger(__name__)
 
@@ -185,6 +198,23 @@ def query_df(df, query, session=None):
         if isinstance(node, Function):
             fnc_name = node.op.lower()
 
+            mysql_to_duck_fn_map = {
+                "char": adapt_char_fn,
+                "locate": adapt_locate_fn,
+                "insrt": adapt_locate_fn,
+                "unhex": adapt_unhex_fn,
+                "format": adapt_format_fn,
+                "sha2": adapt_sha2_fn,
+                "length": adapt_length_fn,
+                "regexp_substr": adapt_regexp_substr_fn,
+                "substring_index": adapt_substring_index_fn,
+                "curtime": adapt_curtime_fn,
+                "timestampdiff": adapt_timestampdiff_fn,
+                "extract": adapt_extract_fn,
+            }
+            if fnc_name in mysql_to_duck_fn_map:
+                return mysql_to_duck_fn_map[fnc_name](node)
+
             if fnc_name == "database" and len(node.args) == 0:
                 if session is not None:
                     cur_db = session.database

mindsdb/api/http/initialize.py

@@ -43,6 +43,7 @@ from mindsdb.api.http.namespaces.webhooks import ns_conf as webhooks_ns
 from mindsdb.interfaces.database.integrations import integration_controller
 from mindsdb.interfaces.database.database import DatabaseController
 from mindsdb.interfaces.file.file_controller import FileController
+from mindsdb.interfaces.jobs.jobs_controller import JobsController
 from mindsdb.interfaces.storage import db
 from mindsdb.metrics.server import init_metrics
 from mindsdb.utilities import log
@@ -109,9 +110,7 @@ def get_last_compatible_gui_version() -> Version:
         return False
 
     if res.status_code != 200:
-        logger.error(
-            f"Cant get compatible-config.json: returned status code = {res.status_code}"
-        )
+        logger.error(f"Cant get compatible-config.json: returned status code = {res.status_code}")
         return False
 
     try:
@@ -132,10 +131,7 @@ def get_last_compatible_gui_version() -> Version:
             else:
                 mindsdb_lv = parse_version(el["mindsdb_version"])
                 gui_lv = parse_version(el["gui_version"])
-                if (
-                    mindsdb_lv.base_version not in gui_versions
-                    or gui_lv > gui_versions[mindsdb_lv.base_version]
-                ):
+                if mindsdb_lv.base_version not in gui_versions or gui_lv > gui_versions[mindsdb_lv.base_version]:
                     gui_versions[mindsdb_lv.base_version] = gui_lv
                 if max_mindsdb_lv is None or max_mindsdb_lv < mindsdb_lv:
                     max_mindsdb_lv = mindsdb_lv
@@ -151,9 +147,7 @@ def get_last_compatible_gui_version() -> Version:
             gui_version_lv = max_gui_lv
         else:
             lower_versions = {
-                key: value
-                for key, value in gui_versions.items()
-                if parse_version(key) < current_mindsdb_lv
+                key: value for key, value in gui_versions.items() if parse_version(key) < current_mindsdb_lv
             }
             if len(lower_versions) == 0:
                 gui_version_lv = gui_versions[all_mindsdb_lv[0].base_version]
@@ -169,7 +163,7 @@ def get_last_compatible_gui_version() -> Version:
 
 
 def get_current_gui_version() -> Version:
-    logger.debug("Getting current frontend version..")
+    logger.debug("Getting current frontend version...")
     config = Config()
     static_path = Path(config["paths"]["static"])
     version_txt_path = static_path.joinpath("version.txt")
@@ -179,9 +173,7 @@ def get_current_gui_version() -> Version:
         with open(version_txt_path, "rt") as f:
             current_gui_version = f.readline()
 
-    current_gui_lv = (
-        None if current_gui_version is None else parse_version(current_gui_version)
-    )
+    current_gui_lv = None if current_gui_version is None else parse_version(current_gui_version)
     logger.debug(f"Current frontend version: {current_gui_lv}.")
 
     return current_gui_lv
@@ -197,10 +189,7 @@ def initialize_static():
     if required_gui_version is not None:
         required_gui_version_lv = parse_version(required_gui_version)
         success = True
-        if (
-            current_gui_version_lv is None
-            or required_gui_version_lv != current_gui_version_lv
-        ):
+        if current_gui_version_lv is None or required_gui_version_lv != current_gui_version_lv:
             logger.debug("Updating gui..")
             success = update_static(required_gui_version_lv)
     else:
@@ -208,16 +197,14 @@ def initialize_static():
             return False
 
         # ignore versions like '23.9.2.2'
-        if (
-            current_gui_version_lv is not None
-            and len(current_gui_version_lv.release) < 3
-        ):
+        if current_gui_version_lv is not None and len(current_gui_version_lv.release) < 3:
             if current_gui_version_lv == last_gui_version_lv:
                 return True
         logger.debug("Updating gui..")
         success = update_static(last_gui_version_lv)
 
-    db.session.close()
+    if db.session:
+        db.session.close()
     return success
 
 
@@ -227,12 +214,8 @@ def initialize_app(config, no_studio):
     gui_exists = Path(static_root).joinpath("index.html").is_file()
     logger.debug(f"Does GUI already exist.. {'YES' if gui_exists else 'NO'}")
     init_static_thread = None
-    if no_studio is False and (
-        config["gui"]["autoupdate"] is True or gui_exists is False
-    ):
-        init_static_thread = threading.Thread(
-            target=initialize_static, name="initialize_static"
-        )
+    if no_studio is False and (config["gui"]["autoupdate"] is True or gui_exists is False):
+        init_static_thread = threading.Thread(target=initialize_static, name="initialize_static")
         init_static_thread.start()
 
     # Wait for static initialization.
@@ -334,9 +317,7 @@ def initialize_app(config, no_studio):
         # region routes where auth is required
         if (
             config["auth"]["http_auth_enabled"] is True
-            and any(
-                request.path.startswith(f"/api{ns.path}") for ns in protected_namespaces
-            )
+            and any(request.path.startswith(f"/api{ns.path}") for ns in protected_namespaces)
             and check_auth() is False
         ):
             return http_error(
@@ -368,18 +349,14 @@ def initialize_app(config, no_studio):
             try:
                 company_id = int(company_id)
             except Exception as e:
-                logger.error(
-                    f"Cloud not parse company id: {company_id} | exception: {e}"
-                )
+                logger.error(f"Cloud not parse company id: {company_id} | exception: {e}")
                 company_id = None
 
         if user_class is not None:
             try:
                 user_class = int(user_class)
             except Exception as e:
-                logger.error(
-                    f"Cloud not parse user_class: {user_class} | exception: {e}"
-                )
+                logger.error(f"Cloud not parse user_class: {user_class} | exception: {e}")
                 user_class = 0
         else:
             user_class = 0
@@ -419,9 +396,7 @@ def initialize_flask(config, init_static_thread, no_studio):
 
     app.config["SECRET_KEY"] = os.environ.get("FLASK_SECRET_KEY", secrets.token_hex(32))
     app.config["SESSION_COOKIE_NAME"] = "session"
-    app.config["PERMANENT_SESSION_LIFETIME"] = config["auth"][
-        "http_permanent_session_lifetime"
-    ]
+    app.config["PERMANENT_SESSION_LIFETIME"] = config["auth"]["http_permanent_session_lifetime"]
     app.config["SEND_FILE_MAX_AGE_DEFAULT"] = 60
     app.config["SWAGGER_HOST"] = "http://localhost:8000/mindsdb"
     app.json = CustomJSONProvider()
@@ -467,6 +442,7 @@ def initialize_interfaces(app):
     app.integration_controller = integration_controller
     app.database_controller = DatabaseController()
     app.file_controller = FileController()
+    app.jobs_controller = JobsController()
     config = Config()
     app.config_obj = config
 
@@ -479,9 +455,7 @@ def _open_webbrowser(url: str, pid: int, port: int, init_static_thread, static_f
     if init_static_thread is not None:
         init_static_thread.join()
     try:
-        is_http_active = wait_func_is_true(
-            func=is_pid_listen_port, timeout=15, pid=pid, port=port
-        )
+        is_http_active = wait_func_is_true(func=is_pid_listen_port, timeout=15, pid=pid, port=port)
         if is_http_active:
             webbrowser.open(url)
     except Exception as e:

mindsdb/api/http/namespaces/agents.py

@@ -27,14 +27,21 @@ def create_agent(project_name, name, agent):
     if name is None:
         return http_error(HTTPStatus.BAD_REQUEST, "Missing field", 'Missing "name" field for agent')
 
-    if "model_name" not in agent:
-        return http_error(HTTPStatus.BAD_REQUEST, "Missing field", 'Missing "model_name" field for agent')
+    if not name.islower():
+        return http_error(HTTPStatus.BAD_REQUEST, "Wrong name", f"The name must be in lower case: {name}")
 
-    model_name = agent["model_name"]
+    model_name = agent.get("model_name")
     provider = agent.get("provider")
-    params = agent.get("params", {})
     skills = agent.get("skills", [])
 
+    params = agent.get("params", {})
+    if agent.get("data"):
+        params["data"] = agent["data"]
+    if agent.get("model"):
+        params["model"] = agent["model"]
+    if agent.get("prompt_template"):
+        params["prompt_template"] = agent["prompt_template"]
+
     agents_controller = AgentsController()
 
     try:
@@ -153,12 +160,6 @@ class AgentResource(Resource):
 
         agent = request.json["agent"]
         name = agent.get("name", None)
-        model_name = agent.get("model_name", None)
-        skills_to_add = agent.get("skills_to_add", [])
-        skills_to_remove = agent.get("skills_to_remove", [])
-        skills_to_rewrite = agent.get("skills", [])
-        provider = agent.get("provider")
-        params = agent.get("params", None)
 
         # Agent must not exist with new name.
         if name is not None and name != agent_name:
@@ -176,9 +177,18 @@ class AgentResource(Resource):
 
         # Update
         try:
-            # Prepare the params dictionary
-            if params is None:
-                params = {}
+            model_name = agent.get("model_name", None)
+            skills_to_add = agent.get("skills_to_add", [])
+            skills_to_remove = agent.get("skills_to_remove", [])
+            skills_to_rewrite = agent.get("skills", [])
+            provider = agent.get("provider")
+            params = agent.get("params", {})
+            if agent.get("data"):
+                params["data"] = agent["data"]
+            if agent.get("model"):
+                params["model"] = agent["model"]
+            if agent.get("prompt_template"):
+                params["prompt_template"] = agent["prompt_template"]
 
             # Check if any of the skills to be added is of type 'retrieval'
             session = SessionController()
@@ -377,13 +387,6 @@ class AgentCompletions(Resource):
                 HTTPStatus.NOT_FOUND, "Project not found", f"Project with name {project_name} does not exist"
             )
 
-        # Add OpenAI API key to agent params if not already present.
-        if not existing_agent.params:
-            existing_agent.params = {}
-        existing_agent.params["openai_api_key"] = existing_agent.params.get(
-            "openai_api_key", os.getenv("OPENAI_API_KEY")
-        )
-
         # set mode to `retrieval` if agent has a skill of type `retrieval` and mode is not set
         if "mode" not in existing_agent.params and any(
             rel.skill.type == "retrieval" for rel in existing_agent.skills_relationships