MindsDB 25.7.3.0__py3-none-any.whl → 25.7.4.0__py3-none-any.whl

mindsdb/api/executor/sql_query/steps/apply_predictor_step.py

@@ -26,38 +26,28 @@ from .base import BaseStepCall
 
 
 def get_preditor_alias(step, mindsdb_database):
-    predictor_name = '.'.join(step.predictor.parts)
-    predictor_alias = '.'.join(step.predictor.alias.parts) if step.predictor.alias is not None else predictor_name
+    predictor_name = ".".join(step.predictor.parts)
+    predictor_alias = ".".join(step.predictor.alias.parts) if step.predictor.alias is not None else predictor_name
     return (mindsdb_database, predictor_name, predictor_alias)
 
 
 class ApplyPredictorBaseCall(BaseStepCall):
-
     def apply_predictor(self, project_name, predictor_name, df, version, params):
         # is it an agent?
         agent = self.session.agents_controller.get_agent(predictor_name, project_name)
         if agent is not None:
-
-            messages = df.to_dict('records')
+            messages = df.to_dict("records")
             predictions = self.session.agents_controller.get_completion(
-                agent,
-                messages=messages,
-                project_name=project_name,
+                agent, messages=messages, project_name=project_name, params=params
             )
 
         else:
             project_datanode = self.session.datahub.get(project_name)
-            predictions = project_datanode.predict(
-                model_name=predictor_name,
-                df=df,
-                version=version,
-                params=params
-            )
+            predictions = project_datanode.predict(model_name=predictor_name, df=df, version=version, params=params)
         return predictions
 
 
 class ApplyPredictorRowStepCall(ApplyPredictorBaseCall):
-
     bind = ApplyPredictorRowStep
 
     def call(self, step):
@@ -89,7 +79,7 @@ class ApplyPredictorRowStepCall(ApplyPredictorBaseCall):
         for k, v in where_data.items():
             predictions[k] = v
 
-        table_name = get_preditor_alias(step, self.context.get('database'))
+        table_name = get_preditor_alias(step, self.context.get("database"))
 
         if len(predictions) == 0:
             columns_names = project_datanode.get_table_columns_names(predictor_name)
@@ -100,12 +90,11 @@ class ApplyPredictorRowStepCall(ApplyPredictorBaseCall):
             database=table_name[0],
             table_name=table_name[1],
             table_alias=table_name[2],
-            is_prediction=True
+            is_prediction=True,
         )
 
 
 class ApplyPredictorStepCall(ApplyPredictorBaseCall):
-
     bind = ApplyPredictorStep
 
     def call(self, step):
@@ -115,20 +104,20 @@ class ApplyPredictorStepCall(ApplyPredictorBaseCall):
         params = step.params or {}
 
         # adding __mindsdb_row_id, use first table if exists
-        if len(data.find_columns('__mindsdb_row_id')) == 0:
+        if len(data.find_columns("__mindsdb_row_id")) == 0:
             table = data.get_tables()[0] if len(data.get_tables()) > 0 else None
 
             row_id_col = Column(
-                name='__mindsdb_row_id',
-                database=table['database'] if table is not None else None,
-                table_name=table['table_name'] if table is not None else None,
-                table_alias=table['table_alias'] if table is not None else None
+                name="__mindsdb_row_id",
+                database=table["database"] if table is not None else None,
+                table_name=table["table_name"] if table is not None else None,
+                table_alias=table["table_alias"] if table is not None else None,
             )
 
-            row_id = self.context.get('row_id')
+            row_id = self.context.get("row_id")
             values = range(row_id, row_id + data.length())
             data.add_column(row_id_col, values)
-            self.context['row_id'] += data.length()
+            self.context["row_id"] += data.length()
 
         project_name = step.namespace
         predictor_name = step.predictor.parts[0]
@@ -143,47 +132,46 @@ class ApplyPredictorStepCall(ApplyPredictorBaseCall):
                 data.set_column_values(k, v)
 
         predictor_metadata = {}
-        for pm in self.context['predictor_metadata']:
-            if pm['name'] == predictor_name and pm['integration_name'].lower() == project_name:
+        for pm in self.context["predictor_metadata"]:
+            if pm["name"] == predictor_name and pm["integration_name"].lower() == project_name:
                 predictor_metadata = pm
                 break
-        is_timeseries = predictor_metadata['timeseries']
+        is_timeseries = predictor_metadata["timeseries"]
         _mdb_forecast_offset = None
         if is_timeseries:
-            if '> LATEST' in self.context['query_str']:
+            if "> LATEST" in self.context["query_str"]:
                 # stream mode -- if > LATEST, forecast starts on inferred next timestamp
                 _mdb_forecast_offset = 1
-            elif '= LATEST' in self.context['query_str']:
+            elif "= LATEST" in self.context["query_str"]:
                 # override: when = LATEST, forecast starts on last provided timestamp instead of inferred next time
                 _mdb_forecast_offset = 0
             else:
                 # normal mode -- emit a forecast ($HORIZON data points on each) for each provided timestamp
-                params['force_ts_infer'] = True
+                params["force_ts_infer"] = True
                 _mdb_forecast_offset = None
 
-            data.add_column(Column(name='__mdb_forecast_offset'), _mdb_forecast_offset)
+            data.add_column(Column(name="__mdb_forecast_offset"), _mdb_forecast_offset)
 
-        table_name = get_preditor_alias(step, self.context['database'])
+        table_name = get_preditor_alias(step, self.context["database"])
 
         project_datanode = self.session.datahub.get(project_name)
         if len(data) == 0:
-            columns_names = project_datanode.get_table_columns_names(predictor_name) + ['__mindsdb_row_id']
+            columns_names = project_datanode.get_table_columns_names(predictor_name) + ["__mindsdb_row_id"]
             result = ResultSet(is_prediction=True)
             for column_name in columns_names:
-                result.add_column(Column(
-                    name=column_name,
-                    database=table_name[0],
-                    table_name=table_name[1],
-                    table_alias=table_name[2]
-                ))
+                result.add_column(
+                    Column(
+                        name=column_name, database=table_name[0], table_name=table_name[1], table_alias=table_name[2]
+                    )
+                )
         else:
-            predictor_id = predictor_metadata['id']
+            predictor_id = predictor_metadata["id"]
             table_df = data.to_df()
 
             if self.session.predictor_cache is not False:
-                key = f'{predictor_name}_{predictor_id}_{dataframe_checksum(table_df)}'
+                key = f"{predictor_name}_{predictor_id}_{dataframe_checksum(table_df)}"
 
-                predictor_cache = get_cache('predict')
+                predictor_cache = get_cache("predict")
                 predictions = predictor_cache.get(key)
             else:
                 predictions = None
@@ -221,7 +209,7 @@ class ApplyPredictorStepCall(ApplyPredictorBaseCall):
 
             # apply filter
             if is_timeseries:
-                pred_data = predictions.to_dict(orient='records')
+                pred_data = predictions.to_dict(orient="records")
                 where_data = list(data.get_records())
                 pred_data = self.apply_ts_filter(pred_data, where_data, step, predictor_metadata)
                 predictions = pd.DataFrame(pred_data)
@@ -231,37 +219,33 @@ class ApplyPredictorStepCall(ApplyPredictorBaseCall):
                 database=table_name[0],
                 table_name=table_name[1],
                 table_alias=table_name[2],
-                is_prediction=True
+                is_prediction=True,
             )
 
         return result
 
     def apply_ts_filter(self, predictor_data, table_data, step, predictor_metadata):
-
         if step.output_time_filter is None:
             # no filter, exit
             return predictor_data
 
             # apply filter
-        group_cols = predictor_metadata['group_by_columns']
-        order_col = predictor_metadata['order_by_column']
+        group_cols = predictor_metadata["group_by_columns"]
+        order_col = predictor_metadata["order_by_column"]
 
         filter_args = step.output_time_filter.args
         filter_op = step.output_time_filter.op
 
         # filter field must be order column
-        if not (
-            isinstance(filter_args[0], Identifier)
-            and filter_args[0].parts[-1] == order_col
-        ):
+        if not (isinstance(filter_args[0], Identifier) and filter_args[0].parts[-1] == order_col):
             # exit otherwise
             return predictor_data
 
         def get_date_format(samples):
             # Try common formats first with explicit patterns
             for date_format, pattern in (
-                ('%Y-%m-%d', r'[\d]{4}-[\d]{2}-[\d]{2}'),
-                ('%Y-%m-%d %H:%M:%S', r'[\d]{4}-[\d]{2}-[\d]{2} [\d]{2}:[\d]{2}:[\d]{2}'),
+                ("%Y-%m-%d", r"[\d]{4}-[\d]{2}-[\d]{2}"),
+                ("%Y-%m-%d %H:%M:%S", r"[\d]{4}-[\d]{2}-[\d]{2} [\d]{2}:[\d]{2}:[\d]{2}"),
                 # ('%Y-%m-%d %H:%M:%S%z', r'[\d]{4}-[\d]{2}-[\d]{2} [\d]{2}:[\d]{2}:[\d]{2}\+[\d]{2}:[\d]{2}'),
                 # ('%Y', '[\d]{4}')
             ):
@@ -281,6 +265,7 @@ class ApplyPredictorStepCall(ApplyPredictorBaseCall):
                 # Parse the first sample to get its format
                 # The import is heavy, so we do it here on-demand
                 import dateparser
+
                 parsed_date = dateparser.parse(samples[0])
                 if parsed_date is None:
                     raise ValueError("Could not parse date")
@@ -290,25 +275,21 @@ class ApplyPredictorStepCall(ApplyPredictorBaseCall):
                     if dateparser.parse(sample) is None:
                         raise ValueError("Inconsistent date formats in samples")
                 # Convert to strftime format based on the input
-                if re.search(r'\d{2}:\d{2}:\d{2}', samples[0]):
-                    return '%Y-%m-%d %H:%M:%S'
-                return '%Y-%m-%d'
+                if re.search(r"\d{2}:\d{2}:\d{2}", samples[0]):
+                    return "%Y-%m-%d %H:%M:%S"
+                return "%Y-%m-%d"
             except (ValueError, AttributeError):
                 # If dateparser fails, return a basic format as last resort
-                return '%Y-%m-%d'
+                return "%Y-%m-%d"
 
-        model_types = predictor_metadata['model_types']
-        if model_types.get(order_col) in ('float', 'integer'):
+        model_types = predictor_metadata["model_types"]
+        if model_types.get(order_col) in ("float", "integer"):
             # convert strings to digits
-            fnc = {
-                'integer': int,
-                'float': float
-            }[model_types[order_col]]
+            fnc = {"integer": int, "float": float}[model_types[order_col]]
 
             # convert predictor_data
             if len(predictor_data) > 0:
                 if isinstance(predictor_data[0][order_col], str):
-
                     for row in predictor_data:
                         row[order_col] = fnc(row[order_col])
                 elif isinstance(predictor_data[0][order_col], dt.date):
@@ -318,7 +299,6 @@ class ApplyPredictorStepCall(ApplyPredictorBaseCall):
 
             # convert predictor_data
             if isinstance(table_data[0][order_col], str):
-
                 for row in table_data:
                     row[order_col] = fnc(row[order_col])
             elif isinstance(table_data[0][order_col], dt.date):
@@ -327,18 +307,13 @@ class ApplyPredictorStepCall(ApplyPredictorBaseCall):
                     row[order_col] = fnc(row[order_col])
 
             # convert args to date
-            samples = [
-                arg.value
-                for arg in filter_args
-                if isinstance(arg, Constant) and isinstance(arg.value, str)
-            ]
+            samples = [arg.value for arg in filter_args if isinstance(arg, Constant) and isinstance(arg.value, str)]
             if len(samples) > 0:
-
                 for arg in filter_args:
                     if isinstance(arg, Constant) and isinstance(arg.value, str):
                         arg.value = fnc(arg.value)
 
-        if model_types.get(order_col) in ('date', 'datetime') or isinstance(predictor_data[0][order_col], pd.Timestamp):  # noqa
+        if model_types.get(order_col) in ("date", "datetime") or isinstance(predictor_data[0][order_col], pd.Timestamp):  # noqa
             # convert strings to date
             # it is making side effect on original data by changing it but let it be
 
@@ -364,11 +339,7 @@ class ApplyPredictorStepCall(ApplyPredictorBaseCall):
             _cast_samples(table_data, order_col)
 
             # convert args to date
-            samples = [
-                arg.value
-                for arg in filter_args
-                if isinstance(arg, Constant) and isinstance(arg.value, str)
-            ]
+            samples = [arg.value for arg in filter_args if isinstance(arg, Constant) and isinstance(arg.value, str)]
             if len(samples) > 0:
                 date_format = get_date_format(samples)
 
@@ -380,7 +351,6 @@ class ApplyPredictorStepCall(ApplyPredictorBaseCall):
         # first pass: get max values for Latest in table data
         latest_vals = {}
         if Latest() in filter_args:
-
             for row in table_data:
                 if group_cols is None:
                     key = 0  # the same for any value
@@ -400,11 +370,11 @@ class ApplyPredictorStepCall(ApplyPredictorBaseCall):
                     data2.append(row)
             elif isinstance(step.output_time_filter, BinaryOperation):
                 op_map = {
-                    '<': '__lt__',
-                    '<=': '__le__',
-                    '>': '__gt__',
-                    '>=': '__ge__',
-                    '=': '__eq__',
+                    "<": "__lt__",
+                    "<=": "__le__",
+                    ">": "__gt__",
+                    ">=": "__ge__",
+                    "=": "__eq__",
                 }
                 arg = filter_args[1]
                 if isinstance(arg, Latest):
@@ -435,5 +405,4 @@ class ApplyPredictorStepCall(ApplyPredictorBaseCall):
 
 
 class ApplyTimeseriesPredictorStepCall(ApplyPredictorStepCall):
-
     bind = ApplyTimeseriesPredictorStep

mindsdb/api/http/initialize.py

@@ -43,6 +43,7 @@ from mindsdb.api.http.namespaces.webhooks import ns_conf as webhooks_ns
 from mindsdb.interfaces.database.integrations import integration_controller
 from mindsdb.interfaces.database.database import DatabaseController
 from mindsdb.interfaces.file.file_controller import FileController
+from mindsdb.interfaces.jobs.jobs_controller import JobsController
 from mindsdb.interfaces.storage import db
 from mindsdb.metrics.server import init_metrics
 from mindsdb.utilities import log
@@ -109,9 +110,7 @@ def get_last_compatible_gui_version() -> Version:
         return False
 
     if res.status_code != 200:
-        logger.error(
-            f"Cant get compatible-config.json: returned status code = {res.status_code}"
-        )
+        logger.error(f"Cant get compatible-config.json: returned status code = {res.status_code}")
         return False
 
     try:
@@ -132,10 +131,7 @@ def get_last_compatible_gui_version() -> Version:
             else:
                 mindsdb_lv = parse_version(el["mindsdb_version"])
                 gui_lv = parse_version(el["gui_version"])
-                if (
-                    mindsdb_lv.base_version not in gui_versions
-                    or gui_lv > gui_versions[mindsdb_lv.base_version]
-                ):
+                if mindsdb_lv.base_version not in gui_versions or gui_lv > gui_versions[mindsdb_lv.base_version]:
                     gui_versions[mindsdb_lv.base_version] = gui_lv
                 if max_mindsdb_lv is None or max_mindsdb_lv < mindsdb_lv:
                     max_mindsdb_lv = mindsdb_lv
@@ -151,9 +147,7 @@ def get_last_compatible_gui_version() -> Version:
             gui_version_lv = max_gui_lv
         else:
             lower_versions = {
-                key: value
-                for key, value in gui_versions.items()
-                if parse_version(key) < current_mindsdb_lv
+                key: value for key, value in gui_versions.items() if parse_version(key) < current_mindsdb_lv
             }
             if len(lower_versions) == 0:
                 gui_version_lv = gui_versions[all_mindsdb_lv[0].base_version]
@@ -169,7 +163,7 @@ def get_last_compatible_gui_version() -> Version:
 
 
 def get_current_gui_version() -> Version:
-    logger.debug("Getting current frontend version..")
+    logger.debug("Getting current frontend version...")
     config = Config()
     static_path = Path(config["paths"]["static"])
     version_txt_path = static_path.joinpath("version.txt")
@@ -179,9 +173,7 @@ def get_current_gui_version() -> Version:
         with open(version_txt_path, "rt") as f:
             current_gui_version = f.readline()
 
-    current_gui_lv = (
-        None if current_gui_version is None else parse_version(current_gui_version)
-    )
+    current_gui_lv = None if current_gui_version is None else parse_version(current_gui_version)
     logger.debug(f"Current frontend version: {current_gui_lv}.")
 
     return current_gui_lv
@@ -197,10 +189,7 @@ def initialize_static():
     if required_gui_version is not None:
         required_gui_version_lv = parse_version(required_gui_version)
         success = True
-        if (
-            current_gui_version_lv is None
-            or required_gui_version_lv != current_gui_version_lv
-        ):
+        if current_gui_version_lv is None or required_gui_version_lv != current_gui_version_lv:
             logger.debug("Updating gui..")
             success = update_static(required_gui_version_lv)
     else:
@@ -208,10 +197,7 @@ def initialize_static():
             return False
 
         # ignore versions like '23.9.2.2'
-        if (
-            current_gui_version_lv is not None
-            and len(current_gui_version_lv.release) < 3
-        ):
+        if current_gui_version_lv is not None and len(current_gui_version_lv.release) < 3:
             if current_gui_version_lv == last_gui_version_lv:
                 return True
         logger.debug("Updating gui..")
@@ -227,12 +213,8 @@ def initialize_app(config, no_studio):
     gui_exists = Path(static_root).joinpath("index.html").is_file()
     logger.debug(f"Does GUI already exist.. {'YES' if gui_exists else 'NO'}")
     init_static_thread = None
-    if no_studio is False and (
-        config["gui"]["autoupdate"] is True or gui_exists is False
-    ):
-        init_static_thread = threading.Thread(
-            target=initialize_static, name="initialize_static"
-        )
+    if no_studio is False and (config["gui"]["autoupdate"] is True or gui_exists is False):
+        init_static_thread = threading.Thread(target=initialize_static, name="initialize_static")
         init_static_thread.start()
 
     # Wait for static initialization.
@@ -334,9 +316,7 @@ def initialize_app(config, no_studio):
         # region routes where auth is required
         if (
             config["auth"]["http_auth_enabled"] is True
-            and any(
-                request.path.startswith(f"/api{ns.path}") for ns in protected_namespaces
-            )
+            and any(request.path.startswith(f"/api{ns.path}") for ns in protected_namespaces)
             and check_auth() is False
         ):
             return http_error(
@@ -368,18 +348,14 @@ def initialize_app(config, no_studio):
             try:
                 company_id = int(company_id)
             except Exception as e:
-                logger.error(
-                    f"Cloud not parse company id: {company_id} | exception: {e}"
-                )
+                logger.error(f"Cloud not parse company id: {company_id} | exception: {e}")
                 company_id = None
 
         if user_class is not None:
             try:
                 user_class = int(user_class)
             except Exception as e:
-                logger.error(
-                    f"Cloud not parse user_class: {user_class} | exception: {e}"
-                )
+                logger.error(f"Cloud not parse user_class: {user_class} | exception: {e}")
                 user_class = 0
         else:
             user_class = 0
@@ -419,9 +395,7 @@ def initialize_flask(config, init_static_thread, no_studio):
 
     app.config["SECRET_KEY"] = os.environ.get("FLASK_SECRET_KEY", secrets.token_hex(32))
     app.config["SESSION_COOKIE_NAME"] = "session"
-    app.config["PERMANENT_SESSION_LIFETIME"] = config["auth"][
-        "http_permanent_session_lifetime"
-    ]
+    app.config["PERMANENT_SESSION_LIFETIME"] = config["auth"]["http_permanent_session_lifetime"]
     app.config["SEND_FILE_MAX_AGE_DEFAULT"] = 60
     app.config["SWAGGER_HOST"] = "http://localhost:8000/mindsdb"
     app.json = CustomJSONProvider()
@@ -467,6 +441,7 @@ def initialize_interfaces(app):
     app.integration_controller = integration_controller
     app.database_controller = DatabaseController()
     app.file_controller = FileController()
+    app.jobs_controller = JobsController()
     config = Config()
     app.config_obj = config
 
@@ -479,9 +454,7 @@ def _open_webbrowser(url: str, pid: int, port: int, init_static_thread, static_f
     if init_static_thread is not None:
         init_static_thread.join()
     try:
-        is_http_active = wait_func_is_true(
-            func=is_pid_listen_port, timeout=15, pid=pid, port=port
-        )
+        is_http_active = wait_func_is_true(func=is_pid_listen_port, timeout=15, pid=pid, port=port)
         if is_http_active:
             webbrowser.open(url)
     except Exception as e:

mindsdb/api/http/namespaces/agents.py

@@ -27,14 +27,21 @@ def create_agent(project_name, name, agent):
     if name is None:
         return http_error(HTTPStatus.BAD_REQUEST, "Missing field", 'Missing "name" field for agent')
 
-    if "model_name" not in agent:
-        return http_error(HTTPStatus.BAD_REQUEST, "Missing field", 'Missing "model_name" field for agent')
+    if not name.islower():
+        return http_error(HTTPStatus.BAD_REQUEST, "Wrong name", f"The name must be in lower case: {name}")
 
-    model_name = agent["model_name"]
+    model_name = agent.get("model_name")
     provider = agent.get("provider")
-    params = agent.get("params", {})
     skills = agent.get("skills", [])
 
+    params = agent.get("params", {})
+    if agent.get("data"):
+        params["data"] = agent["data"]
+    if agent.get("model"):
+        params["model"] = agent["model"]
+    if agent.get("prompt_template"):
+        params["prompt_template"] = agent["prompt_template"]
+
     agents_controller = AgentsController()
 
     try:
@@ -153,12 +160,6 @@ class AgentResource(Resource):
 
         agent = request.json["agent"]
         name = agent.get("name", None)
-        model_name = agent.get("model_name", None)
-        skills_to_add = agent.get("skills_to_add", [])
-        skills_to_remove = agent.get("skills_to_remove", [])
-        skills_to_rewrite = agent.get("skills", [])
-        provider = agent.get("provider")
-        params = agent.get("params", None)
 
         # Agent must not exist with new name.
         if name is not None and name != agent_name:
@@ -176,9 +177,18 @@ class AgentResource(Resource):
 
         # Update
         try:
-            # Prepare the params dictionary
-            if params is None:
-                params = {}
+            model_name = agent.get("model_name", None)
+            skills_to_add = agent.get("skills_to_add", [])
+            skills_to_remove = agent.get("skills_to_remove", [])
+            skills_to_rewrite = agent.get("skills", [])
+            provider = agent.get("provider")
+            params = agent.get("params", {})
+            if agent.get("data"):
+                params["data"] = agent["data"]
+            if agent.get("model"):
+                params["model"] = agent["model"]
+            if agent.get("prompt_template"):
+                params["prompt_template"] = agent["prompt_template"]
 
             # Check if any of the skills to be added is of type 'retrieval'
             session = SessionController()
@@ -377,13 +387,6 @@ class AgentCompletions(Resource):
                 HTTPStatus.NOT_FOUND, "Project not found", f"Project with name {project_name} does not exist"
             )
 
-        # Add OpenAI API key to agent params if not already present.
-        if not existing_agent.params:
-            existing_agent.params = {}
-        existing_agent.params["openai_api_key"] = existing_agent.params.get(
-            "openai_api_key", os.getenv("OPENAI_API_KEY")
-        )
-
         # set mode to `retrieval` if agent has a skill of type `retrieval` and mode is not set
         if "mode" not in existing_agent.params and any(
             rel.skill.type == "retrieval" for rel in existing_agent.skills_relationships