mlrun 1.10.0rc20__py3-none-any.whl → 1.10.0rc22__py3-none-any.whl

mlrun/artifacts/llm_prompt.py

@@ -83,19 +83,20 @@ class LLMPromptArtifactSpec(ArtifactSpec):
             raise mlrun.errors.MLRunInvalidArgumentError(
                 "Expected prompt_template to be a list of dicts"
             )
-        keys_to_pop = []
         for message in prompt_template:
+            if set(key.lower() for key in message.keys()) != set(
+                self.PROMPT_TEMPLATE_KEYS
+            ):
+                raise mlrun.errors.MLRunInvalidArgumentError(
+                    f"Expected prompt_template to contain dicts with keys "
+                    f"{self.PROMPT_TEMPLATE_KEYS}, got {message.keys()}"
+                )
+            keys_to_pop = []
             for key in message.keys():
                 if isinstance(key, str):
-                    if key.lower() not in self.PROMPT_TEMPLATE_KEYS:
-                        raise mlrun.errors.MLRunInvalidArgumentError(
-                            f"Expected prompt_template to contain dict that "
-                            f"only has keys from {self.PROMPT_TEMPLATE_KEYS}"
-                        )
-                    else:
-                        if not key.islower():
-                            message[key.lower()] = message[key]
-                            keys_to_pop.append(key)
+                    if not key.islower():
+                        message[key.lower()] = message[key]
+                        keys_to_pop.append(key)
                 else:
                     raise mlrun.errors.MLRunInvalidArgumentError(
                         f"Expected prompt_template to contain dict that only"

mlrun/artifacts/model.py

@@ -190,10 +190,10 @@ class ModelArtifact(Artifact):
         """
         super().__init__(key, body, format=format, target_path=target_path, **kwargs)
         model_file = str(model_file or "")
-        if model_file and model_url:
+        if (model_file or model_dir or body) and model_url:
             raise mlrun.errors.MLRunInvalidArgumentError(
-                "Arguments 'model_file' and 'model_dir' cannot be"
-                " used together with 'model_url'."
+                "Arguments 'model_file' and 'model_url' cannot be"
+                " used together with 'model_file', 'model_dir' or 'body'."
             )
         if model_file and "/" in model_file:
             if model_dir:

mlrun/common/schemas/auth.py

@@ -55,6 +55,7 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
     secret = "secret"
     run = "run"
     model_endpoint = "model-endpoint"
+    model_monitoring = "model-monitoring"
     pipeline = "pipeline"
     hub_source = "hub-source"
     workflow = "workflow"
@@ -96,6 +97,7 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
             # runtime resource doesn't have an identifier, we don't need any auth granularity behind project level
             AuthorizationResourceTypes.runtime_resource: "/projects/{project_name}/runtime-resources",
             AuthorizationResourceTypes.model_endpoint: "/projects/{project_name}/model-endpoints/{resource_name}",
+            AuthorizationResourceTypes.model_monitoring: "/projects/{project_name}/model-monitoring/{resource_name}",
             AuthorizationResourceTypes.pipeline: "/projects/{project_name}/pipelines/{resource_name}",
             AuthorizationResourceTypes.datastore_profile: "/projects/{project_name}/datastore_profiles",
             # Hub sources are not project-scoped, and auth is globally on the sources endpoint.

mlrun/db/base.py

@@ -1111,6 +1111,15 @@ class RunDBInterface(ABC):
     ) -> None:
         pass
 
+    @abstractmethod
+    def delete_model_monitoring_metrics(
+        self,
+        project: str,
+        application_name: str,
+        endpoint_ids: Optional[list[str]] = None,
+    ) -> None:
+        pass
+
     @abstractmethod
     def get_monitoring_function_summaries(
         self,

mlrun/db/httpdb.py

@@ -3580,7 +3580,7 @@ class HTTPRunDB(RunDBInterface):
                                 intersection {"intersect_metrics":[], "intersect_results":[]}
         :return: A dictionary of application metrics and/or results for the model endpoints formatted by events_format.
         """
-        path = f"projects/{project}/model-endpoints/metrics"
+        path = f"projects/{project}/model-monitoring/metrics"
         params = {
             "type": type,
             "endpoint-id": endpoint_ids,
@@ -4121,6 +4121,26 @@ class HTTPRunDB(RunDBInterface):
             params={**credentials, "replace_creds": replace_creds},
         )
 
+    def delete_model_monitoring_metrics(
+        self,
+        project: str,
+        application_name: str,
+        endpoint_ids: Optional[list[str]] = None,
+    ) -> None:
+        """
+        Delete model endpoints metrics values.
+
+        :param project:           The name of the project.
+        :param application_name:  The name of the application.
+        :param endpoint_ids:      The unique IDs of the model endpoints to delete metrics values from. If none is
+                                  provided, the metrics values will be deleted from all project's model endpoints.
+        """
+        self.api_call(
+            method=mlrun.common.types.HTTPMethod.DELETE,
+            path=f"projects/{project}/model-monitoring/metrics",
+            params={"endpoint-id": endpoint_ids, "application-name": application_name},
+        )
+
     def get_monitoring_function_summaries(
         self,
         project: str,

mlrun/db/nopdb.py

@@ -885,6 +885,14 @@ class NopDB(RunDBInterface):
     ) -> None:
         pass
 
+    def delete_model_monitoring_metrics(
+        self,
+        project: str,
+        application_name: str,
+        endpoint_ids: Optional[list[str]] = None,
+    ) -> None:
+        pass
+
     def get_monitoring_function_summaries(
         self,
         project: str,

mlrun/execution.py

@@ -934,14 +934,51 @@ class MLClientCtx:
 
         Examples::
 
-            # Log an inline prompt
+            # Log directly with an inline prompt template
             context.log_llm_prompt(
-                key="qa-prompt",
-                prompt_template=[{"role: "user", "content": "question with {place_holder}"}],
+                key="customer_support_prompt",
+                prompt_template=[
+                    {
+                        "role": "system",
+                        "content": "You are a helpful customer support assistant.",
+                    },
+                    {
+                        "role": "user",
+                        "content": "The customer reports: {issue_description}",
+                    },
+                ],
+                prompt_legend={
+                    "issue_description": {
+                        "field": "user_issue",
+                        "description": "Detailed description of the customer's issue",
+                    },
+                    "solution": {
+                        "field": "proposed_solution",
+                        "description": "Suggested fix for the customer's issue",
+                    },
+                },
                 model_artifact=model,
-                prompt_legend={"question": "user_input"},
-                model_configuration={"temperature": 0.7, "max_tokens": 128},
-                tag="latest",
+                model_configuration={"temperature": 0.5, "max_tokens": 200},
+                description="Prompt for handling customer support queries",
+                tag="support-v1",
+                labels={"domain": "support"},
+            )
+
+            # Log a prompt from file
+            context.log_llm_prompt(
+                key="qa_prompt",
+                prompt_path="prompts/template.json",
+                prompt_legend={
+                    "question": {
+                        "field": "user_question",
+                        "description": "The actual question asked by the user",
+                    }
+                },
+                model_artifact=model,
+                model_configuration={"temperature": 0.7, "max_tokens": 256},
+                description="Q&A prompt template with user-provided question",
+                tag="v2",
+                labels={"task": "qa", "stage": "experiment"},
             )
 
         :param key: Unique name of the artifact.
@@ -950,7 +987,10 @@ class MLClientCtx:
          "role": "user", "content": "I need your help with {profession}"]. only "role" and "content" keys allow in any
          str format (upper/lower case), keys will be modified to lower case.
          Cannot be used with `prompt_path`.
-        :param prompt_path: Path to a file containing the prompt content. Cannot be used with `prompt_string`.
+        :param prompt_path: Path to a JSON file containing the prompt template.
+                    Cannot be used together with `prompt_template`.
+                    The file should define a list of dictionaries in the same format
+                    supported by `prompt_template`.
         :param prompt_legend: A dictionary where each key is a placeholder in the prompt (e.g., ``{user_name}``)
                and the value is a dictionary holding two keys, "field", "description". "field" points to the field in
                the event where the value of the place-holder inside the event, if None or not exist will be replaced
@@ -958,9 +998,11 @@ class MLClientCtx:
                Useful for documenting and clarifying dynamic parts of the prompt.
         :param model_artifact: Reference to the parent model (either `ModelArtifact` or model URI string).
         :param model_configuration: Dictionary of generation parameters (e.g., temperature, max_tokens).
-        :param description: Optional description of the prompt.
-        :param target_path: Path to write the artifact locally.
-        :param artifact_path: Path in the artifact store (defaults to project artifact path).
+        :param description:   Optional description of the prompt.
+        :param target_path:   Absolute target path (instead of using artifact_path + local_path)
+        :param artifact_path: Target artifact path (when not using the default)
+                              To define a subpath under the default location use:
+                              `artifact_path=context.artifact_subpath('data')`
         :param tag: Tag/version to assign to the prompt artifact.
         :param labels: Labels to tag the artifact (e.g., list or dict of key-value pairs).
         :param upload: Whether to upload the artifact to the store (defaults to True).

mlrun/model_monitoring/applications/__init__.py

@@ -12,6 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .base import ModelMonitoringApplicationBase
+from .base import ExistingDataHandling, ModelMonitoringApplicationBase
 from .context import MonitoringApplicationContext
 from .results import ModelMonitoringApplicationMetric, ModelMonitoringApplicationResult

mlrun/model_monitoring/applications/base.py

@@ -27,6 +27,7 @@ import mlrun
 import mlrun.common.constants as mlrun_constants
 import mlrun.common.helpers
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
+import mlrun.common.types
 import mlrun.datastore.datastore_profile as ds_profile
 import mlrun.errors
 import mlrun.model_monitoring.api as mm_api
@@ -39,6 +40,12 @@ from mlrun.serving.utils import MonitoringApplicationToDict
 from mlrun.utils import logger
 
 
+class ExistingDataHandling(mlrun.common.types.StrEnum):
+    fail_on_overlap = "fail_on_overlap"
+    skip_overlap = "skip_overlap"
+    delete_all = "delete_all"
+
+
 def _serialize_context_and_result(
     *,
     context: mm_context.MonitoringApplicationContext,
@@ -288,7 +295,7 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         end: Optional[str] = None,
         base_period: Optional[int] = None,
         write_output: bool = False,
-        fail_on_overlap: bool = True,
+        existing_data_handling: ExistingDataHandling = ExistingDataHandling.fail_on_overlap,
         stream_profile: Optional[ds_profile.DatastoreProfile] = None,
     ):
         """
@@ -350,6 +357,24 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
                 resolved_endpoints = self._handle_endpoints_type_evaluate(
                     project=project, endpoints=endpoints
                 )
+                if (
+                    write_output
+                    and existing_data_handling == ExistingDataHandling.delete_all
+                ):
+                    endpoint_ids = [
+                        endpoint_id for _, endpoint_id in resolved_endpoints
+                    ]
+                    context.logger.info(
+                        "Deleting all the application data before running the application",
+                        application_name=application_name,
+                        endpoint_ids=endpoint_ids,
+                    )
+                    self._delete_application_data(
+                        project_name=project.name,
+                        application_name=application_name,
+                        endpoint_ids=endpoint_ids,
+                        application_schedules=application_schedules,
+                    )
                 for endpoint_name, endpoint_id in resolved_endpoints:
                     for window_start, window_end in self._window_generator(
                         start=start,
@@ -358,7 +383,7 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
                         application_schedules=application_schedules,
                         endpoint_id=endpoint_id,
                         application_name=application_name,
-                        fail_on_overlap=fail_on_overlap,
+                        existing_data_handling=existing_data_handling,
                     ):
                         result = call_do_tracking(
                             event={
@@ -481,7 +506,7 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         end_dt: datetime,
         base_period: Optional[int],
         application_name: str,
-        fail_on_overlap: bool,
+        existing_data_handling: ExistingDataHandling,
     ) -> datetime:
         """Make sure that the (app, endpoint) pair doesn't write output before the last analyzed window"""
         if application_schedules:
@@ -490,7 +515,7 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
             )
             if last_analyzed:
                 if start_dt < last_analyzed:
-                    if not fail_on_overlap:
+                    if existing_data_handling == ExistingDataHandling.skip_overlap:
                         if last_analyzed < end_dt and base_period is None:
                             logger.warn(
                                 "Setting the start time to last_analyzed since the original start time precedes "
@@ -525,6 +550,25 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
                 )
         return start_dt
 
+    @staticmethod
+    def _delete_application_data(
+        project_name: str,
+        application_name: str,
+        endpoint_ids: list[str],
+        application_schedules: Optional[
+            mm_schedules.ModelMonitoringSchedulesFileApplication
+        ],
+    ) -> None:
+        mlrun.get_run_db().delete_model_monitoring_metrics(
+            project=project_name,
+            application_name=application_name,
+            endpoint_ids=endpoint_ids,
+        )
+        if application_schedules:
+            application_schedules.delete_endpoints_last_analyzed(
+                endpoint_uids=endpoint_ids
+            )
+
     @classmethod
     def _window_generator(
         cls,
@@ -537,7 +581,7 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         ],
         endpoint_id: str,
         application_name: str,
-        fail_on_overlap: bool,
+        existing_data_handling: ExistingDataHandling,
     ) -> Iterator[tuple[Optional[datetime], Optional[datetime]]]:
         if start is None or end is None:
             # A single window based on the `sample_data` input - see `_handler`.
@@ -547,15 +591,16 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         start_dt = datetime.fromisoformat(start)
         end_dt = datetime.fromisoformat(end)
 
-        start_dt = cls._validate_monotonically_increasing_data(
-            application_schedules=application_schedules,
-            endpoint_id=endpoint_id,
-            start_dt=start_dt,
-            end_dt=end_dt,
-            base_period=base_period,
-            application_name=application_name,
-            fail_on_overlap=fail_on_overlap,
-        )
+        if existing_data_handling != ExistingDataHandling.delete_all:
+            start_dt = cls._validate_monotonically_increasing_data(
+                application_schedules=application_schedules,
+                endpoint_id=endpoint_id,
+                start_dt=start_dt,
+                end_dt=end_dt,
+                base_period=base_period,
+                application_name=application_name,
+                existing_data_handling=existing_data_handling,
+            )
 
         if base_period is None:
             yield start_dt, end_dt
@@ -702,7 +747,7 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         * ``end``, ``datetime``
         * ``base_period``, ``int``
         * ``write_output``, ``bool``
-        * ``fail_on_overlap``, ``bool``
+        * ``existing_data_handling``, ``str``
 
         For Git sources, add the source archive to the returned job and change the handler:
 
@@ -788,7 +833,7 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         end: Optional[datetime] = None,
         base_period: Optional[int] = None,
         write_output: bool = False,
-        fail_on_overlap: bool = True,
+        existing_data_handling: ExistingDataHandling = ExistingDataHandling.fail_on_overlap,
         stream_profile: Optional[ds_profile.DatastoreProfile] = None,
     ) -> "mlrun.RunObject":
         """
@@ -856,11 +901,18 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         :param write_output:      Whether to write the results and metrics to the time-series DB. Can be ``True`` only
                                   if ``endpoints`` are passed.
                                   Note: the model monitoring infrastructure must be up for the writing to work.
-        :param fail_on_overlap:   Relevant only when ``write_output=True``. When ``True``, and the
-                                  requested ``start`` time precedes the ``end`` time of a previous run that also
-                                  wrote to the database - an error is raised.
-                                  If ``False``, when the previously described situation occurs, the relevant time
-                                  window is cut so that it starts at the earliest possible time after ``start``.
+        :param existing_data_handling:
+                                  How to handle the existing application data for the model endpoints when writing the
+                                  new data. Relevant only when ``write_output=True``. The default is
+                                  ``"fail_on_overlap"``. The options are:
+
+                                  - ``"fail_on_overlap"``: when the requested ``start`` time precedes the
+                                    ``end`` time of a previous run that also wrote to the database - an error is raised.
+                                  - ``"skip_overlap"``: when the previously described situation occurs, the relevant
+                                    time window is cut so that it starts at the earliest possible time after ``start``.
+                                  - ``"delete_all"``: delete all the data that was written by the application to the
+                                    model endpoints, regardless of the time window, and write the new data.
+
         :param stream_profile:    The stream datastore profile. It should be provided only when running locally and
                                   writing the outputs to the database (i.e., when both ``run_local`` and
                                   ``write_output`` are set to ``True``).
@@ -899,18 +951,6 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
                 )
                 params["end"] = end.isoformat() if isinstance(end, datetime) else end
                 params["base_period"] = base_period
-                params["write_output"] = write_output
-                params["fail_on_overlap"] = fail_on_overlap
-                if stream_profile:
-                    if not run_local:
-                        raise mlrun.errors.MLRunValueError(
-                            "Passing a `stream_profile` is relevant only when running locally"
-                        )
-                    if not write_output:
-                        raise mlrun.errors.MLRunValueError(
-                            "Passing a `stream_profile` is relevant only when writing the outputs"
-                        )
-                params["stream_profile"] = stream_profile
         elif start or end or base_period:
             raise mlrun.errors.MLRunValueError(
                 "Custom `start` and `end` times or base_period are supported only with endpoints data"
@@ -920,6 +960,19 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
                 "Writing the application output or passing `stream_profile` are supported only with endpoints data"
             )
 
+        params["write_output"] = write_output
+        params["existing_data_handling"] = existing_data_handling
+        if stream_profile:
+            if not run_local:
+                raise mlrun.errors.MLRunValueError(
+                    "Passing a `stream_profile` is relevant only when running locally"
+                )
+            if not write_output:
+                raise mlrun.errors.MLRunValueError(
+                    "Passing a `stream_profile` is relevant only when writing the outputs"
+                )
+        params["stream_profile"] = stream_profile
+
         inputs: dict[str, str] = {}
         for data, identifier in [
             (sample_data, "sample_data"),

mlrun/model_monitoring/db/_schedules.py

@@ -170,6 +170,16 @@ class ModelMonitoringSchedulesFileEndpoint(ModelMonitoringSchedulesFileBase):
         self._check_open_schedules()
         self._schedules[application] = float(timestamp)
 
+    def delete_application_time(self, application: str) -> None:
+        self._check_open_schedules()
+        if application in self._schedules:
+            logger.debug(
+                "Deleting application time from schedules",
+                application=application,
+                endpoint_id=self._endpoint_id,
+            )
+            del self._schedules[application]
+
     def get_application_list(self) -> set[str]:
         self._check_open_schedules()
         return set(self._schedules.keys())
@@ -275,6 +285,17 @@ class ModelMonitoringSchedulesFileApplication(ModelMonitoringSchedulesFileBase):
             timezone.utc
         ).isoformat()
 
+    def delete_endpoints_last_analyzed(self, endpoint_uids: list[str]) -> None:
+        self._check_open_schedules()
+        for endpoint_uid in endpoint_uids:
+            if endpoint_uid in self._schedules:
+                logger.debug(
+                    "Deleting endpoint last analyzed from schedules",
+                    endpoint_uid=endpoint_uid,
+                    application=self._application,
+                )
+                del self._schedules[endpoint_uid]
+
 
 def _delete_folder(folder: str) -> None:
     fs = mlrun.datastore.store_manager.object(folder).store.filesystem

mlrun/model_monitoring/db/tsdb/base.py

@@ -96,14 +96,23 @@ class TSDBConnector(ABC):
         """
 
     @abstractmethod
-    def delete_tsdb_records(
-        self,
-        endpoint_ids: list[str],
-    ) -> None:
+    def delete_tsdb_records(self, endpoint_ids: list[str]) -> None:
         """
         Delete model endpoint records from the TSDB connector.
+
         :param endpoint_ids: List of model endpoint unique identifiers.
-        :param delete_timeout: The timeout in seconds to wait for the deletion to complete.
+        """
+        pass
+
+    @abstractmethod
+    def delete_application_records(
+        self, application_name: str, endpoint_ids: Optional[list[str]] = None
+    ) -> None:
+        """
+        Delete application records from the TSDB for the given model endpoints or all if ``None``.
+
+        :param application_name: The name of the application to delete records for.
+        :param endpoint_ids:     List of model endpoint unique identifiers.
         """
         pass
 

mlrun/model_monitoring/db/tsdb/tdengine/schemas.py

@@ -122,10 +122,7 @@ class TDEngineSchema:
             )
         return f"DELETE FROM {self.database}.{subtable} WHERE {values};"
 
-    def drop_subtable_query(
-        self,
-        subtable: str,
-    ) -> str:
+    def drop_subtable_query(self, subtable: str) -> str:
         return f"DROP TABLE if EXISTS {self.database}.`{subtable}`;"
 
     def drop_supertable_query(self) -> str:
@@ -145,8 +142,10 @@ class TDEngineSchema:
         values = f" {operator} ".join(
             f"{filter_tag} LIKE '{val}'" for val in filter_values
         )
+        return self._get_tables_query_by_condition(values)
 
-        return f"SELECT DISTINCT tbname FROM {self.database}.{self.super_table} WHERE {values};"
+    def _get_tables_query_by_condition(self, condition: str) -> str:
+        return f"SELECT DISTINCT TBNAME FROM {self.database}.{self.super_table} WHERE {condition};"
 
     @staticmethod
     def _get_records_query(