mlrun 1.10.0rc7__py3-none-any.whl → 1.10.0rc8__py3-none-any.whl

mlrun/__init__.py

@@ -61,6 +61,7 @@ from .run import (
     import_function,
     new_function,
     retry_pipeline,
+    terminate_pipeline,
     wait_for_pipeline_completion,
 )
 from .runtimes import mounts, new_model_server
@@ -217,5 +218,6 @@ def set_env_from_file(env_file: str, return_dict: bool = False) -> Optional[dict
     for key, value in env_vars.items():
         environ[key] = value
 
-    mlconf.reload()  # reload mlrun configuration
+    # reload mlrun configuration
+    mlconf.reload()
     return env_vars if return_dict else None

mlrun/common/schemas/background_task.py

@@ -22,6 +22,10 @@ import mlrun.common.types
 from .object import ObjectKind
 
 
+class BackGroundTaskLabel(mlrun.common.types.StrEnum):
+    pipeline = "pipeline"
+
+
 class BackgroundTaskState(mlrun.common.types.StrEnum):
     succeeded = "succeeded"
     failed = "failed"
@@ -37,6 +41,7 @@ class BackgroundTaskState(mlrun.common.types.StrEnum):
 
 class BackgroundTaskMetadata(pydantic.v1.BaseModel):
     name: str
+    id: typing.Optional[int]
     kind: typing.Optional[str]
     project: typing.Optional[str]
     created: typing.Optional[datetime.datetime]

mlrun/common/schemas/model_monitoring/__init__.py

@@ -28,6 +28,7 @@ from .constants import (
     ModelEndpointCreationStrategy,
     ModelEndpointMonitoringMetricType,
     ModelEndpointSchema,
+    ModelMonitoringAppLabel,
     ModelMonitoringMode,
     MonitoringFunctionNames,
     PredictionsQueryConstants,
@@ -36,6 +37,7 @@ from .constants import (
     ResultKindApp,
     ResultStatusApp,
     SpecialApps,
+    StreamProcessingEvent,
     TDEngineSuperTables,
     TSDBTarget,
     V3IOTSDBTables,

mlrun/common/schemas/model_monitoring/constants.py

@@ -142,6 +142,22 @@ class EventFieldType:
     EFFECTIVE_SAMPLE_COUNT = "effective_sample_count"
 
 
+class StreamProcessingEvent:
+    MODEL = "model"
+    MODEL_CLASS = "model_class"
+    MICROSEC = "microsec"
+    WHEN = "when"
+    ERROR = "error"
+    ENDPOINT_ID = "endpoint_id"
+    SAMPLING_PERCENTAGE = "sampling_percentage"
+    EFFECTIVE_SAMPLE_COUNT = "effective_sample_count"
+    LABELS = "labels"
+    FUNCTION_URI = "function_uri"
+    REQUEST = "request"
+    RESPONSE = "resp"
+    METRICS = "metrics"
+
+
 class FeatureSetFeatures(MonitoringStrEnum):
     LATENCY = EventFieldType.LATENCY
     METRICS = EventFieldType.METRICS

mlrun/common/schemas/project.py

@@ -148,6 +148,10 @@ class ProjectSummary(pydantic.v1.BaseModel):
     datasets_count: int = 0
     documents_count: int = 0
     llm_prompts_count: int = 0
+    running_model_monitoring_functions: int = 0
+    failed_model_monitoring_functions: int = 0
+    real_time_model_endpoint_count: int = 0
+    batch_model_endpoint_count: int = 0
 
 
 class IguazioProject(pydantic.v1.BaseModel):

mlrun/common/schemas/serving.py

@@ -33,7 +33,9 @@ class MonitoringData(StrEnum):
     INPUTS = "inputs"
     OUTPUTS = "outputs"
     INPUT_PATH = "input_path"
+    RESULT_PATH = "result_path"
     CREATION_STRATEGY = "creation_strategy"
     LABELS = "labels"
     MODEL_PATH = "model_path"
     MODEL_ENDPOINT_UID = "model_endpoint_uid"
+    MODEL_CLASS = "model_class"

mlrun/config.py

@@ -107,6 +107,8 @@ default_config = {
     "submit_timeout": "280",  # timeout when submitting a new k8s resource
     # runtimes cleanup interval in seconds
     "runtimes_cleanup_interval": "300",
+    "background_task_cleanup_interval": "86400",  # 24 hours in seconds
+    "background_task_max_age": "21600",  # 6 hours in seconds
     "monitoring": {
         "runs": {
             # runs monitoring interval in seconds
@@ -233,6 +235,7 @@ default_config = {
                 "delete_function": "900",
                 "model_endpoint_creation": "600",
                 "model_endpoint_tsdb_leftovers": "900",
+                "terminate_pipeline": "300",
             },
             "runtimes": {
                 "dask": "600",
@@ -638,6 +641,7 @@ default_config = {
         "offline_storage_path": "model-endpoints/{kind}",
         "parquet_batching_max_events": 10_000,
         "parquet_batching_timeout_secs": timedelta(minutes=1).total_seconds(),
+        "model_endpoint_creation_check_period": "15",
     },
     "secret_stores": {
         # Use only in testing scenarios (such as integration tests) to avoid using k8s for secrets (will use in-memory
@@ -896,11 +900,7 @@ class Config:
         return result
 
     def __setattr__(self, attr, value):
-        # in order for the dbpath setter to work
-        if attr == "dbpath":
-            super().__setattr__(attr, value)
-        else:
-            self._cfg[attr] = value
+        self._cfg[attr] = value
 
     def __dir__(self):
         return list(self._cfg) + dir(self.__class__)
@@ -1244,23 +1244,6 @@ class Config:
         # since the property will need to be url, which exists in other structs as well
         return config.ui.url or config.ui_url
 
-    @property
-    def dbpath(self):
-        return self._dbpath
-
-    @dbpath.setter
-    def dbpath(self, value):
-        self._dbpath = value
-        if value:
-            # importing here to avoid circular dependency
-            import mlrun.db
-
-            # It ensures that SSL verification is set before establishing a connection
-            _configure_ssl_verification(self.httpdb.http.verify)
-
-            # when dbpath is set we want to connect to it which will sync configuration from it to the client
-            mlrun.db.get_run_db(value, force_reconnect=True)
-
     def is_api_running_on_k8s(self):
         # determine if the API service is attached to K8s cluster
         # when there is a cluster the .namespace is set
@@ -1436,6 +1419,12 @@ def _do_populate(env=None, skip_errors=False):
     _configure_ssl_verification(config.httpdb.http.verify)
     _validate_config(config)
 
+    if config.dbpath:
+        from mlrun.db import get_run_db
+
+        # when dbpath is set we want to connect to it which will sync configuration from it to the client
+        get_run_db(config.dbpath, force_reconnect=True)
+
 
 def _validate_config(config):
     try:

mlrun/datastore/utils.py

@@ -236,9 +236,11 @@ class KafkaParameters:
             "partitions": "",
             "sasl": "",
             "worker_allocation_mode": "",
-            "tls_enable": "",  # for Nuclio with Confluent Kafka (Sarama client)
+            # for Nuclio with Confluent Kafka
+            "tls_enable": "",
             "tls": "",
             "new_topic": "",
+            "nuclio_annotations": "",
         }
         self._reference_dicts = (
             self._custom_attributes,

mlrun/db/base.py

@@ -439,31 +439,32 @@ class RunDBInterface(ABC):
     ) -> dict:
         pass
 
+    # TODO: remove in 1.10.0
+    @deprecated(
+        version="1.7.0",
+        reason="'list_features' will be removed in 1.10.0, use 'list_features_v2' instead",
+        category=FutureWarning,
+    )
     @abstractmethod
-    def list_features_v2(
+    def list_features(
         self,
         project: str,
         name: Optional[str] = None,
         tag: Optional[str] = None,
         entities: Optional[list[str]] = None,
         labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
-    ) -> mlrun.common.schemas.FeaturesOutputV2:
+    ) -> mlrun.common.schemas.FeaturesOutput:
         pass
 
-    # TODO: remove in 1.10.0
-    @deprecated(
-        version="1.7.0",
-        reason="'list_entities' will be removed in 1.10.0, use 'list_entities_v2' instead",
-        category=FutureWarning,
-    )
     @abstractmethod
-    def list_entities(
+    def list_features_v2(
         self,
         project: str,
         name: Optional[str] = None,
         tag: Optional[str] = None,
+        entities: Optional[list[str]] = None,
         labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
-    ) -> mlrun.common.schemas.EntitiesOutput:
+    ) -> mlrun.common.schemas.FeaturesOutputV2:
         pass
 
     @abstractmethod

mlrun/db/httpdb.py

@@ -46,6 +46,7 @@ import mlrun.utils
 from mlrun.alerts.alert import AlertConfig
 from mlrun.db.auth_utils import OAuthClientIDTokenProvider, StaticTokenProvider
 from mlrun.errors import MLRunInvalidArgumentError, err_to_str
+from mlrun.secrets import get_secret_or_env
 from mlrun_pipelines.utils import compile_pipeline
 
 from ..artifacts import Artifact
@@ -156,9 +157,9 @@ class HTTPRunDB(RunDBInterface):
 
         if config.auth_with_client_id.enabled:
             self.token_provider = OAuthClientIDTokenProvider(
-                token_endpoint=mlrun.get_secret_or_env("MLRUN_AUTH_TOKEN_ENDPOINT"),
-                client_id=mlrun.get_secret_or_env("MLRUN_AUTH_CLIENT_ID"),
-                client_secret=mlrun.get_secret_or_env("MLRUN_AUTH_CLIENT_SECRET"),
+                token_endpoint=get_secret_or_env("MLRUN_AUTH_TOKEN_ENDPOINT"),
+                client_id=get_secret_or_env("MLRUN_AUTH_CLIENT_ID"),
+                client_secret=get_secret_or_env("MLRUN_AUTH_CLIENT_SECRET"),
                 timeout=config.auth_with_client_id.request_timeout,
             )
         else:
@@ -2352,8 +2353,7 @@ class HTTPRunDB(RunDBInterface):
     ):
         """
         Retry a specific pipeline run using its run ID. This function sends an API request
-        to retry a pipeline run. If a project is specified, the run must belong to that
-        project; otherwise, all projects are queried.
+        to retry a pipeline run.
 
         :param run_id: The unique ID of the pipeline run to retry.
         :param namespace: Kubernetes namespace where the pipeline is running. Optional.
@@ -2394,7 +2394,7 @@ class HTTPRunDB(RunDBInterface):
                 namespace=namespace,
                 response_code=resp_code,
                 response_text=resp_text,
-                error=str(exc),
+                error=err_to_str(exc),
             )
             if isinstance(exc, mlrun.errors.MLRunHTTPError):
                 raise exc  # Re-raise known HTTP errors
@@ -2410,6 +2410,72 @@ class HTTPRunDB(RunDBInterface):
         )
         return resp.json()
 
+    def terminate_pipeline(
+        self,
+        run_id: str,
+        project: str,
+        namespace: Optional[str] = None,
+        timeout: int = 30,
+    ):
+        """
+        Terminate a specific pipeline run using its run ID. This function sends an API request
+        to terminate a pipeline run.
+
+        :param run_id: The unique ID of the pipeline run to terminate.
+        :param namespace: Kubernetes namespace where the pipeline is running. Optional.
+        :param timeout: Timeout (in seconds) for the API call. Defaults to 30 seconds.
+        :param project: Name of the MLRun project associated with the pipeline.
+
+        :raises ValueError: Raised if the API response is not successful or contains an
+            error.
+
+        :return: JSON response containing details of the terminate pipeline run background task.
+        """
+
+        params = {}
+        if namespace:
+            params["namespace"] = namespace
+
+        resp_text = ""
+        resp_code = None
+        try:
+            resp = self.api_call(
+                "POST",
+                f"projects/{project}/pipelines/{run_id}/terminate",
+                params=params,
+                timeout=timeout,
+            )
+            resp_code = resp.status_code
+            resp_text = resp.text
+            if not resp.ok:
+                raise mlrun.errors.MLRunHTTPError(
+                    f"Failed to retry pipeline run '{run_id}'. "
+                    f"HTTP {resp_code}: {resp_text}"
+                )
+        except Exception as exc:
+            logger.error(
+                "Failed to invoke terminate pipeline API",
+                run_id=run_id,
+                project=project,
+                namespace=namespace,
+                response_code=resp_code,
+                response_text=resp_text,
+                error=err_to_str(exc),
+            )
+            if isinstance(exc, mlrun.errors.MLRunHTTPError):
+                raise exc  # Re-raise known HTTP errors
+            raise mlrun.errors.MLRunRuntimeError(
+                f"Unexpected error while terminating pipeline run '{run_id}'."
+            ) from exc
+
+        logger.info(
+            "Successfully scheduled terminate pipeline run background task",
+            run_id=run_id,
+            project=project,
+            namespace=namespace,
+        )
+        return resp.json()
+
     @staticmethod
     def _resolve_reference(tag, uid):
         if uid and tag:
@@ -2478,14 +2544,14 @@ class HTTPRunDB(RunDBInterface):
         resp = self.api_call("GET", path, error_message)
         return FeatureSet.from_dict(resp.json())
 
-    def list_features_v2(
+    def list_features(
         self,
         project: Optional[str] = None,
         name: Optional[str] = None,
         tag: Optional[str] = None,
         entities: Optional[list[str]] = None,
         labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
-    ) -> dict[str, list[dict]]:
+    ) -> list[dict]:
         """List feature-sets which contain specific features. This function may return multiple versions of the same
         feature-set if a specific tag is not requested. Note that the various filters of this function actually
         refer to the feature-set object containing the features, not to the features themselves.
@@ -2502,7 +2568,9 @@ class HTTPRunDB(RunDBInterface):
             or just `"label"` for key existence.
             - A comma-separated string formatted as `"label1=value1,label2"` to match entities with
             the specified key-value pairs or key existence.
-        :returns: A list of features, and a list of their corresponding feature sets.
+        :returns: A list of mapping from feature to a digest of the feature-set, which contains the feature-set
+            meta-data. Multiple entries may be returned for any specific feature due to multiple tags or versions
+            of the feature-set.
         """
 
         project = project or config.active_project
@@ -2517,31 +2585,34 @@ class HTTPRunDB(RunDBInterface):
         path = f"projects/{project}/features"
 
         error_message = f"Failed listing features, project: {project}, query: {params}"
-        resp = self.api_call("GET", path, error_message, params=params, version="v2")
-        return resp.json()
+        resp = self.api_call("GET", path, error_message, params=params)
+        return resp.json()["features"]
 
-    def list_entities(
+    def list_features_v2(
         self,
         project: Optional[str] = None,
         name: Optional[str] = None,
         tag: Optional[str] = None,
+        entities: Optional[list[str]] = None,
         labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
-    ) -> list[dict]:
-        """Retrieve a list of entities and their mapping to the containing feature-sets. This function is similar
-        to the :py:func:`~list_features` function, and uses the same logic. However, the entities are matched
-        against the name rather than the features.
+    ) -> dict[str, list[dict]]:
+        """List feature-sets which contain specific features. This function may return multiple versions of the same
+        feature-set if a specific tag is not requested. Note that the various filters of this function actually
+        refer to the feature-set object containing the features, not to the features themselves.
 
-        :param project: The project containing the entities.
-        :param name: The name of the entities to retrieve.
-        :param tag: The tag of the specific entity version to retrieve.
-        :param labels: Filter entities by label key-value pairs or key existence. This can be provided as:
+        :param project: Project which contains these features.
+        :param name: Name of the feature to look for. The name is used in a like query, and is not case-sensitive. For
+            example, looking for ``feat`` will return features which are named ``MyFeature`` as well as ``defeat``.
+        :param tag: Return feature-sets which contain the features looked for, and are tagged with the specific tag.
+        :param entities: Return only feature-sets which contain an entity whose name is contained in this list.
+        :param labels: Filter feature-sets by label key-value pairs or key existence. This can be provided as:
             - A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
             or `{"label": None}` to check for key existence.
             - A list of strings formatted as `"label=value"` to match specific label key-value pairs,
             or just `"label"` for key existence.
             - A comma-separated string formatted as `"label1=value1,label2"` to match entities with
             the specified key-value pairs or key existence.
-        :returns: A list of entities.
+        :returns: A list of features, and a list of their corresponding feature sets.
         """
 
         project = project or config.active_project
@@ -2549,14 +2620,15 @@ class HTTPRunDB(RunDBInterface):
         params = {
             "name": name,
             "tag": tag,
+            "entity": entities or [],
             "label": labels,
         }
 
-        path = f"projects/{project}/entities"
+        path = f"projects/{project}/features"
 
-        error_message = f"Failed listing entities, project: {project}, query: {params}"
-        resp = self.api_call("GET", path, error_message, params=params)
-        return resp.json()["entities"]
+        error_message = f"Failed listing features, project: {project}, query: {params}"
+        resp = self.api_call("GET", path, error_message, params=params, version="v2")
+        return resp.json()
 
     def list_entities_v2(
         self,

mlrun/db/nopdb.py

@@ -371,23 +371,24 @@ class NopDB(RunDBInterface):
     ) -> dict:
         pass
 
-    def list_features_v2(
+    def list_features(
         self,
         project: str,
         name: Optional[str] = None,
         tag: Optional[str] = None,
         entities: Optional[list[str]] = None,
         labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
-    ) -> mlrun.common.schemas.FeaturesOutputV2:
+    ) -> mlrun.common.schemas.FeaturesOutput:
         pass
 
-    def list_entities(
+    def list_features_v2(
         self,
         project: str,
         name: Optional[str] = None,
         tag: Optional[str] = None,
+        entities: Optional[list[str]] = None,
         labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
-    ) -> mlrun.common.schemas.EntitiesOutput:
+    ) -> mlrun.common.schemas.FeaturesOutputV2:
         pass
 
     def list_entities_v2(

mlrun/frameworks/tf_keras/__init__.py

@@ -14,7 +14,7 @@
 
 from typing import Any, Optional, Union
 
-from tensorflow import keras
+import tensorflow as tf
 
 import mlrun
 import mlrun.common.constants as mlrun_constants
@@ -27,11 +27,11 @@ from .utils import TFKerasTypes, TFKerasUtils
 
 
 def apply_mlrun(
-    model: keras.Model = None,
+    model: tf.keras.Model = None,
     model_name: Optional[str] = None,
     tag: str = "",
     model_path: Optional[str] = None,
-    model_format: str = TFKerasModelHandler.ModelFormats.SAVED_MODEL,
+    model_format: Optional[str] = None,
     save_traces: bool = False,
     modules_map: Optional[Union[dict[str, Union[None, str, list[str]]], str]] = None,
     custom_objects_map: Optional[Union[dict[str, Union[str, list[str]]], str]] = None,
@@ -54,7 +54,7 @@ def apply_mlrun(
     :param model_path:                  The model's store object path. Mandatory for evaluation (to know which model to
                                         update). If model is not provided, it will be loaded from this path.
     :param model_format:                The format to use for saving and loading the model. Should be passed as a
-                                        member of the class 'ModelFormats'. Default: 'ModelFormats.SAVED_MODEL'.
+                                        member of the class 'ModelFormats'.
     :param save_traces:                 Whether or not to use functions saving (only available for the 'SavedModel'
                                         format) for loading the model later without the custom objects dictionary. Only
                                         from tensorflow version >= 2.4.0. Using this setting will increase the model

mlrun/frameworks/tf_keras/callbacks/logging_callback.py

@@ -16,14 +16,14 @@ from typing import Callable, Optional, Union
 
 import numpy as np
 import tensorflow as tf
-from tensorflow import Tensor, Variable
+from tensorflow import keras
 from tensorflow.python.keras.callbacks import Callback
 
 import mlrun
 
 from ..._common import LoggingMode
 from ..._dl_common.loggers import Logger
-from ..utils import TFKerasTypes
+from ..utils import TFKerasTypes, is_keras_3
 
 
 class LoggingCallback(Callback):
@@ -70,7 +70,7 @@ class LoggingCallback(Callback):
                                         {
                                             "epochs": 7
                                         }
-        :param auto_log:                Whether or not to enable auto logging, trying to track common static and dynamic
+        :param auto_log:                Whether to enable auto logging, trying to track common static and dynamic
                                         hyperparameters.
         """
         super().__init__()
@@ -385,18 +385,24 @@ class LoggingCallback(Callback):
             self._logger.log_context_parameters()
 
         # Add learning rate:
-        learning_rate_key = "lr"
-        learning_rate_key_chain = ["optimizer", "lr"]
-        if learning_rate_key not in self._dynamic_hyperparameters_keys and hasattr(
-            self.model, "optimizer"
-        ):
-            try:
-                self._get_hyperparameter(key_chain=learning_rate_key_chain)
+        learning_rate_keys = [
+            "learning_rate",
+            "lr",
+        ]  # "lr" is for backward compatibility in older keras versions.
+        if all(
+            learning_rate_key not in self._dynamic_hyperparameters_keys
+            for learning_rate_key in learning_rate_keys
+        ) and hasattr(self.model, "optimizer"):
+            for learning_rate_key in learning_rate_keys:
+                learning_rate_key_chain = ["optimizer", learning_rate_key]
+                try:
+                    self._get_hyperparameter(key_chain=learning_rate_key_chain)
+                except (KeyError, IndexError, AttributeError, ValueError):
+                    continue
                 self._dynamic_hyperparameters_keys[learning_rate_key] = (
                     learning_rate_key_chain
                 )
-            except (KeyError, IndexError, ValueError):
-                pass
+                break
 
     def _get_hyperparameter(
         self,
@@ -427,7 +433,7 @@ class LoggingCallback(Callback):
                         value = value[key]
                     else:
                         value = getattr(value, key)
-                except KeyError or IndexError as KeyChainError:
+                except KeyError or IndexError or AttributeError as KeyChainError:
                     raise KeyChainError(
                         f"Error during getting a hyperparameter value with the key chain {key_chain}. "
                         f"The {value.__class__} in it does not have the following key/index from the key provided: "
@@ -435,7 +441,9 @@ class LoggingCallback(Callback):
                     )
 
         # Parse the value:
-        if isinstance(value, Tensor) or isinstance(value, Variable):
+        if isinstance(value, (tf.Tensor, tf.Variable)) or (
+            is_keras_3() and isinstance(value, (keras.KerasTensor, keras.Variable))
+        ):
             if int(tf.size(value)) == 1:
                 value = float(value)
             else:
@@ -451,12 +459,7 @@ class LoggingCallback(Callback):
                     f"The parameter with the following key chain: {key_chain} is a numpy.ndarray with {value.size} "
                     f"elements. numpy arrays are trackable only if they have 1 element."
                 )
-        elif not (
-            isinstance(value, float)
-            or isinstance(value, int)
-            or isinstance(value, str)
-            or isinstance(value, bool)
-        ):
+        elif not (isinstance(value, (float, int, str, bool))):
             raise mlrun.errors.MLRunInvalidArgumentError(
                 f"The parameter with the following key chain: {key_chain} is of type '{type(value)}'. The only "
                 f"trackable types are: float, int, str and bool."