mlrun 1.7.0rc22__py3-none-any.whl → 1.7.0rc23__py3-none-any.whl

mlrun/common/schemas/__init__.py

@@ -82,6 +82,7 @@ from .events import (
 )
 from .feature_store import (
     EntitiesOutput,
+    EntitiesOutputV2,
     Entity,
     EntityListOutput,
     EntityRecord,
@@ -98,6 +99,7 @@ from .feature_store import (
     FeatureSetSpec,
     FeatureSetsTagsOutput,
     FeaturesOutput,
+    FeaturesOutputV2,
     FeatureVector,
     FeatureVectorRecord,
     FeatureVectorsOutput,

mlrun/common/schemas/feature_store.py

@@ -14,7 +14,7 @@
 #
 from typing import Optional
 
-from pydantic import BaseModel, Extra, Field
+import pydantic
 
 from .auth import AuthorizationResourceTypes, Credentials
 from .object import (
@@ -27,32 +27,62 @@ from .object import (
 )
 
 
-class Feature(BaseModel):
+class FeatureStoreBaseModel(pydantic.BaseModel):
+    """
+    Intermediate base class, in order to override pydantic's configuration, as per
+    https://docs.pydantic.dev/1.10/usage/model_config/#change-behaviour-globally
+    """
+
+    class Config:
+        copy_on_model_validation = "none"
+
+
+class Feature(FeatureStoreBaseModel):
+    name: str
+    value_type: str
+    labels: Optional[dict] = {}
+
+    class Config:
+        extra = pydantic.Extra.allow
+
+
+class QualifiedFeature(FeatureStoreBaseModel):
+    name: str
+    value_type: str
+    feature_set_index: int
+    labels: Optional[dict] = {}
+
+    class Config:
+        extra = pydantic.Extra.allow
+
+
+class Entity(FeatureStoreBaseModel):
     name: str
     value_type: str
     labels: Optional[dict] = {}
 
     class Config:
-        extra = Extra.allow
+        extra = pydantic.Extra.allow
 
 
-class Entity(BaseModel):
+class QualifiedEntity(FeatureStoreBaseModel):
     name: str
     value_type: str
+    feature_set_index: int
     labels: Optional[dict] = {}
 
     class Config:
-        extra = Extra.allow
+        extra = pydantic.Extra.allow
 
 
 class FeatureSetSpec(ObjectSpec):
     entities: list[Entity] = []
     features: list[Feature] = []
-    engine: Optional[str] = Field(default="storey")
+    engine: Optional[str] = pydantic.Field(default="storey")
 
 
-class FeatureSet(BaseModel):
-    kind: ObjectKind = Field(ObjectKind.feature_set, const=True)
+class FeatureSet(FeatureStoreBaseModel):
+    kind: ObjectKind = pydantic.Field(ObjectKind.feature_set, const=True)
     metadata: ObjectMetadata
     spec: FeatureSetSpec
     status: ObjectStatus
@@ -62,7 +92,7 @@ class FeatureSet(BaseModel):
         return AuthorizationResourceTypes.feature_set
 
 
-class EntityRecord(BaseModel):
+class EntityRecord(FeatureStoreBaseModel):
     name: str
     value_type: str
     labels: list[LabelRecord]
@@ -71,7 +101,7 @@ class EntityRecord(BaseModel):
         orm_mode = True
 
 
-class FeatureRecord(BaseModel):
+class FeatureRecord(FeatureStoreBaseModel):
     name: str
     value_type: str
     labels: list[LabelRecord]
@@ -88,44 +118,64 @@ class FeatureSetRecord(ObjectRecord):
         orm_mode = True
 
 
-class FeatureSetsOutput(BaseModel):
+class FeatureSetsOutput(FeatureStoreBaseModel):
     feature_sets: list[FeatureSet]
 
 
-class FeatureSetsTagsOutput(BaseModel):
+class FeatureSetsTagsOutput(FeatureStoreBaseModel):
     tags: list[str] = []
 
 
-class FeatureSetDigestSpec(BaseModel):
+class FeatureSetDigestSpec(FeatureStoreBaseModel):
     entities: list[Entity]
     features: list[Feature]
 
 
-class FeatureSetDigestOutput(BaseModel):
+class FeatureSetDigestOutput(FeatureStoreBaseModel):
     metadata: ObjectMetadata
     spec: FeatureSetDigestSpec
 
 
-class FeatureListOutput(BaseModel):
+class FeatureSetDigestSpecV2(FeatureStoreBaseModel):
+    entities: list[Entity]
+
+
+class FeatureSetDigestOutputV2(FeatureStoreBaseModel):
+    feature_set_index: int
+    metadata: ObjectMetadata
+    spec: FeatureSetDigestSpecV2
+
+
+class FeatureListOutput(FeatureStoreBaseModel):
     feature: Feature
     feature_set_digest: FeatureSetDigestOutput
 
 
-class FeaturesOutput(BaseModel):
+class FeaturesOutput(FeatureStoreBaseModel):
     features: list[FeatureListOutput]
 
 
-class EntityListOutput(BaseModel):
+class FeaturesOutputV2(FeatureStoreBaseModel):
+    features: list[QualifiedFeature]
+    feature_set_digests: list[FeatureSetDigestOutputV2]
+
+
+class EntityListOutput(FeatureStoreBaseModel):
     entity: Entity
     feature_set_digest: FeatureSetDigestOutput
 
 
-class EntitiesOutput(BaseModel):
+class EntitiesOutputV2(FeatureStoreBaseModel):
+    entities: list[QualifiedEntity]
+    feature_set_digests: list[FeatureSetDigestOutputV2]
+
+
+class EntitiesOutput(FeatureStoreBaseModel):
     entities: list[EntityListOutput]
 
 
-class FeatureVector(BaseModel):
-    kind: ObjectKind = Field(ObjectKind.feature_vector, const=True)
+class FeatureVector(FeatureStoreBaseModel):
+    kind: ObjectKind = pydantic.Field(ObjectKind.feature_vector, const=True)
     metadata: ObjectMetadata
     spec: ObjectSpec
     status: ObjectStatus
@@ -139,39 +189,39 @@ class FeatureVectorRecord(ObjectRecord):
     pass
 
 
-class FeatureVectorsOutput(BaseModel):
+class FeatureVectorsOutput(FeatureStoreBaseModel):
     feature_vectors: list[FeatureVector]
 
 
-class FeatureVectorsTagsOutput(BaseModel):
+class FeatureVectorsTagsOutput(FeatureStoreBaseModel):
     tags: list[str] = []
 
 
-class DataSource(BaseModel):
+class DataSource(FeatureStoreBaseModel):
     kind: str
     name: str
     path: str
 
     class Config:
-        extra = Extra.allow
+        extra = pydantic.Extra.allow
 
 
-class DataTarget(BaseModel):
+class DataTarget(FeatureStoreBaseModel):
     kind: str
     name: str
     path: Optional[str]
 
     class Config:
-        extra = Extra.allow
+        extra = pydantic.Extra.allow
 
 
-class FeatureSetIngestInput(BaseModel):
+class FeatureSetIngestInput(FeatureStoreBaseModel):
     source: Optional[DataSource]
     targets: Optional[list[DataTarget]]
     infer_options: Optional[int]
     credentials: Credentials = Credentials()
 
 
-class FeatureSetIngestOutput(BaseModel):
+class FeatureSetIngestOutput(FeatureStoreBaseModel):
     feature_set: FeatureSet
     run_object: dict

mlrun/db/base.py

@@ -838,6 +838,7 @@ class RunDBInterface(ABC):
         base_period: int = 10,
         image: str = "mlrun/mlrun",
         deploy_histogram_data_drift_app: bool = True,
+        rebuild_images: bool = False,
     ) -> None:
         pass
 

mlrun/db/httpdb.py

@@ -3320,6 +3320,7 @@ class HTTPRunDB(RunDBInterface):
         base_period: int = 10,
         image: str = "mlrun/mlrun",
         deploy_histogram_data_drift_app: bool = True,
+        rebuild_images: bool = False,
     ) -> None:
         """
         Deploy model monitoring application controller, writer and stream functions.
@@ -3329,13 +3330,14 @@ class HTTPRunDB(RunDBInterface):
         The stream function goal is to monitor the log of the data stream. It is triggered when a new log entry
         is detected. It processes the new events into statistics that are then written to statistics databases.
 
-        :param project:     Project name.
-        :param base_period: The time period in minutes in which the model monitoring controller function
-                            triggers. By default, the base period is 10 minutes.
-        :param image:       The image of the model monitoring controller, writer & monitoring
-                            stream functions, which are real time nuclio functions.
-                            By default, the image is mlrun/mlrun.
+        :param project:                         Project name.
+        :param base_period:                     The time period in minutes in which the model monitoring controller
+                                                function triggers. By default, the base period is 10 minutes.
+        :param image:                           The image of the model monitoring controller, writer & monitoring
+                                                stream functions, which are real time nuclio functions.
+                                                By default, the image is mlrun/mlrun.
         :param deploy_histogram_data_drift_app: If true, deploy the default histogram-based data drift application.
+        :param rebuild_images:                  If true, force rebuild of model monitoring infrastructure images.
         """
         self.api_call(
             method=mlrun.common.types.HTTPMethod.POST,
@@ -3344,6 +3346,7 @@ class HTTPRunDB(RunDBInterface):
                 "base_period": base_period,
                 "image": image,
                 "deploy_histogram_data_drift_app": deploy_histogram_data_drift_app,
+                "rebuild_images": rebuild_images,
             },
         )
 

mlrun/db/nopdb.py

@@ -675,6 +675,7 @@ class NopDB(RunDBInterface):
         base_period: int = 10,
         image: str = "mlrun/mlrun",
         deploy_histogram_data_drift_app: bool = True,
+        rebuild_images: bool = False,
     ) -> None:
         pass
 

mlrun/errors.py

@@ -92,9 +92,7 @@ def raise_for_status(
     try:
         response.raise_for_status()
     except (requests.HTTPError, aiohttp.ClientResponseError) as exc:
-        error_message = err_to_str(exc)
-        if message:
-            error_message = f"{error_message}: {message}"
+        error_message = err_to_str(exc) if not message else message
         status_code = (
             response.status_code
             if hasattr(response, "status_code")

mlrun/frameworks/__init__.py

@@ -12,11 +12,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-"""
-MLRun provides a quick and easy integration into your code with mlrun.frameworks: a collection of sub-modules
-for the most commonly used machine and deep learning frameworks, providing features such as automatic logging,
-model management, and distributed training.
-"""
-
 # flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 from .parallel_coordinates import compare_db_runs, compare_run_objects

mlrun/model_monitoring/db/stores/__init__.py

@@ -31,17 +31,12 @@ class ObjectStoreFactory(enum.Enum):
     def to_object_store(
         self,
         project: str,
-        access_key: str = None,
-        secret_provider: typing.Callable = None,
+        **kwargs,
     ) -> StoreBase:
         """
         Return a StoreBase object based on the provided enum value.
 
         :param project:                   The name of the project.
-        :param access_key:                Access key with permission to the DB table. Note that if access key is None
-                                          and the endpoint target is from type KV then the access key will be
-                                          retrieved from the environment variable.
-        :param secret_provider:           An optional secret provider to get the connection string secret.
 
         :return: `StoreBase` object.
 
@@ -50,10 +45,7 @@ class ObjectStoreFactory(enum.Enum):
         if self == self.v3io_nosql:
             from mlrun.model_monitoring.db.stores.v3io_kv.kv_store import KVStoreBase
 
-            # Get V3IO access key from env
-            access_key = access_key or mlrun.mlconf.get_v3io_access_key()
-
-            return KVStoreBase(project=project, access_key=access_key)
+            return KVStoreBase(project=project)
 
         # Assuming SQL store target if store type is not KV.
         # Update these lines once there are more than two store target types.
@@ -62,7 +54,7 @@ class ObjectStoreFactory(enum.Enum):
 
         return SQLStoreBase(
             project=project,
-            secret_provider=secret_provider,
+            **kwargs,
         )
 
     @classmethod
@@ -79,7 +71,7 @@ class ObjectStoreFactory(enum.Enum):
 def get_model_endpoint_store(
     project: str,
     access_key: str = None,
-    secret_provider: typing.Callable = None,
+    secret_provider: typing.Optional[typing.Callable[[str], str]] = None,
 ) -> StoreBase:
     # Leaving here for backwards compatibility
     warnings.warn(
@@ -95,24 +87,38 @@ def get_model_endpoint_store(
 
 def get_store_object(
     project: str,
-    access_key: str = None,
-    secret_provider: typing.Callable = None,
+    secret_provider: typing.Optional[typing.Callable[[str], str]] = None,
+    **kwargs,
 ) -> StoreBase:
     """
-    Getting the DB target type based on mlrun.config.model_endpoint_monitoring.store_type.
+    Generate a store object. If a connection string is provided, the store type will be updated according to the
+    connection string. Currently, the supported store types are SQL and v3io-nosql.
 
     :param project:         The name of the project.
-    :param access_key:      Access key with permission to the DB table.
     :param secret_provider: An optional secret provider to get the connection string secret.
 
-    :return: `StoreBase` object. Using this object, the user can apply different operations on the
-             model monitoring record such as write, update, get and delete a model endpoint.
+    :return: `StoreBase` object. Using this object, the user can apply different operations such as write, update, get
+    and delete a model endpoint record.
     """
 
+    store_connection_string = mlrun.model_monitoring.helpers.get_connection_string(
+        secret_provider=secret_provider
+    )
+
+    if store_connection_string and (
+        store_connection_string.startswith("mysql")
+        or store_connection_string.startswith("sqlite")
+    ):
+        store_type = mlrun.common.schemas.model_monitoring.ModelEndpointTarget.SQL
+        kwargs["store_connection_string"] = store_connection_string
+    else:
+        # Set the default store type if no connection has been set
+        store_type = mlrun.mlconf.model_endpoint_monitoring.store_type
+
     # Get store type value from ObjectStoreFactory enum class
-    store_type = ObjectStoreFactory(mlrun.mlconf.model_endpoint_monitoring.store_type)
+    store_type_fact = ObjectStoreFactory(store_type)
 
     # Convert into store target object
-    return store_type.to_object_store(
-        project=project, access_key=access_key, secret_provider=secret_provider
+    return store_type_fact.to_object_store(
+        project=project, secret_provider=secret_provider, **kwargs
     )

mlrun/model_monitoring/db/stores/base/store.py

@@ -19,6 +19,7 @@ import mlrun.common.schemas.model_monitoring as mm_schemas
 
 
 class StoreBase(ABC):
+    type: typing.ClassVar[str]
     """
     An abstract class to handle the store object in the DB target.
     """

mlrun/model_monitoring/db/stores/sqldb/sql_store.py

@@ -25,14 +25,15 @@ from sqlalchemy.sql.elements import BinaryExpression
 
 import mlrun.common.model_monitoring.helpers
 import mlrun.common.schemas.model_monitoring as mm_schemas
-import mlrun.model_monitoring.db
 import mlrun.model_monitoring.db.stores.sqldb.models
 import mlrun.model_monitoring.helpers
 from mlrun.common.db.sql_session import create_session, get_engine
+from mlrun.model_monitoring.db import StoreBase
 from mlrun.utils import datetime_now, logger
 
 
-class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
+class SQLStoreBase(StoreBase):
+    type: typing.ClassVar[str] = mm_schemas.ModelEndpointTarget.SQL
     """
     Handles the DB operations when the DB target is from type SQL. For the SQL operations, we use SQLAlchemy, a Python
     SQL toolkit that handles the communication with the database.  When using SQL for storing the model monitoring
@@ -44,23 +45,22 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
     def __init__(
         self,
         project: str,
-        secret_provider: typing.Callable = None,
+        **kwargs,
     ):
         """
         Initialize SQL store target object.
 
         :param project:               The name of the project.
-        :param secret_provider:       An optional secret provider to get the connection string secret.
         """
 
         super().__init__(project=project)
 
-        self._sql_connection_string = (
-            mlrun.model_monitoring.helpers.get_connection_string(
-                secret_provider=secret_provider
+        if "store_connection_string" not in kwargs:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "connection_string is a required parameter for SQLStoreBase."
             )
-        )
 
+        self._sql_connection_string = kwargs.get("store_connection_string")
         self._engine = get_engine(dsn=self._sql_connection_string)
 
     def _init_tables(self):

mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 
 import json
-import os
 import typing
 from dataclasses import dataclass
 from http import HTTPStatus
@@ -24,8 +23,8 @@ import v3io.dataplane.response
 
 import mlrun.common.model_monitoring.helpers
 import mlrun.common.schemas.model_monitoring as mm_schemas
-import mlrun.model_monitoring.db
 import mlrun.utils.v3io_clients
+from mlrun.model_monitoring.db import StoreBase
 from mlrun.utils import logger
 
 # Fields to encode before storing in the KV table or to decode after retrieving
@@ -89,18 +88,21 @@ _KIND_TO_SCHEMA_PARAMS: dict[mm_schemas.WriterEventKind, SchemaParams] = {
 _EXCLUDE_SCHEMA_FILTER_EXPRESSION = '__name!=".#schema"'
 
 
-class KVStoreBase(mlrun.model_monitoring.db.StoreBase):
+class KVStoreBase(StoreBase):
+    type: typing.ClassVar[str] = "v3io-nosql"
     """
     Handles the DB operations when the DB target is from type KV. For the KV operations, we use an instance of V3IO
     client and usually the KV table can be found under v3io:///users/pipelines/project-name/model-endpoints/endpoints/.
     """
 
-    def __init__(self, project: str, access_key: typing.Optional[str] = None) -> None:
+    def __init__(
+        self,
+        project: str,
+    ) -> None:
         super().__init__(project=project)
         # Initialize a V3IO client instance
-        self.access_key = access_key or os.environ.get("V3IO_ACCESS_KEY")
         self.client = mlrun.utils.v3io_clients.get_v3io_client(
-            endpoint=mlrun.mlconf.v3io_api, access_key=self.access_key
+            endpoint=mlrun.mlconf.v3io_api,
         )
         # Get the KV table path and container
         self.path, self.container = self._get_path_and_container()
@@ -186,7 +188,6 @@ class KVStoreBase(mlrun.model_monitoring.db.StoreBase):
             table_path=self.path,
             key=endpoint_id,
             raise_for_status=v3io.dataplane.RaiseForStatus.never,
-            access_key=self.access_key,
         )
         endpoint = endpoint.output.item
 
@@ -499,7 +500,6 @@ class KVStoreBase(mlrun.model_monitoring.db.StoreBase):
 
     def _get_frames_client(self):
         return mlrun.utils.v3io_clients.get_frames_client(
-            token=self.access_key,
             address=mlrun.mlconf.v3io_framesd,
             container=self.container,
         )

mlrun/model_monitoring/db/tsdb/__init__.py

@@ -65,7 +65,7 @@ class ObjectTSDBFactory(enum.Enum):
 def get_tsdb_connector(
     project: str,
     tsdb_connector_type: str = "",
-    secret_provider: typing.Optional[typing.Callable] = None,
+    secret_provider: typing.Optional[typing.Callable[[str], str]] = None,
     **kwargs,
 ) -> TSDBConnector:
     """

mlrun/model_monitoring/db/tsdb/base.py

@@ -25,7 +25,7 @@ from mlrun.utils import logger
 
 
 class TSDBConnector(ABC):
-    type: str = ""
+    type: typing.ClassVar[str]
 
     def __init__(self, project: str):
         """

mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py

@@ -418,9 +418,8 @@ class V3IOTSDBConnector(TSDBConnector):
                 f"Available tables: {list(self.tables.keys())}"
             )
 
-        if agg_funcs:
-            # Frames client expects the aggregators to be a comma-separated string
-            aggregators = ",".join(agg_funcs)
+        # Frames client expects the aggregators to be a comma-separated string
+        aggregators = ",".join(agg_funcs) if agg_funcs else None
         table_path = self.tables[table]
         try:
             df = self._frames_client.read(

mlrun/model_monitoring/helpers.py

@@ -97,7 +97,7 @@ def get_monitoring_parquet_path(
     return parquet_path
 
 
-def get_connection_string(secret_provider: typing.Callable = None) -> str:
+def get_connection_string(secret_provider: typing.Callable[[str], str] = None) -> str:
     """Get endpoint store connection string from the project secret. If wasn't set, take it from the system
     configurations.
 
@@ -117,7 +117,7 @@ def get_connection_string(secret_provider: typing.Callable = None) -> str:
 
 
 def get_tsdb_connection_string(
-    secret_provider: typing.Optional[typing.Callable] = None,
+    secret_provider: typing.Optional[typing.Callable[[str], str]] = None,
 ) -> str:
     """Get TSDB connection string from the project secret. If wasn't set, take it from the system
     configurations.
@@ -278,9 +278,13 @@ def calculate_inputs_statistics(
     return inputs_statistics
 
 
-def get_endpoint_record(project: str, endpoint_id: str):
+def get_endpoint_record(
+    project: str,
+    endpoint_id: str,
+    secret_provider: typing.Optional[typing.Callable[[str], str]] = None,
+) -> dict[str, typing.Any]:
     model_endpoint_store = mlrun.model_monitoring.get_store_object(
-        project=project,
+        project=project, secret_provider=secret_provider
     )
     return model_endpoint_store.get_model_endpoint(endpoint_id=endpoint_id)
 

mlrun/model_monitoring/stream_processing.py

@@ -66,10 +66,6 @@ class EventStreamProcessor:
         self.parquet_batching_max_events = parquet_batching_max_events
         self.parquet_batching_timeout_secs = parquet_batching_timeout_secs
 
-        self.model_endpoint_store_target = (
-            mlrun.mlconf.model_endpoint_monitoring.store_type
-        )
-
         logger.info(
             "Initializing model monitoring event stream processor",
             parquet_path=self.parquet_path,
@@ -139,7 +135,7 @@ class EventStreamProcessor:
     def apply_monitoring_serving_graph(
         self,
         fn: mlrun.runtimes.ServingRuntime,
-        tsdb_service_provider: typing.Optional[typing.Callable] = None,
+        secret_provider: typing.Optional[typing.Callable[[str], str]] = None,
     ) -> None:
         """
         Apply monitoring serving graph to a given serving function. The following serving graph includes about 4 main
@@ -167,7 +163,8 @@ class EventStreamProcessor:
            using CE, the parquet target path is based on the defined MLRun artifact path.
 
         :param fn: A serving function.
-        :param tsdb_service_provider: An optional callable function that provides the TSDB connection string.
+        :param secret_provider: An optional callable function that provides the connection string from the project
+                                secret.
         """
 
         graph = typing.cast(
@@ -293,7 +290,6 @@ class EventStreamProcessor:
                 name="UpdateEndpoint",
                 after="ProcessBeforeEndpointUpdate",
                 project=self.project,
-                model_endpoint_store_target=self.model_endpoint_store_target,
             )
 
         apply_update_endpoint()
@@ -310,7 +306,10 @@ class EventStreamProcessor:
                 table=self.kv_path,
             )
 
-        if self.model_endpoint_store_target == ModelEndpointTarget.V3IO_NOSQL:
+        store_object = mlrun.model_monitoring.get_store_object(
+            project=self.project, secret_provider=secret_provider
+        )
+        if store_object.type == ModelEndpointTarget.V3IO_NOSQL:
             apply_infer_schema()
 
         # Emits the event in window size of events based on sample_window size (10 by default)
@@ -328,7 +327,7 @@ class EventStreamProcessor:
         # TSDB branch (skip to Prometheus if in CE env)
         if not mlrun.mlconf.is_ce_mode():
             tsdb_connector = mlrun.model_monitoring.get_tsdb_connector(
-                project=self.project, secret_provider=tsdb_service_provider
+                project=self.project, secret_provider=secret_provider
             )
             tsdb_connector.apply_monitoring_stream_steps(graph=graph)
 
@@ -904,7 +903,7 @@ class MapFeatureNames(mlrun.feature_store.steps.MapClass):
 
 
 class UpdateEndpoint(mlrun.feature_store.steps.MapClass):
-    def __init__(self, project: str, model_endpoint_store_target: str, **kwargs):
+    def __init__(self, project: str, **kwargs):
         """
         Update the model endpoint record in the DB. Note that the event at this point includes metadata and stats about
         the average latency and the amount of predictions over time. This data will be used in the monitoring dashboards
@@ -914,7 +913,6 @@ class UpdateEndpoint(mlrun.feature_store.steps.MapClass):
         """
         super().__init__(**kwargs)
         self.project = project
-        self.model_endpoint_store_target = model_endpoint_store_target
 
     def do(self, event: dict):
         # Remove labels from the event

mlrun/model_monitoring/writer.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import json
-from typing import Any, NewType
+from typing import Any, Callable, NewType
 
 import mlrun.common.model_monitoring
 import mlrun.common.schemas
@@ -30,7 +30,7 @@ from mlrun.common.schemas.model_monitoring.constants import (
     WriterEventKind,
 )
 from mlrun.common.schemas.notification import NotificationKind, NotificationSeverity
-from mlrun.model_monitoring.helpers import get_endpoint_record, get_result_instance_fqn
+from mlrun.model_monitoring.helpers import get_result_instance_fqn
 from mlrun.serving.utils import StepToDict
 from mlrun.utils import logger
 from mlrun.utils.notifications.notification_pusher import CustomNotificationPusher
@@ -102,7 +102,11 @@ class ModelMonitoringWriter(StepToDict):
 
     kind = "monitoring_application_stream_pusher"
 
-    def __init__(self, project: str, tsdb_secret_provider=None) -> None:
+    def __init__(
+        self,
+        project: str,
+        secret_provider: Callable = None,
+    ) -> None:
         self.project = project
         self.name = project  # required for the deployment process
 
@@ -111,10 +115,10 @@ class ModelMonitoringWriter(StepToDict):
         )
 
         self._app_result_store = mlrun.model_monitoring.get_store_object(
-            project=self.project
+            project=self.project, secret_provider=secret_provider
         )
         self._tsdb_connector = mlrun.model_monitoring.get_tsdb_connector(
-            project=self.project, secret_provider=tsdb_secret_provider
+            project=self.project, secret_provider=secret_provider
         )
         self._endpoints_records = {}
 
@@ -223,7 +227,7 @@ class ModelMonitoringWriter(StepToDict):
             endpoint_id = event[WriterEvent.ENDPOINT_ID]
             endpoint_record = self._endpoints_records.setdefault(
                 endpoint_id,
-                get_endpoint_record(project=self.project, endpoint_id=endpoint_id),
+                self._app_result_store.get_model_endpoint(endpoint_id=endpoint_id),
             )
             event_value = {
                 "app_name": event[WriterEvent.APPLICATION_NAME],

mlrun/package/__init__.py

@@ -12,19 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-"""
-MLRun package enables fully-automated experiment and pipeline tracking and reproducibility, and easy passing of
-python objects between remote jobs, while not requiring any form of editing to the actual function original code.
-Simply set the function code in a project and run it, MLRun takes care of the rest.
-
-MLRun uses packagers: classes that perform 2 tasks:
-
-#. **Parsing inputs** - automatically cast the runtime's inputs (user's input passed to the function via
-the ``inputs`` parameter of the ``run`` method) to the relevant hinted type.  (Does not require handling of data items.)
-#. **Logging outputs** - automatically save, log, and upload the function's returned objects by the provided
-log hints (user's input passed to the function via the ``returns`` parameter of the ``run`` method).
-(Does not require handling of files and artifacts.)
-"""
+
 # flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 
 import functools

mlrun/package/packagers/__init__.py

@@ -12,12 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-r"""
-MLRun comes with the following list of modules, out of the box. All of the packagers listed here
-use the implementation of :ref:`DefaultPackager <mlrun.package.packagers.default\_packager.DefaultPackager>` and are
-available by default at the start of each run.
-"""
-# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 
+# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 from .default_packager import DefaultPackager
 from .numpy_packagers import NumPySupportedFormat

mlrun/projects/project.py

@@ -2116,6 +2116,7 @@ class MlrunProject(ModelObj):
         *,
         deploy_histogram_data_drift_app: bool = True,
         wait_for_deployment: bool = False,
+        rebuild_images: bool = False,
     ) -> None:
         """
         Deploy model monitoring application controller, writer and stream functions.
@@ -2135,6 +2136,7 @@ class MlrunProject(ModelObj):
         :param wait_for_deployment:             If true, return only after the deployment is done on the backend.
                                                 Otherwise, deploy the model monitoring infrastructure on the
                                                 background, including the histogram data drift app if selected.
+        :param rebuild_images:                  If true, force rebuild of model monitoring infrastructure images.
         """
         if default_controller_image != "mlrun/mlrun":
             # TODO: Remove this in 1.9.0
@@ -2150,6 +2152,7 @@ class MlrunProject(ModelObj):
             image=image,
             base_period=base_period,
             deploy_histogram_data_drift_app=deploy_histogram_data_drift_app,
+            rebuild_images=rebuild_images,
         )
 
         if wait_for_deployment:
@@ -3192,7 +3195,8 @@ class MlrunProject(ModelObj):
         tsdb_connection: Optional[str] = None,
     ):
         """Set the credentials that will be used by the project's model monitoring
-        infrastructure functions.
+        infrastructure functions. Important to note that you have to set the credentials before deploying any
+        model monitoring or serving function.
 
         :param access_key:                Model Monitoring access key for managing user permissions
         :param endpoint_store_connection: Endpoint store connection string

mlrun/runtimes/nuclio/application/application.py

@@ -263,7 +263,6 @@ class ApplicationRuntime(RemoteRuntime):
         is_kfp=False,
         mlrun_version_specifier=None,
         show_on_failure: bool = False,
-        skip_access_key_auth: bool = False,
         direct_port_access: bool = False,
         authentication_mode: schemas.APIGatewayAuthenticationMode = None,
         authentication_creds: tuple[str] = None,
@@ -283,7 +282,6 @@ class ApplicationRuntime(RemoteRuntime):
         :param is_kfp:                  Deploy as part of a kfp pipeline
         :param mlrun_version_specifier: Which mlrun package version to include (if not current)
         :param show_on_failure:         Show logs only in case of build failure
-        :param skip_access_key_auth:    Skip adding access key auth to the API Gateway
         :param direct_port_access:      Set True to allow direct port access to the application sidecar
         :param authentication_mode:     API Gateway authentication mode
         :param authentication_creds:    API Gateway authentication credentials as a tuple (username, password)

mlrun/runtimes/nuclio/serving.py

@@ -312,15 +312,18 @@ class ServingRuntime(RemoteRuntime):
         sample: Optional[int] = None,
         stream_args: Optional[dict] = None,
         tracking_policy: Optional[Union["TrackingPolicy", dict]] = None,
+        enable_tracking: bool = True,
     ) -> None:
         """apply on your serving function to monitor a deployed model, including real-time dashboards to detect drift
            and analyze performance.
 
-        :param stream_path:     Path/url of the tracking stream e.g. v3io:///users/mike/mystream
-                                you can use the "dummy://" path for test/simulation.
-        :param batch:           Micro batch size (send micro batches of N records at a time).
-        :param sample:          Sample size (send only one of N records).
-        :param stream_args:     Stream initialization parameters, e.g. shards, retention_in_hours, ..
+        :param stream_path:         Path/url of the tracking stream e.g. v3io:///users/mike/mystream
+                                    you can use the "dummy://" path for test/simulation.
+        :param batch:               Micro batch size (send micro batches of N records at a time).
+        :param sample:              Sample size (send only one of N records).
+        :param stream_args:         Stream initialization parameters, e.g. shards, retention_in_hours, ..
+        :param enable_tracking:     Enabled/Disable model-monitoring tracking.
+                                    Default True (tracking enabled).
 
                                 example::
 
@@ -331,7 +334,7 @@ class ServingRuntime(RemoteRuntime):
 
         """
         # Applying model monitoring configurations
-        self.spec.track_models = True
+        self.spec.track_models = enable_tracking
 
         if stream_path:
             self.spec.parameters["log_stream"] = stream_path

mlrun/serving/v2_serving.py

@@ -542,48 +542,64 @@ def _init_endpoint_record(
         function_uri=graph_server.function_uri, versioned_model=versioned_model_name
     ).uid
 
-    # If model endpoint object was found in DB, skip the creation process.
     try:
-        mlrun.get_run_db().get_model_endpoint(project=project, endpoint_id=uid)
-
+        model_ep = mlrun.get_run_db().get_model_endpoint(
+            project=project, endpoint_id=uid
+        )
     except mlrun.errors.MLRunNotFoundError:
-        logger.info("Creating a new model endpoint record", endpoint_id=uid)
-
-        try:
-            model_endpoint = mlrun.common.schemas.ModelEndpoint(
-                metadata=mlrun.common.schemas.ModelEndpointMetadata(
-                    project=project, labels=model.labels, uid=uid
-                ),
-                spec=mlrun.common.schemas.ModelEndpointSpec(
-                    function_uri=graph_server.function_uri,
-                    model=versioned_model_name,
-                    model_class=model.__class__.__name__,
-                    model_uri=model.model_path,
-                    stream_path=config.model_endpoint_monitoring.store_prefixes.default.format(
-                        project=project, kind="stream"
-                    ),
-                    active=True,
-                    monitoring_mode=mlrun.common.schemas.model_monitoring.ModelMonitoringMode.enabled
-                    if model.context.server.track_models
-                    else mlrun.common.schemas.model_monitoring.ModelMonitoringMode.disabled,
-                ),
-                status=mlrun.common.schemas.ModelEndpointStatus(
-                    endpoint_type=mlrun.common.schemas.model_monitoring.EndpointType.NODE_EP
+        model_ep = None
+
+    if model.context.server.track_models and not model_ep:
+        logger.debug("Creating a new model endpoint record", endpoint_id=uid)
+        model_endpoint = mlrun.common.schemas.ModelEndpoint(
+            metadata=mlrun.common.schemas.ModelEndpointMetadata(
+                project=project, labels=model.labels, uid=uid
+            ),
+            spec=mlrun.common.schemas.ModelEndpointSpec(
+                function_uri=graph_server.function_uri,
+                model=versioned_model_name,
+                model_class=model.__class__.__name__,
+                model_uri=model.model_path,
+                stream_path=config.model_endpoint_monitoring.store_prefixes.default.format(
+                    project=project, kind="stream"
                 ),
-            )
-
-            db = mlrun.get_run_db()
-
-            db.create_model_endpoint(
-                project=project,
-                endpoint_id=uid,
-                model_endpoint=model_endpoint.dict(),
-            )
+                active=True,
+                monitoring_mode=mlrun.common.schemas.model_monitoring.ModelMonitoringMode.enabled,
+            ),
+            status=mlrun.common.schemas.ModelEndpointStatus(
+                endpoint_type=mlrun.common.schemas.model_monitoring.EndpointType.NODE_EP
+            ),
+        )
 
-        except Exception as e:
-            logger.error("Failed to create endpoint record", exc=err_to_str(e))
+        db = mlrun.get_run_db()
+        db.create_model_endpoint(
+            project=project,
+            endpoint_id=uid,
+            model_endpoint=model_endpoint.dict(),
+        )
 
-    except Exception as e:
-        logger.error("Failed to retrieve model endpoint object", exc=err_to_str(e))
+    elif (
+        model_ep
+        and (
+            model_ep.spec.monitoring_mode
+            == mlrun.common.schemas.model_monitoring.ModelMonitoringMode.enabled
+        )
+        != model.context.server.track_models
+    ):
+        monitoring_mode = (
+            mlrun.common.schemas.model_monitoring.ModelMonitoringMode.enabled
+            if model.context.server.track_models
+            else mlrun.common.schemas.model_monitoring.ModelMonitoringMode.disabled
+        )
+        db = mlrun.get_run_db()
+        db.patch_model_endpoint(
+            project=project,
+            endpoint_id=uid,
+            attributes={"monitoring_mode": monitoring_mode},
+        )
+        logger.debug(
+            f"Updating model endpoint monitoring_mode to {monitoring_mode}",
+            endpoint_id=uid,
+        )
 
     return uid

mlrun/utils/notifications/notification/base.py

@@ -69,16 +69,27 @@ class NotificationBase:
         if custom_html:
             return custom_html
 
-        if self.name:
-            message = f"{self.name}: {message}"
-
         if alert:
             if not event_data:
                 return f"[{severity}] {message}"
-            return (
-                f"[{severity}] {message} for project {alert.project} "
-                f"UID {event_data.entity.ids[0]}. Values {event_data.value_dict}"
-            )
+
+            html = f"<h3>[{severity}] {message}</h3>"
+            html += f"<br>{alert.name} alert has occurred<br>"
+            html += f"<br><h4>Project:</h4>{alert.project}<br>"
+            html += f"<br><h4>ID:</h4>{event_data.entity.ids[0]}<br>"
+            html += f"<br><h4>Summary:</h4>{mlrun.utils.helpers.format_alert_summary(alert, event_data)}<br>"
+
+            if event_data.value_dict:
+                html += "<br><h4>Event data:</h4>"
+                for key, value in event_data.value_dict.items():
+                    html += f"{key}: {value}<br>"
+
+            overview_type, url = self._get_overview_type_and_url(alert, event_data)
+            html += f"<br><h4>Overview:</h4><a href={url}>{overview_type}</a>"
+            return html
+
+        if self.name:
+            message = f"{self.name}: {message}"
 
         if not runs:
             return f"[{severity}] {message}"
@@ -90,3 +101,24 @@ class NotificationBase:
         html += "<br>click the hyper links below to see detailed results<br>"
         html += runs.show(display=False, short=True)
         return html
+
+    def _get_overview_type_and_url(
+        self,
+        alert: mlrun.common.schemas.AlertConfig,
+        event_data: mlrun.common.schemas.Event,
+    ) -> (str, str):
+        if (
+            event_data.entity.kind == mlrun.common.schemas.alert.EventEntityKind.JOB
+        ):  # JOB entity
+            uid = event_data.value_dict.get("uid")
+            url = mlrun.utils.helpers.get_ui_url(alert.project, uid)
+            overview_type = "Job overview"
+        else:  # MODEL entity
+            model_name = event_data.value_dict.get("model")
+            model_endpoint_id = event_data.value_dict.get("model_endpoint_id")
+            url = mlrun.utils.helpers.get_model_endpoint_url(
+                alert.project, model_name, model_endpoint_id
+            )
+            overview_type = "Model endpoint"
+
+        return overview_type, url

mlrun/utils/notifications/notification/slack.py

@@ -153,20 +153,7 @@ class SlackNotification(NotificationBase):
             data_text = "\n".join(data_lines)
             line.append(self._get_slack_row(f"*Event data:*\n{data_text}"))
 
-        if (
-            event_data.entity.kind == mlrun.common.schemas.alert.EventEntityKind.JOB
-        ):  # JOB entity
-            uid = event_data.value_dict.get("uid")
-            url = mlrun.utils.helpers.get_ui_url(alert.project, uid)
-            overview_type = "Job overview"
-        else:  # MODEL entity
-            model_name = event_data.value_dict.get("model")
-            model_endpoint_id = event_data.value_dict.get("model_endpoint_id")
-            url = mlrun.utils.helpers.get_model_endpoint_url(
-                alert.project, model_name, model_endpoint_id
-            )
-            overview_type = "Model endpoint"
-
+        overview_type, url = self._get_overview_type_and_url(alert, event_data)
         line.append(self._get_slack_row(f"*Overview:*\n<{url}|*{overview_type}*>"))
 
         return line

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "0cbcbb9417af7ac58072ad981672602728fee57e",
-  "version": "1.7.0-rc22"
+  "git_commit": "e2058b393890ec83e5a2a1ed99b4dfbc7d806b4e",
+  "version": "1.7.0-rc23"
 }

{mlrun-1.7.0rc22.dist-info → mlrun-1.7.0rc23.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mlrun
-Version: 1.7.0rc22
+Version: 1.7.0rc23
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv

{mlrun-1.7.0rc22.dist-info → mlrun-1.7.0rc23.dist-info}/RECORD

@@ -1,7 +1,7 @@
 mlrun/__init__.py,sha256=y08M1JcKXy5-9_5WaI9fn5aV5BxIQ5QkbduJK0OxWbA,7470
 mlrun/__main__.py,sha256=F65N1MUdAn5hO4qFuJ1v5M3XSCLHUKv7C010toZd-P4,45852
 mlrun/config.py,sha256=sG_gjNcQ8UPoa9-nTfeT2UMLcZhbezcojs4RMHN4Xvk,65499
-mlrun/errors.py,sha256=53oT_uQliD-CEe7jxJZMFlNOT86zCTYBl802MZYluaE,7395
+mlrun/errors.py,sha256=Y26Czt93m5HCSPd6CyrHFiWO4R7spiTrOnk4FqaUUxE,7345
 mlrun/execution.py,sha256=ZSk61dXqnEmxnUYxaAjtSzobMlD1yF9VaW56KLmaiUs,41717
 mlrun/features.py,sha256=m17K_3l9Jktwb9dOwlHLTAPTlemsWrRF7dJhXUX0iJU,15429
 mlrun/k8s_utils.py,sha256=WdUajadvAhTR7sAMQdwFqKeJMimuTyqm02VdwK1A4xU,7023
@@ -35,7 +35,7 @@ mlrun/common/formatters/project.py,sha256=rdGf7fq_CfwFwd8iKWl8sW-tqTJilK3gJtV5oL
 mlrun/common/model_monitoring/__init__.py,sha256=x0EMEvxVjHsm858J1t6IEA9dtKTdFpJ9sKhss10ld8A,721
 mlrun/common/model_monitoring/helpers.py,sha256=1CpxIDQPumFnpUB1eqcvCpLlyPFVeW2sL6prM-N5A1A,4405
 mlrun/common/runtimes/constants.py,sha256=Rl0Sd8n_L7Imo-uF1LL9CJ5Szi0W1gUm36yrF8PXfSc,10989
-mlrun/common/schemas/__init__.py,sha256=afsTyAkc-KUh-1D1WBsUHQ99D3C2RoYg2dIn9kfOUyc,5134
+mlrun/common/schemas/__init__.py,sha256=Vg3S4JKA41y43S4nxCYfGOkpAw86sC9tn6BsD5qe_s4,5178
 mlrun/common/schemas/alert.py,sha256=yzXcmrhW8EHb-NxArVSlH0pRPrBLyqpMCTgMCDz4ExM,6644
 mlrun/common/schemas/api_gateway.py,sha256=CEvDsGHRH9okB52HeGx5HJXYxnDzPta1atwNECJq8NM,4797
 mlrun/common/schemas/artifact.py,sha256=hlKBK3L1Rg4Hv2XSu3u_gxdrsm0NbsaPjIS9cE_Equ4,3247
@@ -47,7 +47,7 @@ mlrun/common/schemas/common.py,sha256=00upzVLPN7O511Q87yt-fvRcDQFbXra4j0_lqMGg6r
 mlrun/common/schemas/constants.py,sha256=UnnhyLeF-SSjy8KaV5a-TBw4Ws675gueYGiP1fr5NfU,6476
 mlrun/common/schemas/datastore_profile.py,sha256=hJ8q54A8VZKsnOvSIjcllj4MZ1bBhb_EmBgsqpwSF_Y,750
 mlrun/common/schemas/events.py,sha256=ROHJLo_fqYjc96pek7yhAUPpPRIuAR76lwxvNz8LIr8,1026
-mlrun/common/schemas/feature_store.py,sha256=577OHVRZbV3kqYei4drVdTmCcGLJU8UXSGEpfo9FTQk,3717
+mlrun/common/schemas/feature_store.py,sha256=Y_83p3tQmlIj-GiymkDbm056I9DHeocSqL_kO7TXza8,5189
 mlrun/common/schemas/frontend_spec.py,sha256=CNPq3YV0U1jCbCMbb84_Oid2Snow5EXYt1F5wsuhgD8,2454
 mlrun/common/schemas/function.py,sha256=Khd5Jd6ept1nHWkW1WdIy1u8iK4O8_Ddf3a7cv3Hf1I,4652
 mlrun/common/schemas/http.py,sha256=1PtYFhF6sqLSBRcuPMtYcUGmroBhaleqLmYidSdL9LM,705
@@ -100,10 +100,10 @@ mlrun/datastore/wasbfs/__init__.py,sha256=s5Ul-0kAhYqFjKDR2X0O2vDGDbLQQduElb32Ev
 mlrun/datastore/wasbfs/fs.py,sha256=MnSj7Q4OKA2L55ihCmUnj2t3GA3B77oLMdAw-yxvN9w,6151
 mlrun/db/__init__.py,sha256=WqJ4x8lqJ7ZoKbhEyFqkYADd9P6E3citckx9e9ZLcIU,1163
 mlrun/db/auth_utils.py,sha256=hpg8D2r82oN0BWabuWN04BTNZ7jYMAF242YSUpK7LFM,5211
-mlrun/db/base.py,sha256=bPlaTiiDHs4oOMQ0LJ2CEl3HyDDr82fBbawQtP6jQ0I,22366
+mlrun/db/base.py,sha256=g1x5cIxuLjHocmVIsbMmUgDseS0nNfitl_aN9wixH4M,22404
 mlrun/db/factory.py,sha256=ibIrE5QkIIyzDU1FXKrfbc31cZiRLYKDZb8dqCpQwyU,2397
-mlrun/db/httpdb.py,sha256=nZbGcoxZ_vpo1rzBsw6GioX43HbPk7jIKLH_tOelCX8,179494
-mlrun/db/nopdb.py,sha256=deB-tk_r3boAwuVKD9EtV9DDF5EXnaNO_y--Xe7b5eg,19638
+mlrun/db/httpdb.py,sha256=52rjQ0e0MnZHEegwcmwjsCm9vx7dbMNcMlckwB1YWWQ,179816
+mlrun/db/nopdb.py,sha256=fO34H7vxiy2tmSu37EQ8lxm6OW_yz8AwUEtTJjANblo,19676
 mlrun/feature_store/__init__.py,sha256=FhHRc8NdqL_HWpCs7A8dKruxJS5wEm55Gs3dcgBiRUg,1522
 mlrun/feature_store/api.py,sha256=uYheyPkJOVCrz1jivvpGatgy_JBAq0It0XZqPpNVQkE,48699
 mlrun/feature_store/common.py,sha256=DKmoRk04NCS1gv7qZuEUa2-g8WsfR6IWjYctcrqKVlg,12853
@@ -119,7 +119,7 @@ mlrun/feature_store/retrieval/job.py,sha256=vm50yAqvaazuTGbCOgN_e1Ax8gh-d-qQN4Eb
 mlrun/feature_store/retrieval/local_merger.py,sha256=jM-8ta44PeNUc1cKMPs-TxrO9t8pXbwu_Tw8MZrLxUY,4513
 mlrun/feature_store/retrieval/spark_merger.py,sha256=I2KKEqSwao1AX1l6QqKRaXExUiry4P4ox-Vpc4AUNCg,11659
 mlrun/feature_store/retrieval/storey_merger.py,sha256=5YM0UPrLjGOobulHkowRO-1LuvFD2cm_0GxcpnTdu0I,6314
-mlrun/frameworks/__init__.py,sha256=Qp9pZRiUcU-drO_NlZnr1Z3RkUSjK44ET9drfGGfiFM,1017
+mlrun/frameworks/__init__.py,sha256=qRHe_nUfxpoLaSASAkIxcW6IyunMtxq5LXhjzZMO_1E,743
 mlrun/frameworks/parallel_coordinates.py,sha256=AJ3TuvffAC4_zN-RVcyTkq1T3lomDqgeNf7hVBmscEw,11517
 mlrun/frameworks/_common/__init__.py,sha256=7afutDCDVp999gyWSWQZMJRKGuW3VP3MFil8cobRsyg,962
 mlrun/frameworks/_common/artifacts_library.py,sha256=f0rtDRQI3BYT2ZvXR4drSXZPYPJG19Sbej-_ru-i0II,8497
@@ -216,12 +216,12 @@ mlrun/model_monitoring/controller.py,sha256=MQ4BF3vfJSyYZv6HuTuSLt_nqaflgBYyOSwC
 mlrun/model_monitoring/controller_handler.py,sha256=J9Y9ppLsQaxyYRl21165Rr7QuI9EM-mk-5veAqs4Bi0,1336
 mlrun/model_monitoring/evidently_application.py,sha256=iOc42IVjj8m6PDBmVcKIMWm46Bu0EdO9SDcH40Eqhyo,769
 mlrun/model_monitoring/features_drift_table.py,sha256=c6GpKtpOJbuT1u5uMWDL_S-6N4YPOmlktWMqPme3KFY,25308
-mlrun/model_monitoring/helpers.py,sha256=ROGQVZVoX7kgKI17dqIQSH-8r3t0K4oiwMRXPvfUGxE,11479
+mlrun/model_monitoring/helpers.py,sha256=RHelVFMoUPm9-KqOwF018ctZT0kHVQfipnigeK1k-uA,11645
 mlrun/model_monitoring/model_endpoint.py,sha256=7VX0cBATqLsA4sSinDzouf41ndxqh2mf5bO9BW0G5Z4,4017
 mlrun/model_monitoring/prometheus.py,sha256=cUR4y73GutJB_pA_VCBDl9YtK4PcIJp2wj2rnLVmYi4,7578
-mlrun/model_monitoring/stream_processing.py,sha256=TuFb1W-WYvynxU_kJl9h3xDn5Ir9b1MUwuynak-DK58,42467
+mlrun/model_monitoring/stream_processing.py,sha256=JRZP8raWUqG47z0yqES6LzHSEFYS5wcDxNXuJeFgjSA,42336
 mlrun/model_monitoring/tracking_policy.py,sha256=sQq956akAQpntkrJwIgFWcEq-JpyVcg0FxgNa4h3V70,5502
-mlrun/model_monitoring/writer.py,sha256=cAQ24HbtWGA8czzaemmjLT4WfDInJ7gPpkkIp9LePBY,10013
+mlrun/model_monitoring/writer.py,sha256=FJYkM_phr8mmiEcHHn0Qz-DCAYNCPw3IZiGEmO2IP4U,10068
 mlrun/model_monitoring/applications/__init__.py,sha256=i793GqYee01mRh_KD6GShvX7UbPBgdJDO4qf9Z3BXEQ,970
 mlrun/model_monitoring/applications/_application_steps.py,sha256=-g9jxIAFM5f22iJaUAQVlM8QRSv6KFT92I4WHmZe_f0,6028
 mlrun/model_monitoring/applications/base.py,sha256=buVKyghH4AB3chZ5py1vyMIFnTF-deY8YDf_fPC9BnQ,11307
@@ -230,19 +230,19 @@ mlrun/model_monitoring/applications/evidently_base.py,sha256=AE_eIz-GEYm3AZTrMCi
 mlrun/model_monitoring/applications/histogram_data_drift.py,sha256=HZmNg09SCjAKkIlKmJwqR7hr-8sXrwFEqXgJCitVbXc,13039
 mlrun/model_monitoring/applications/results.py,sha256=VVlu9Si7Tj2LNJzPQrp4_Qeyh9mxOVMu1Jwb5K2LfvY,3577
 mlrun/model_monitoring/db/__init__.py,sha256=6Ic-X3Fh9XLPYMytmevGNSs-Hii1rAjLLoFTSPwTguw,736
-mlrun/model_monitoring/db/stores/__init__.py,sha256=qesQRwo0tT4rDlQHMVliXFYiMRHZv2-rd-2JHvAQF-Y,4203
+mlrun/model_monitoring/db/stores/__init__.py,sha256=8lCEvKBJsOa2DPiqw4Y9D___nkfYM8T_-OuD5AAlKa4,4144
 mlrun/model_monitoring/db/stores/base/__init__.py,sha256=JufJETW3BXzPhFwbRa8dMf7BFGGZKceIWIMgr5x9n9c,599
-mlrun/model_monitoring/db/stores/base/store.py,sha256=ZzzXDRBnX0VJbvBinfgUdkXDyRQJwsjgPolgiyVLfwM,5858
+mlrun/model_monitoring/db/stores/base/store.py,sha256=x4hv2agOz3Ru2oL7PtuX6vV1IP-jWjP9VvlOMFrUIxk,5889
 mlrun/model_monitoring/db/stores/sqldb/__init__.py,sha256=6CsTXAxeLbbf8yfCADTaxmiavqwrLEdYFJ-qc5kgDAY,569
-mlrun/model_monitoring/db/stores/sqldb/sql_store.py,sha256=_p12LEpmGNVP6h5DTqAVwMVkwyfAbesBZigED-Lbduw,27500
+mlrun/model_monitoring/db/stores/sqldb/sql_store.py,sha256=j_IGyaOApwJHiTTpXe4a0Og6anTesmBkDX90IPX8boM,27523
 mlrun/model_monitoring/db/stores/sqldb/models/__init__.py,sha256=lCiGw9WKPtHAIgrtNS2jyvM5OZvZvogBh76iurNYblg,2453
 mlrun/model_monitoring/db/stores/sqldb/models/base.py,sha256=YKRWgWvi8yyzXvratpePqSZ3ZVjgfZ6Q_QhgaLUnAsE,5215
 mlrun/model_monitoring/db/stores/sqldb/models/mysql.py,sha256=86D1WX016u6TQdxhNDtlGn3QvJJHOQrXiw1hcSMq0Ao,2570
 mlrun/model_monitoring/db/stores/sqldb/models/sqlite.py,sha256=yJJZppbKj3PsOANS_DXAQFFHKX4cQcm6Pz2DoxRiXMk,1104
 mlrun/model_monitoring/db/stores/v3io_kv/__init__.py,sha256=6CsTXAxeLbbf8yfCADTaxmiavqwrLEdYFJ-qc5kgDAY,569
-mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py,sha256=o8P80yowPj4NRLonJ8rdSffot1OB-V6i3Ji1m_TWvzs,27399
-mlrun/model_monitoring/db/tsdb/__init__.py,sha256=NR895JSsEvNmINL223GLf8IbJ16b9Wn4XnxobDwivM8,3724
-mlrun/model_monitoring/db/tsdb/base.py,sha256=l5SNjO3btJSeglOCn4NHnckntAaP6hcZBVRhOty3KSQ,13122
+mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py,sha256=Ijm5W_P3C66inT4Y0jZ43mqKLS2TS2slbUMAM3eCTac,27230
+mlrun/model_monitoring/db/tsdb/__init__.py,sha256=0Da4WhlseFRTeUmfss_95DOsP2ijBaEtu4WxIWd5bsA,3736
+mlrun/model_monitoring/db/tsdb/base.py,sha256=sESs5U71a-iJKI-999sAloYH-mjOR3uSEQG7BxRs6No,13134
 mlrun/model_monitoring/db/tsdb/helpers.py,sha256=0oUXc4aUkYtP2SGP6jTb3uPPKImIUsVsrb9otX9a7O4,1189
 mlrun/model_monitoring/db/tsdb/tdengine/__init__.py,sha256=vgBdsKaXUURKqIf3M0y4sRatmSVA4CQiJs7J5dcVBkQ,620
 mlrun/model_monitoring/db/tsdb/tdengine/schemas.py,sha256=94u886UtyK40YNtdOX8WiJUImDytygdaqIzFwo_ExzI,8881
@@ -250,15 +250,15 @@ mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py,sha256=x1cWM2ystgh
 mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py,sha256=oplt9s-C-OGa__V456nkHwvyBe5YHxcuIJcYV9GFQHY,15521
 mlrun/model_monitoring/db/tsdb/v3io/__init__.py,sha256=aL3bfmQsUQ-sbvKGdNihFj8gLCK3mSys0qDcXtYOwgc,616
 mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py,sha256=qbiyBzrdWLJAKLmJV4K8jUxsAMbKGZ1vip7WNfRcpXM,4764
-mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py,sha256=L6XXXr4Mkfad3iTwDyfKScdeuEamL-xtTjoKSwefi7w,26194
+mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py,sha256=nlINlf-nHpUWzYnCyFVdYyPtGQUmImu2_ocve9ubt4o,26187
 mlrun/model_monitoring/metrics/__init__.py,sha256=6CsTXAxeLbbf8yfCADTaxmiavqwrLEdYFJ-qc5kgDAY,569
 mlrun/model_monitoring/metrics/histogram_distance.py,sha256=E9_WIl2vd6qNvoHVHoFcnuQk3ekbFWOdi8aU7sHrfk4,4724
-mlrun/package/__init__.py,sha256=_h308qe3NijbH2xymZ_G6XIJSGAa3OX24Je76gP855Q,7941
+mlrun/package/__init__.py,sha256=uWILzN42bcq5vFRk6ptxEmn1I5uBWAnhaJr7e4H834w,7082
 mlrun/package/context_handler.py,sha256=Z8v7cXAZXa5l3Tgg6IiEVm74Qbp5cOxx30jvkAY3dwo,14589
 mlrun/package/errors.py,sha256=LKF8SSaRIdbkB7JQz6b9U4mZV42Ebnf6ZHu4wKuWqK4,1204
 mlrun/package/packager.py,sha256=xE7U1njB2RXhmiA0kCSmA4i5j84Dd7Bt-H4Fk5OcVLk,15064
 mlrun/package/packagers_manager.py,sha256=g4XuqpKJGrGKYrA38FXZd9gquDv8KUcW1eXA-DesaMA,37161
-mlrun/package/packagers/__init__.py,sha256=Mq7-KCjDFYPGcDByAROYT9dXHZx9-QDmeqCepqrwFm0,1039
+mlrun/package/packagers/__init__.py,sha256=rpxpuATMoxCMgHDaVamm0uwocy71e0CSXm85Q5X9tkU,769
 mlrun/package/packagers/default_packager.py,sha256=QaZyxm03fRTJy5OGBeyVvSpEqnWj3-hSQVbsCjlTpLM,26625
 mlrun/package/packagers/numpy_packagers.py,sha256=k7Vke41LOp1ExbXCKf4FyahBIDlBqSiYrGPMeH0yI7M,25602
 mlrun/package/packagers/pandas_packagers.py,sha256=KPOZj1yiHxV2b1iah4hlwoNQP4JKzt95Fe9Tn9OUPs8,35761
@@ -275,7 +275,7 @@ mlrun/platforms/iguazio.py,sha256=1h5BpdAEQJBg2vIt7ySjUADU0ip5OkaMYr0_VREi9ys,13
 mlrun/projects/__init__.py,sha256=Lv5rfxyXJrw6WGOWJKhBz66M6t3_zsNMCfUD6waPwx4,1153
 mlrun/projects/operations.py,sha256=NEN4PmSvLO9QMwSG4TncmBgTKC9wJp7hGo5lA7OYN_Q,19199
 mlrun/projects/pipelines.py,sha256=c9HhMtXk-6kmTCiY-f0Cmd3GWgL_fBFE6HXp2lrhRtE,40009
-mlrun/projects/project.py,sha256=umxA0OnsZYtVNu2lmXuXCMMA-0s6iNpsa7uv3jT7THM,181355
+mlrun/projects/project.py,sha256=cG8yEmViov3c6pqM0K3l8Bbs4GDMVD8X_aKw3S0Jy9k,181672
 mlrun/runtimes/__init__.py,sha256=0-tYDkew-Cr4DM-wztvMbzDA5xq385Jjo-GrtO_84Sc,8741
 mlrun/runtimes/base.py,sha256=SvaKfWtAKFhrX2RW7wy3B76OUAg9XdTDaJPpbCEhsdY,37323
 mlrun/runtimes/daskjob.py,sha256=_3jQIEroNxG587ZJ0cW5nVJVBb1IcOECor_bkgZHtMk,19194
@@ -298,9 +298,9 @@ mlrun/runtimes/nuclio/__init__.py,sha256=gx1kizzKv8pGT5TNloN1js1hdbxqDw3rM90sLVY
 mlrun/runtimes/nuclio/api_gateway.py,sha256=_tkqyH63cyIVcjZEiSP--bh_mOAOGGclYY7wyQ5q_JA,24683
 mlrun/runtimes/nuclio/function.py,sha256=dZoWA4VX4izKcAP3_3F14WcICwtVYuhD3d0u3Qdu9Tg,49408
 mlrun/runtimes/nuclio/nuclio.py,sha256=sLK8KdGO1LbftlL3HqPZlFOFTAAuxJACZCVl1c0Ha6E,2942
-mlrun/runtimes/nuclio/serving.py,sha256=H3bSI33FmfOBkL99ZU6_xKbFx4qKdyQVdpIwwfhK5qo,29649
+mlrun/runtimes/nuclio/serving.py,sha256=ucSDp2YbaKEFpuCxCTEPwOEPngPkuP-TRviHJ9J4D60,29866
 mlrun/runtimes/nuclio/application/__init__.py,sha256=rRs5vasy_G9IyoTpYIjYDafGoL6ifFBKgBtsXn31Atw,614
-mlrun/runtimes/nuclio/application/application.py,sha256=f1GwB5IeavDYls1vHeEqkaOTMkINWeffNICAVRR4htA,19193
+mlrun/runtimes/nuclio/application/application.py,sha256=8VVKKUeviZPpC_j3WBm_oiJQis4TkmxZGnWfsHGlXR8,19062
 mlrun/runtimes/nuclio/application/reverse_proxy.go,sha256=JIIYae6bXzCLf3jXuu49KWPQYoXr_FDQ2Rbo1OWKAd0,3150
 mlrun/runtimes/sparkjob/__init__.py,sha256=_KPvk0qefeLtHO6lxQE_AMOGiMTG_OT48eRCE4Z2ldw,709
 mlrun/runtimes/sparkjob/spark3job.py,sha256=1bNRy72Migrh_ZASQOx7UlSZTbB-xpNc76sz4kfc9UM,41191
@@ -313,7 +313,7 @@ mlrun/serving/serving_wrapper.py,sha256=R670-S6PX_d5ER6jiHtRvacuPyFzQH0mEf2K0sBI
 mlrun/serving/states.py,sha256=n3RPtzwqfQB1o4H80AoVsP5exL3L3i39ONs-CorWGyM,58539
 mlrun/serving/utils.py,sha256=lej7XcUPX1MmHkEOi_0KZRGSpfbmpnE0GK_Sn4zLkHY,4025
 mlrun/serving/v1_serving.py,sha256=by4myxlnwyZ0ijQ5fURilGCK1sUpdQL2Il1VR3Xqpxg,11805
-mlrun/serving/v2_serving.py,sha256=l2J-kWRPf5vH6T-i96WJwmmzRF2QrWAjepmLVnq04rg,23863
+mlrun/serving/v2_serving.py,sha256=wJmDZc-YEof0vXmYHM5t8aKSaWuzUbjVtWYuVyl1hPg,24226
 mlrun/track/__init__.py,sha256=LWRUHJt8JyFW17FyNPOVyWd-NXTf1iptzsK9KFj5fuY,765
 mlrun/track/tracker.py,sha256=hSi9sMxB7hhZalt6Q8GXDnK4UoCbXHzKTrpUPC9hZv4,3555
 mlrun/track/tracker_manager.py,sha256=IYBl99I62IC6VCCmG1yt6JoHNOQXa53C4DURJ2sWgio,5726
@@ -336,18 +336,18 @@ mlrun/utils/vault.py,sha256=xUiKL17dCXjwQJ33YRzQj0oadUXATlFWPzKKYAESoQk,10447
 mlrun/utils/notifications/__init__.py,sha256=eUzQDBxSQmMZASRY-YAnYS6tL5801P0wEjycp3Dvoe0,990
 mlrun/utils/notifications/notification_pusher.py,sha256=v-bB05rkDW04f52Wsh_1-UD77M2FNFukmQO9wglVIRg,26918
 mlrun/utils/notifications/notification/__init__.py,sha256=2in3F2q8gtYDiDoQ4i9BIIE2I06OokT2EW49vs2krRA,2168
-mlrun/utils/notifications/notification/base.py,sha256=b0nncv0oV01wNeT-3upWQkcvyVVbBbJkrFgk6PMAusw,2788
+mlrun/utils/notifications/notification/base.py,sha256=Bv6tUEFlF27SdkzFr1aY9a0gYX7tIbxsHbI7vOSqTsU,4196
 mlrun/utils/notifications/notification/console.py,sha256=MAVk7v5PJ52vdGRv76YcEPixWgV0licBPWGpR01uR40,2643
 mlrun/utils/notifications/notification/git.py,sha256=ELZ-ZmbFDb39A0OUIhtvuSbqJoVfF_o_IOxMD5eBlv4,5351
 mlrun/utils/notifications/notification/ipython.py,sha256=ZtVL30B_Ha0VGoo4LxO-voT1U41IYwyytovv5X_LsI4,2066
-mlrun/utils/notifications/notification/slack.py,sha256=Vc6EHdnVAZe-p4ZWMvLc23YjMIDE3h2flf2b83ATVCA,7286
+mlrun/utils/notifications/notification/slack.py,sha256=EXhIDyhFkwCSzq8trX3TqGHh5ppFKzMxItxM0s0-ukM,6728
 mlrun/utils/notifications/notification/webhook.py,sha256=WgfxX1cpm8n2A-O08pwnsP4tzbxxv_vNUSnyXG4uKts,2752
 mlrun/utils/version/__init__.py,sha256=7kkrB7hEZ3cLXoWj1kPoDwo4MaswsI2JVOBpbKgPAgc,614
-mlrun/utils/version/version.json,sha256=YTYzIsYgfu2QIFU3klixQlisgpqRt5PED9zmOJONRXA,89
+mlrun/utils/version/version.json,sha256=0D0ZRgXjQ9n0Ol6wTHUIJk9ckFYse84wtjP_dbJwvTQ,89
 mlrun/utils/version/version.py,sha256=eEW0tqIAkU9Xifxv8Z9_qsYnNhn3YH7NRAfM-pPLt1g,1878
-mlrun-1.7.0rc22.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-mlrun-1.7.0rc22.dist-info/METADATA,sha256=1YFwgv7LW2rwwHez3LXjihC5KrpjncBo3J5DUb6iDx8,19237
-mlrun-1.7.0rc22.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-mlrun-1.7.0rc22.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
-mlrun-1.7.0rc22.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
-mlrun-1.7.0rc22.dist-info/RECORD,,
+mlrun-1.7.0rc23.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+mlrun-1.7.0rc23.dist-info/METADATA,sha256=ZABqKbSkVYEvJvZ8jSwhecmgFxaROzUntGRUUov5zbc,19237
+mlrun-1.7.0rc23.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+mlrun-1.7.0rc23.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
+mlrun-1.7.0rc23.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
+mlrun-1.7.0rc23.dist-info/RECORD,,