mlrun 1.8.0rc43__py3-none-any.whl → 1.8.0rc45__py3-none-any.whl

mlrun/common/schemas/model_monitoring/constants.py

@@ -281,6 +281,7 @@ class FileTargetKind:
     MONITORING_APPLICATION = "monitoring_application"
     ERRORS = "errors"
     STATS = "stats"
+    LAST_REQUEST = "last_request"
 
 
 class ModelMonitoringMode(StrEnum):

mlrun/config.py

@@ -549,6 +549,10 @@ default_config = {
         },
     },
     "model_endpoint_monitoring": {
+        # Scaling Rule
+        # The fundamental scaling rule to maintain is: Shards/Partitions = Replicas * Workers
+        # In other words, the number of shards (V3IO) or partitions (Kafka) must be equal to the
+        # total number of worker processes across all pods.
         "serving_stream": {
             "v3io": {
                 "shard_count": 2,
@@ -822,6 +826,8 @@ default_config = {
         # maximum allowed alert config cache size in alert's CRUD
         # for the best performance, it is recommended to set this value to the maximum number of alerts
         "max_allowed_cache_size": 20000,
+        # default limit for listing alert configs
+        "default_list_alert_configs_limit": 2000,
     },
     "auth_with_client_id": {
         "enabled": False,

mlrun/db/base.py

@@ -889,7 +889,9 @@ class RunDBInterface(ABC):
         pass
 
     @abstractmethod
-    def list_alerts_configs(self, project=""):
+    def list_alerts_configs(
+        self, project="", limit: Optional[int] = None, offset: Optional[int] = None
+    ):
         pass
 
     @abstractmethod
@@ -1105,12 +1107,6 @@ class RunDBInterface(ABC):
     ) -> bool:
         pass
 
-    @abstractmethod
-    def deploy_histogram_data_drift_app(
-        self, project: str, image: str = "mlrun/mlrun"
-    ) -> None:
-        pass
-
     @abstractmethod
     def set_model_monitoring_credentials(
         self,

mlrun/db/httpdb.py

@@ -4080,21 +4080,6 @@ class HTTPRunDB(RunDBInterface):
                     deletion_failed = True
         return not deletion_failed
 
-    def deploy_histogram_data_drift_app(
-        self, project: str, image: str = "mlrun/mlrun"
-    ) -> None:
-        """
-        Deploy the histogram data drift application.
-
-        :param project: Project name.
-        :param image:   The image on which the application will run.
-        """
-        self.api_call(
-            method=mlrun.common.types.HTTPMethod.PUT,
-            path=f"projects/{project}/model-monitoring/histogram-data-drift-app",
-            params={"image": image},
-        )
-
     def set_model_monitoring_credentials(
         self,
         project: str,
@@ -4818,20 +4803,33 @@ class HTTPRunDB(RunDBInterface):
         response = self.api_call("GET", endpoint_path, error_message)
         return AlertConfig.from_dict(response.json())
 
-    def list_alerts_configs(self, project="") -> list[AlertConfig]:
+    def list_alerts_configs(
+        self, project="", limit: Optional[int] = None, offset: Optional[int] = None
+    ) -> list[AlertConfig]:
         """
         Retrieve list of alerts of a project.
 
         :param project: The project name.
+        :param limit: The maximum number of alerts to return.
+            Defaults to `mlconf.alerts.default_list_alert_configs_limit` if not provided.
+        :param offset: The number of alerts to skip.
 
         :returns: All the alerts objects of the project.
         """
         project = project or config.default_project
         endpoint_path = f"projects/{project}/alerts"
         error_message = f"get alerts {project}/alerts"
-        response = self.api_call("GET", endpoint_path, error_message).json()
+        params = {}
+        # TODO: Deprecate limit and offset when pagination is implemented
+        if limit:
+            params["page-size"] = limit
+        if offset:
+            params["offset"] = offset
+        response = self.api_call(
+            "GET", endpoint_path, error_message, params=params
+        ).json()
         results = []
-        for item in response:
+        for item in response.get("alerts", []):
             results.append(AlertConfig(**item))
         return results
 

mlrun/db/nopdb.py

@@ -883,11 +883,6 @@ class NopDB(RunDBInterface):
     ) -> bool:
         pass
 
-    def deploy_histogram_data_drift_app(
-        self, project: str, image: str = "mlrun/mlrun"
-    ) -> None:
-        pass
-
     def set_model_monitoring_credentials(
         self,
         project: str,

mlrun/feature_store/__init__.py

@@ -43,6 +43,8 @@ from .feature_set import FeatureSet
 from .feature_vector import (
     FeatureVector,
     FixedWindowType,
+)
+from .feature_vector_utils import (
     JoinGraph,
     OfflineVectorResponse,
     OnlineVectorService,

mlrun/feature_store/api.py

@@ -47,159 +47,20 @@ from .common import (
     get_feature_vector_by_uri,
     verify_feature_set_exists,
     verify_feature_set_permissions,
-    verify_feature_vector_permissions,
 )
 from .feature_set import FeatureSet
-from .feature_vector import (
-    FeatureVector,
-    FixedWindowType,
-    OfflineVectorResponse,
-    OnlineVectorService,
-)
 from .ingestion import (
     context_to_ingestion_params,
     init_featureset_graph,
     run_ingestion_job,
     run_spark_graph,
 )
-from .retrieval import RemoteVectorResponse, get_merger, run_merge_job
 
 _v3iofs = None
 spark_transform_handler = "transform"
 _TRANS_TABLE = str.maketrans({" ": "_", "(": "", ")": ""})
 
 
-def _features_to_vector_and_check_permissions(features, update_stats):
-    if isinstance(features, str):
-        vector = get_feature_vector_by_uri(features, update=update_stats)
-    elif isinstance(features, FeatureVector):
-        vector = features
-        if not vector.metadata.name:
-            raise mlrun.errors.MLRunInvalidArgumentError(
-                "feature vector name must be specified"
-            )
-        verify_feature_vector_permissions(
-            vector, mlrun.common.schemas.AuthorizationAction.update
-        )
-
-        vector.save()
-    else:
-        raise mlrun.errors.MLRunInvalidArgumentError(
-            f"illegal features value/type ({type(features)})"
-        )
-    return vector
-
-
-def _get_offline_features(
-    feature_vector: Union[str, FeatureVector],
-    entity_rows=None,
-    entity_timestamp_column: Optional[str] = None,
-    target: DataTargetBase = None,
-    run_config: RunConfig = None,
-    drop_columns: Optional[list[str]] = None,
-    start_time: Optional[Union[str, datetime]] = None,
-    end_time: Optional[Union[str, datetime]] = None,
-    with_indexes: bool = False,
-    update_stats: bool = False,
-    engine: Optional[str] = None,
-    engine_args: Optional[dict] = None,
-    query: Optional[str] = None,
-    order_by: Optional[Union[str, list[str]]] = None,
-    spark_service: Optional[str] = None,
-    timestamp_for_filtering: Optional[Union[str, dict[str, str]]] = None,
-    additional_filters=None,
-) -> Union[OfflineVectorResponse, RemoteVectorResponse]:
-    if entity_rows is None and entity_timestamp_column is not None:
-        raise mlrun.errors.MLRunInvalidArgumentError(
-            "entity_timestamp_column param "
-            "can not be specified without entity_rows param"
-        )
-    if isinstance(target, BaseStoreTarget) and not target.support_pandas:
-        raise mlrun.errors.MLRunInvalidArgumentError(
-            f"get_offline_features does not support targets that do not support pandas engine."
-            f" Target kind: {target.kind}"
-        )
-
-    if isinstance(feature_vector, FeatureVector):
-        update_stats = True
-
-    feature_vector = _features_to_vector_and_check_permissions(
-        feature_vector, update_stats
-    )
-
-    entity_timestamp_column = (
-        entity_timestamp_column or feature_vector.spec.timestamp_field
-    )
-
-    merger_engine = get_merger(engine)
-
-    if run_config and not run_config.local:
-        return run_merge_job(
-            feature_vector,
-            target,
-            merger_engine,
-            engine,
-            engine_args,
-            spark_service,
-            entity_rows,
-            entity_timestamp_column=entity_timestamp_column,
-            run_config=run_config,
-            drop_columns=drop_columns,
-            with_indexes=with_indexes,
-            query=query,
-            order_by=order_by,
-            start_time=start_time,
-            end_time=end_time,
-            timestamp_for_filtering=timestamp_for_filtering,
-            additional_filters=additional_filters,
-        )
-
-    merger = merger_engine(feature_vector, **(engine_args or {}))
-    return merger.start(
-        entity_rows,
-        entity_timestamp_column,
-        target=target,
-        drop_columns=drop_columns,
-        start_time=start_time,
-        end_time=end_time,
-        timestamp_for_filtering=timestamp_for_filtering,
-        with_indexes=with_indexes,
-        update_stats=update_stats,
-        query=query,
-        order_by=order_by,
-        additional_filters=additional_filters,
-    )
-
-
-def _get_online_feature_service(
-    feature_vector: Union[str, FeatureVector],
-    run_config: RunConfig = None,
-    fixed_window_type: FixedWindowType = FixedWindowType.LastClosedWindow,
-    impute_policy: Optional[dict] = None,
-    update_stats: bool = False,
-    entity_keys: Optional[list[str]] = None,
-) -> OnlineVectorService:
-    if isinstance(feature_vector, FeatureVector):
-        update_stats = True
-    feature_vector = _features_to_vector_and_check_permissions(
-        feature_vector, update_stats
-    )
-
-    # Impute policies rely on statistics in many cases, so verifying that the fvec has stats in it
-    if impute_policy and not feature_vector.status.stats:
-        update_stats = True
-
-    engine_args = {"impute_policy": impute_policy}
-    merger_engine = get_merger("storey")
-    # todo: support remote service (using remote nuclio/mlrun function if run_config)
-
-    merger = merger_engine(feature_vector, **engine_args)
-
-    return merger.init_online_vector_service(
-        entity_keys, fixed_window_type, update_stats=update_stats
-    )
-
-
 def norm_column_name(name: str) -> str:
     """
     Remove parentheses () and replace whitespaces with an underscore _.