mlrun 1.8.0rc34__py3-none-any.whl → 1.8.0rc36__py3-none-any.whl

mlrun/feature_store/api.py

@@ -20,7 +20,6 @@ from datetime import datetime
 from typing import Any, Optional, Union
 
 import pandas as pd
-from deprecated import deprecated
 
 import mlrun
 import mlrun.errors
@@ -91,122 +90,6 @@ def _features_to_vector_and_check_permissions(features, update_stats):
     return vector
 
 
-@deprecated(
-    version="1.6.0",
-    reason="get_offline_features() will be removed in 1.8.0, please instead use "
-    "get_feature_vector('store://feature_vector_name').get_offline_features()",
-    category=FutureWarning,
-)
-def get_offline_features(
-    feature_vector: Union[str, FeatureVector],
-    entity_rows=None,
-    entity_timestamp_column: Optional[str] = None,
-    target: DataTargetBase = None,
-    run_config: RunConfig = None,
-    drop_columns: Optional[list[str]] = None,
-    start_time: Optional[Union[str, datetime]] = None,
-    end_time: Optional[Union[str, datetime]] = None,
-    with_indexes: bool = False,
-    update_stats: bool = False,
-    engine: Optional[str] = None,
-    engine_args: Optional[dict] = None,
-    query: Optional[str] = None,
-    order_by: Optional[Union[str, list[str]]] = None,
-    spark_service: Optional[str] = None,
-    timestamp_for_filtering: Optional[Union[str, dict[str, str]]] = None,
-    additional_filters: Optional[list] = None,
-):
-    """retrieve offline feature vector results
-
-    specify a feature vector object/uri and retrieve the desired features, their metadata
-    and statistics. returns :py:class:`~mlrun.feature_store.OfflineVectorResponse`,
-    results can be returned as a dataframe or written to a target
-
-    The start_time and end_time attributes allow filtering the data to a given time range, they accept
-    string values or pandas `Timestamp` objects, string values can also be relative, for example:
-    "now", "now - 1d2h", "now+5m", where a valid pandas Timedelta string follows the verb "now",
-    for time alignment you can use the verb "floor" e.g. "now -1d floor 1H" will align the time to the last hour
-    (the floor string is passed to pandas.Timestamp.floor(), can use D, H, T, S for day, hour, min, sec alignment).
-    Another option to filter the data is by the `query` argument - can be seen in the example.
-    example::
-
-        features = [
-            "stock-quotes.bid",
-            "stock-quotes.asks_sum_5h",
-            "stock-quotes.ask as mycol",
-            "stocks.*",
-        ]
-        vector = FeatureVector(features=features)
-        resp = get_offline_features(
-            vector,
-            entity_rows=trades,
-            entity_timestamp_column="time",
-            query="ticker in ['GOOG'] and bid>100",
-        )
-        print(resp.to_dataframe())
-        print(vector.get_stats_table())
-        resp.to_parquet("./out.parquet")
-
-    :param feature_vector:          feature vector uri or FeatureVector object. passing feature vector obj requires
-                                    update permissions
-    :param entity_rows:             dataframe with entity rows to join with
-    :param target:                  where to write the results to
-    :param drop_columns:            list of columns to drop from the final result
-    :param entity_timestamp_column: timestamp column name in the entity rows dataframe. can be specified
-                                    only if param entity_rows was specified.
-    :param run_config:              function and/or run configuration
-                                    see :py:class:`~mlrun.feature_store.RunConfig`
-    :param start_time:              datetime, low limit of time needed to be filtered. Optional.
-    :param end_time:                datetime, high limit of time needed to be filtered. Optional.
-    :param with_indexes:            Return vector with/without the entities and the timestamp_key of the feature sets
-                                    and with/without entity_timestamp_column and timestamp_for_filtering columns.
-                                    This property can be specified also in the feature vector spec
-                                    (feature_vector.spec.with_indexes)
-                                    (default False)
-    :param update_stats:            update features statistics from the requested feature sets on the vector.
-                                    (default False).
-    :param engine:                  processing engine kind ("local", "dask", or "spark")
-    :param engine_args:             kwargs for the processing engine
-    :param query:                   The query string used to filter rows on the output
-    :param spark_service:           Name of the spark service to be used (when using a remote-spark runtime)
-    :param order_by:                Name or list of names to order by. The name or the names in the list can be the
-                                    feature name or the alias of the feature you pass in the feature list.
-    :param timestamp_for_filtering: name of the column to filter by, can be str for all the feature sets or a
-                                    dictionary ({<feature set name>: <timestamp column name>, ...})
-                                    that indicates the timestamp column name for each feature set. Optional.
-                                    By default, the filter executes on the timestamp_key of each feature set.
-                                    Note: the time filtering is performed on each feature set before the
-                                    merge process using start_time and end_time params.
-    :param additional_filters: List of additional_filter conditions as tuples.
-                                Each tuple should be in the format (column_name, operator, value).
-                                Supported operators: "=", ">=", "<=", ">", "<".
-                                Example: [("Product", "=", "Computer")]
-                                For all supported filters, please see:
-                                https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetDataset.html
-
-
-    """
-    return _get_offline_features(
-        feature_vector,
-        entity_rows,
-        entity_timestamp_column,
-        target,
-        run_config,
-        drop_columns,
-        start_time,
-        end_time,
-        with_indexes,
-        update_stats,
-        engine,
-        engine_args,
-        query,
-        order_by,
-        spark_service,
-        timestamp_for_filtering,
-        additional_filters,
-    )
-
-
 def _get_offline_features(
     feature_vector: Union[str, FeatureVector],
     entity_rows=None,
@@ -288,93 +171,6 @@ def _get_offline_features(
     )
 
 
-@deprecated(
-    version="1.6.0",
-    reason="get_online_feature_service() will be removed in 1.8.0, please instead use "
-    "get_feature_vector('store://feature_vector_name').get_online_feature_service()",
-    category=FutureWarning,
-)
-def get_online_feature_service(
-    feature_vector: Union[str, FeatureVector],
-    run_config: RunConfig = None,
-    fixed_window_type: FixedWindowType = FixedWindowType.LastClosedWindow,
-    impute_policy: Optional[dict] = None,
-    update_stats: bool = False,
-    entity_keys: Optional[list[str]] = None,
-):
-    """initialize and return online feature vector service api,
-    returns :py:class:`~mlrun.feature_store.OnlineVectorService`
-
-    :**usage**:
-        There are two ways to use the function:
-
-        1. As context manager
-
-            Example::
-
-                with get_online_feature_service(vector_uri) as svc:
-                    resp = svc.get([{"ticker": "GOOG"}, {"ticker": "MSFT"}])
-                    print(resp)
-                    resp = svc.get([{"ticker": "AAPL"}], as_list=True)
-                    print(resp)
-
-            Example with imputing::
-
-                with get_online_feature_service(vector_uri, entity_keys=['id'],
-                                                impute_policy={"*": "$mean", "amount": 0)) as svc:
-                    resp = svc.get([{"id": "C123487"}])
-
-        2. as simple function, note that in that option you need to close the session.
-
-            Example::
-
-                svc = get_online_feature_service(vector_uri, entity_keys=["ticker"])
-                try:
-                    resp = svc.get([{"ticker": "GOOG"}, {"ticker": "MSFT"}])
-                    print(resp)
-                    resp = svc.get([{"ticker": "AAPL"}], as_list=True)
-                    print(resp)
-
-                finally:
-                    svc.close()
-
-            Example with imputing::
-
-                svc = get_online_feature_service(vector_uri, entity_keys=['id'],
-                                                 impute_policy={"*": "$mean", "amount": 0))
-                try:
-                    resp = svc.get([{"id": "C123487"}])
-                except Exception as e:
-                    handling exception...
-                finally:
-                    svc.close()
-
-    :param feature_vector:      feature vector uri or FeatureVector object. passing feature vector obj requires update
-                                permissions.
-    :param run_config:          function and/or run configuration for remote jobs/services
-    :param impute_policy:       a dict with `impute_policy` per feature, the dict key is the feature name and the dict
-                                value indicate which value will be used in case the feature is NaN/empty, the replaced
-                                value can be fixed number for constants or $mean, $max, $min, $std, $count
-                                for statistical
-                                values. "*" is used to specify the default for all features, example: `{"*": "$mean"}`
-    :param fixed_window_type:   determines how to query the fixed window values which were previously inserted by ingest
-    :param update_stats:        update features statistics from the requested feature sets on the vector.
-                                Default: False.
-    :param entity_keys:         Entity list of the first feature_set in the vector.
-                                The indexes that are used to query the online service.
-    :return:                    Initialize the `OnlineVectorService`.
-                                Will be used in subclasses where `support_online=True`.
-    """
-    return _get_online_feature_service(
-        feature_vector,
-        run_config,
-        fixed_window_type,
-        impute_policy,
-        update_stats,
-        entity_keys,
-    )
-
-
 def _get_online_feature_service(
     feature_vector: Union[str, FeatureVector],
     run_config: RunConfig = None,
@@ -448,86 +244,6 @@ def _get_namespace(run_config: RunConfig) -> dict[str, Any]:
         return get_caller_globals()
 
 
-def ingest(
-    featureset: Union[FeatureSet, str] = None,
-    source=None,
-    targets: Optional[list[DataTargetBase]] = None,
-    namespace=None,
-    return_df: bool = True,
-    infer_options: InferOptions = InferOptions.default(),
-    run_config: RunConfig = None,
-    mlrun_context=None,
-    spark_context=None,
-    overwrite=None,
-) -> Optional[pd.DataFrame]:
-    """Read local DataFrame, file, URL, or source into the feature store
-    Ingest reads from the source, run the graph transformations, infers  metadata and stats
-    and writes the results to the default of specified targets
-
-    when targets are not specified data is stored in the configured default targets
-    (will usually be NoSQL for real-time and Parquet for offline).
-
-    the `run_config` parameter allow specifying the function and job configuration,
-    see: :py:class:`~mlrun.feature_store.RunConfig`
-
-    example::
-
-        stocks_set = FeatureSet("stocks", entities=[Entity("ticker")])
-        stocks = pd.read_csv("stocks.csv")
-        df = ingest(stocks_set, stocks, infer_options=fstore.InferOptions.default())
-
-        # for running as remote job
-        config = RunConfig(image="mlrun/mlrun")
-        df = ingest(stocks_set, stocks, run_config=config)
-
-        # specify source and targets
-        source = CSVSource("mycsv", path="measurements.csv")
-        targets = [CSVTarget("mycsv", path="./mycsv.csv")]
-        ingest(measurements, source, targets)
-
-    :param featureset:    feature set object or featureset.uri. (uri must be of a feature set that is in the DB,
-                          call `.save()` if it's not)
-    :param source:        source dataframe or other sources (e.g. parquet source see:
-                          :py:class:`~mlrun.datastore.ParquetSource` and other classes in mlrun.datastore with suffix
-                          Source)
-    :param targets:       optional list of data target objects
-    :param namespace:     namespace or module containing graph classes
-    :param return_df:     indicate if to return a dataframe with the graph results
-    :param infer_options: schema (for discovery of entities, features in featureset), index, stats,
-                          histogram and preview infer options (:py:class:`~mlrun.feature_store.InferOptions`)
-    :param run_config:    function and/or run configuration for remote jobs,
-                          see :py:class:`~mlrun.feature_store.RunConfig`
-    :param mlrun_context: mlrun context (when running as a job), for internal use !
-    :param spark_context: local spark session for spark ingestion, example for creating the spark context:
-                          `spark = SparkSession.builder.appName("Spark function").getOrCreate()`
-                          For remote spark ingestion, this should contain the remote spark service name
-    :param overwrite:     delete the targets' data prior to ingestion
-                          (default: True for non scheduled ingest - deletes the targets that are about to be ingested.
-                          False for scheduled ingest - does not delete the target)
-    :return:              if return_df is True, a dataframe will be returned based on the graph
-    """
-    if mlrun_context is None:
-        deprecated(
-            version="1.6.0",
-            reason="Calling 'ingest' with mlrun_context=None is deprecated and will be removed in 1.8.0,\
-            use 'FeatureSet.ingest()' instead",
-            category=FutureWarning,
-        )
-
-    return _ingest(
-        featureset,
-        source,
-        targets,
-        namespace,
-        return_df,
-        infer_options,
-        run_config,
-        mlrun_context,
-        spark_context,
-        overwrite,
-    )
-
-
 def _ingest(
     featureset: Union[FeatureSet, str] = None,
     source=None,
@@ -776,53 +492,6 @@ def _ingest(
         return df
 
 
-@deprecated(
-    version="1.6.0",
-    reason="'preview' will be removed in 1.8.0, use 'FeatureSet.preview()' instead",
-    category=FutureWarning,
-)
-def preview(
-    featureset: FeatureSet,
-    source,
-    entity_columns: Optional[list] = None,
-    namespace=None,
-    options: InferOptions = None,
-    verbose: bool = False,
-    sample_size: Optional[int] = None,
-) -> pd.DataFrame:
-    """run the ingestion pipeline with local DataFrame/file data and infer features schema and stats
-
-    example::
-
-        quotes_set = FeatureSet("stock-quotes", entities=[Entity("ticker")])
-        quotes_set.add_aggregation("ask", ["sum", "max"], ["1h", "5h"], "10m")
-        quotes_set.add_aggregation("bid", ["min", "max"], ["1h"], "10m")
-        df = preview(
-            quotes_set,
-            quotes_df,
-            entity_columns=["ticker"],
-        )
-
-    :param featureset:     feature set object or uri
-    :param source:         source dataframe or csv/parquet file path
-    :param entity_columns: list of entity (index) column names
-    :param namespace:      namespace or module containing graph classes
-    :param options:        schema (for discovery of entities, features in featureset), index, stats,
-                           histogram and preview infer options (:py:class:`~mlrun.feature_store.InferOptions`)
-    :param verbose:        verbose log
-    :param sample_size:    num of rows to sample from the dataset (for large datasets)
-    """
-    return _preview(
-        featureset,
-        source,
-        entity_columns,
-        namespace,
-        options,
-        verbose,
-        sample_size,
-    )
-
-
 def _preview(
     featureset: FeatureSet,
     source,
@@ -912,55 +581,6 @@ def _run_ingestion_job(
     return run_ingestion_job(name, featureset, run_config, source.schedule)
 
 
-@deprecated(
-    version="1.6.0",
-    reason="'deploy_ingestion_service_v2' will be removed in 1.8.0, "
-    "use 'FeatureSet.deploy_ingestion_service()' instead",
-    category=FutureWarning,
-)
-def deploy_ingestion_service_v2(
-    featureset: Union[FeatureSet, str],
-    source: DataSource = None,
-    targets: Optional[list[DataTargetBase]] = None,
-    name: Optional[str] = None,
-    run_config: RunConfig = None,
-    verbose=False,
-) -> tuple[str, BaseRuntime]:
-    """Start real-time ingestion service using nuclio function
-
-    Deploy a real-time function implementing feature ingestion pipeline
-    the source maps to Nuclio event triggers (http, kafka, v3io stream, etc.)
-
-    the `run_config` parameter allow specifying the function and job configuration,
-    see: :py:class:`~mlrun.feature_store.RunConfig`
-
-    example::
-
-        source = HTTPSource()
-        func = mlrun.code_to_function("ingest", kind="serving").apply(mount_v3io())
-        config = RunConfig(function=func)
-        deploy_ingestion_service_v2(my_set, source, run_config=config)
-
-    :param featureset:    feature set object or uri
-    :param source:        data source object describing the online or offline source
-    :param targets:       list of data target objects
-    :param name:          name for the job/function
-    :param run_config:    service runtime configuration (function object/uri, resources, etc..)
-    :param verbose:       verbose log
-
-    :return: URL to access the deployed ingestion service, and the function that was deployed (which will
-             differ from the function passed in via the run_config parameter).
-    """
-    return _deploy_ingestion_service_v2(
-        featureset,
-        source,
-        targets,
-        name,
-        run_config,
-        verbose,
-    )
-
-
 def _deploy_ingestion_service_v2(
     featureset: Union[FeatureSet, str],
     source: DataSource = None,

mlrun/model.py

@@ -430,6 +430,19 @@ class ObjectList:
         self._children[child_obj.name] = child_obj
         return child_obj
 
+    def move_to_end(self, child, last=True):
+        self._children.move_to_end(child, last)
+
+    def update_list(self, object_list: "ObjectList", push_at_start: bool = False):
+        if push_at_start:
+            self._children = OrderedDict(
+                list(object_list._children.items()) + list(self._children.items())
+            )
+        else:
+            self._children = OrderedDict(
+                list(self._children.items()) + list(object_list._children.items())
+            )
+
 
 class Credentials(ModelObj):
     generate_access_key = "$generate"

mlrun/model_monitoring/api.py

@@ -107,7 +107,7 @@ def get_or_create_model_endpoint(
             sample_set_statistics=sample_set_statistics,
         )
 
-    except mlrun.errors.MLRunNotFoundError:
+    except (mlrun.errors.MLRunNotFoundError, mlrun.errors.MLRunInvalidArgumentError):
         # Create a new model endpoint with the provided details
         pass
     if not model_endpoint:

mlrun/model_monitoring/applications/evidently/base.py

@@ -23,7 +23,7 @@ import mlrun.model_monitoring.applications.base as mm_base
 import mlrun.model_monitoring.applications.context as mm_context
 from mlrun.errors import MLRunIncompatibleVersionError
 
-SUPPORTED_EVIDENTLY_VERSION = semver.Version.parse("0.4.39")
+SUPPORTED_EVIDENTLY_VERSION = semver.Version.parse("0.6.0")
 
 
 def _check_evidently_version(*, cur: semver.Version, ref: semver.Version) -> None:

mlrun/model_monitoring/applications/histogram_data_drift.py

@@ -102,10 +102,10 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
     Each metric is calculated over all the features individually and the mean is taken as the metric value.
     The average of Hellinger and total variance distance is taken as the result.
 
-    The application logs two artifacts:
+    The application can log two artifacts:
 
-    * A JSON with the general drift per feature.
-    * A plotly table different metrics per feature.
+    * JSON with the general drift value per feature, produced by default.
+    * Plotly table with the various metrics and histograms per feature (disabled by default due to performance issues).
 
     This application is deployed by default when calling:
 
@@ -114,12 +114,18 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
         project.enable_model_monitoring()
 
     To avoid it, pass :code:`deploy_histogram_data_drift_app=False`.
+
+    If you want to change the application defaults, such as the classifier or which artifacts to produce, you
+    need to inherit from this class and deploy it as any other model monitoring application.
     """
 
     NAME: Final[str] = HistogramDataDriftApplicationConstants.NAME
 
     _REQUIRED_METRICS = {HellingerDistance, TotalVarianceDistance}
-    _STATS_TYPES: tuple[StatsKind] = (StatsKind.CURRENT_STATS, StatsKind.DRIFT_MEASURES)
+    _STATS_TYPES: tuple[StatsKind, StatsKind] = (
+        StatsKind.CURRENT_STATS,
+        StatsKind.DRIFT_MEASURES,
+    )
 
     metrics: list[type[HistogramDistanceMetric]] = [
         HellingerDistance,
@@ -127,7 +133,12 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
         TotalVarianceDistance,
     ]
 
-    def __init__(self, value_classifier: Optional[ValueClassifier] = None) -> None:
+    def __init__(
+        self,
+        value_classifier: Optional[ValueClassifier] = None,
+        produce_json_artifact: bool = True,
+        produce_plotly_artifact: bool = False,
+    ) -> None:
         """
         :param value_classifier: Classifier object that adheres to the `ValueClassifier` protocol.
                                  If not provided, the default `DataDriftClassifier()` is used.
@@ -137,6 +148,9 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
             self.metrics
         ), "TVD and Hellinger distance are required for the general data drift result"
 
+        self._produce_json_artifact = produce_json_artifact
+        self._produce_plotly_artifact = produce_plotly_artifact
+
     def _compute_metrics_per_feature(
         self, monitoring_context: mm_context.MonitoringApplicationContext
     ) -> DataFrame:
@@ -295,40 +309,43 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
             cast(str, key): (self._value_classifier.value_to_status(value), value)
             for key, value in drift_per_feature_values.items()
         }
-        monitoring_context.logger.debug("Logging plotly artifact")
-        monitoring_context.log_artifact(
-            mm_drift_table.FeaturesDriftTablePlot().produce(
-                sample_set_statistics=sample_set_statistics,
-                inputs_statistics=inputs_statistics,
-                metrics=metrics_per_feature.T.to_dict(),  # pyright: ignore[reportArgumentType]
-                drift_results=drift_results,
-            )
+        monitoring_context.logger.debug("Producing plotly artifact")
+        artifact = mm_drift_table.FeaturesDriftTablePlot().produce(
+            sample_set_statistics=sample_set_statistics,
+            inputs_statistics=inputs_statistics,
+            metrics=metrics_per_feature.T.to_dict(),  # pyright: ignore[reportArgumentType]
+            drift_results=drift_results,
         )
+        monitoring_context.logger.debug("Logging plotly artifact")
+        monitoring_context.log_artifact(artifact)
         monitoring_context.logger.debug("Logged plotly artifact successfully")
 
     def _log_drift_artifacts(
         self,
         monitoring_context: mm_context.MonitoringApplicationContext,
         metrics_per_feature: DataFrame,
-        log_json_artifact: bool = True,
     ) -> None:
         """Log JSON and Plotly drift data per feature artifacts"""
+        if not self._produce_json_artifact and not self._produce_plotly_artifact:
+            return
+
         drift_per_feature_values = metrics_per_feature[
             [HellingerDistance.NAME, TotalVarianceDistance.NAME]
         ].mean(axis=1)
 
-        if log_json_artifact:
+        if self._produce_json_artifact:
             self._log_json_artifact(drift_per_feature_values, monitoring_context)
 
-        self._log_plotly_table_artifact(
-            sample_set_statistics=self._get_shared_features_sample_stats(
-                monitoring_context
-            ),
-            inputs_statistics=monitoring_context.feature_stats,
-            metrics_per_feature=metrics_per_feature,
-            drift_per_feature_values=drift_per_feature_values,
-            monitoring_context=monitoring_context,
-        )
+        if self._produce_plotly_artifact:
+            self._log_plotly_table_artifact(
+                sample_set_statistics=self._get_shared_features_sample_stats(
+                    monitoring_context
+                ),
+                inputs_statistics=monitoring_context.feature_stats,
+                metrics_per_feature=metrics_per_feature,
+                drift_per_feature_values=drift_per_feature_values,
+                monitoring_context=monitoring_context,
+            )
 
     def do_tracking(
         self, monitoring_context: mm_context.MonitoringApplicationContext

mlrun/projects/project.py

@@ -4678,7 +4678,6 @@ class MlrunProject(ModelObj):
         ] = None,  # Backward compatibility
         states: typing.Optional[list[mlrun.common.runtimes.constants.RunStates]] = None,
         sort: bool = True,
-        last: int = 0,
         iter: bool = False,
         start_time_from: Optional[datetime.datetime] = None,
         start_time_to: Optional[datetime.datetime] = None,
@@ -4751,7 +4750,6 @@ class MlrunProject(ModelObj):
                 else states or None
             ),
             sort=sort,
-            last=last,
             iter=iter,
             start_time_from=start_time_from,
             start_time_to=start_time_to,

mlrun/runtimes/databricks_job/databricks_runtime.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import typing
 from ast import FunctionDef, parse, unparse
 from base64 import b64decode
 from typing import Callable, Optional, Union
@@ -139,7 +139,7 @@ class DatabricksRuntime(kubejob.KubejobRuntime):
             )
 
     def _get_modified_user_code(self, original_handler: str, log_artifacts_code: str):
-        encoded_code = (
+        encoded_code: typing.Optional[str] = (
             self.spec.build.functionSourceCode if hasattr(self.spec, "build") else None
         )
         if not encoded_code:

mlrun/runtimes/nuclio/function.py

@@ -527,6 +527,17 @@ class RemoteRuntime(KubeResource):
         access_key = kwargs.pop("access_key", None)
         if not access_key:
             access_key = self._resolve_v3io_access_key()
+        engine = "sync"
+        explicit_ack_mode = kwargs.pop("explicit_ack_mode", None)
+        if (
+            self.spec
+            and hasattr(self.spec, "graph")
+            and self.spec.graph
+            and self.spec.graph.engine
+        ):
+            engine = self.spec.graph.engine
+        if mlrun.mlconf.is_explicit_ack_enabled() and engine == "async":
+            explicit_ack_mode = explicit_ack_mode or "explicitOnly"
 
         self.add_trigger(
             name,
@@ -540,6 +551,7 @@ class RemoteRuntime(KubeResource):
                 extra_attributes=extra_attributes,
                 read_batch_size=256,
                 access_key=access_key,
+                explicit_ack_mode=explicit_ack_mode,
                 **kwargs,
             ),
         )

mlrun/runtimes/nuclio/serving.py

@@ -376,6 +376,7 @@ class ServingRuntime(RemoteRuntime):
         creation_strategy: Optional[
             schemas.ModelEndpointCreationStrategy
         ] = schemas.ModelEndpointCreationStrategy.INPLACE,
+        outputs: Optional[list[str]] = None,
         **class_args,
     ):
         """add ml model and/or route to the function.
@@ -408,6 +409,9 @@ class ServingRuntime(RemoteRuntime):
             * **archive**:
             1. If model endpoints with the same name exist, preserve them.
             2. Create a new model endpoint with the same name and set it to `latest`.
+        :param outputs: list of the model outputs (e.g. labels) ,if provided will override the outputs that been
+                        configured in the model artifact, please note that those outputs need to be equal to the
+                        model serving function outputs (length, and order)
         :param class_args:  extra kwargs to pass to the model serving class __init__
                             (can be read in the model using .get_param(key) method)
         """
@@ -443,6 +447,8 @@ class ServingRuntime(RemoteRuntime):
         if class_name and hasattr(class_name, "to_dict"):
             if model_path:
                 class_name.model_path = model_path
+            if outputs:
+                class_name.outputs = outputs
             key, state = params_to_step(
                 class_name,
                 key,
@@ -470,6 +476,7 @@ class ServingRuntime(RemoteRuntime):
             else:
                 class_args = deepcopy(class_args)
                 class_args["model_path"] = model_path
+                class_args["outputs"] = outputs
                 state = TaskStep(
                     class_name,
                     class_args,