mlrun 1.8.0rc34__py3-none-any.whl → 1.8.0rc36__py3-none-any.whl

mlrun/artifacts/model.py

@@ -279,7 +279,11 @@ class ModelArtifact(Artifact):
         )
         if label_columns:
             inferer.infer_schema(
-                df[label_columns], self.spec.outputs, {}, options=InferOptions.Features
+                df[label_columns],
+                self.spec.outputs,
+                {},
+                options=InferOptions.Features,
+                push_at_start=True,
             )
         if with_stats:
             self.spec.feature_stats = inferer.get_stats(

mlrun/common/schemas/artifact.py

@@ -57,6 +57,15 @@ class ArtifactCategories(mlrun.common.types.StrEnum):
             return cls(kind)
         return cls.other
 
+    @staticmethod
+    def all():
+        """Return all applicable artifact categories"""
+        return [
+            ArtifactCategories.model,
+            ArtifactCategories.dataset,
+            ArtifactCategories.document,
+        ]
+
 
 class ArtifactIdentifier(pydantic.v1.BaseModel):
     # artifact kind

mlrun/common/schemas/auth.py

@@ -114,6 +114,8 @@ class AuthorizationVerificationInput(pydantic.v1.BaseModel):
 
 
 class AuthInfo(pydantic.v1.BaseModel):
+    # Keep request headers for inter-service communication
+    request_headers: typing.Optional[dict[str, str]] = None
     # Basic + Iguazio auth
     username: typing.Optional[str] = None
     # Basic auth

mlrun/config.py

@@ -816,6 +816,8 @@ default_config = {
         "max_criteria_count": 100,
         # interval for periodic events generation job
         "events_generation_interval": 30,  # seconds
+        # number of alerts to delete in each chunk
+        "chunk_size_during_project_deletion": 100,
         # maximum allowed alert config cache size in alert's CRUD
         # for the best performance, it is recommended to set this value to the maximum number of alerts
         "max_allowed_cache_size": 20000,

mlrun/data_types/infer.py

@@ -20,6 +20,8 @@ import pandas as pd
 import pyarrow
 from pandas.io.json._table_schema import convert_pandas_type_to_json_field
 
+import mlrun.features
+from mlrun.model import ObjectList
 from mlrun.utils import logger
 
 from .data_types import InferOptions, pa_type_to_value_type, pd_schema_to_value_type
@@ -29,17 +31,19 @@ default_num_bins = 20
 
 def infer_schema_from_df(
     df: pd.DataFrame,
-    features,
+    features: ObjectList,
     entities,
     timestamp_key: Optional[str] = None,
     entity_columns=None,
     options: InferOptions = InferOptions.Null,
+    push_at_start: Optional[bool] = False,
 ):
     """infer feature set schema from dataframe"""
     timestamp_fields = []
     current_entities = list(entities.keys())
     entity_columns = entity_columns or []
     index_columns = dict()
+    temp_features = ObjectList(mlrun.features.Feature)
 
     def upsert_entity(name, value_type):
         if name in current_entities:
@@ -74,10 +78,14 @@ def infer_schema_from_df(
             if column in features.keys():
                 features[column].value_type = value_type
             else:
-                features[column] = {"name": column, "value_type": value_type}
+                temp_features[column] = {"name": column, "value_type": value_type}
         if value_type == "datetime" and not is_entity:
             timestamp_fields.append(column)
 
+    features.update_list(
+        object_list=temp_features, push_at_start=push_at_start
+    )  # Push to start of the Object list
+
     index_type = None
     if InferOptions.get_common_options(options, InferOptions.Index):
         # infer types of index fields

mlrun/datastore/sources.py

@@ -18,7 +18,7 @@ import warnings
 from base64 import b64encode
 from copy import copy
 from datetime import datetime
-from typing import Optional, Union
+from typing import Any, Optional, Union
 
 import pandas as pd
 import semver
@@ -34,6 +34,7 @@ from mlrun.datastore.utils import transform_list_filters_to_tuple
 from mlrun.secrets import SecretsStore
 from mlrun.utils import logger
 
+from ..common.schemas.function import Function
 from ..model import DataSource
 from ..platforms.iguazio import parse_path
 from ..utils import get_class, is_explicit_ack_supported
@@ -966,6 +967,26 @@ class OnlineSource(BaseSourceDriver):
             "This source type is not supported with ingestion service yet"
         )
 
+    @staticmethod
+    def set_explicit_ack_mode(function: Function, **extra_arguments) -> dict[str, Any]:
+        extra_arguments = extra_arguments or {}
+        engine = "sync"
+        if (
+            function.spec
+            and hasattr(function.spec, "graph")
+            and function.spec.graph
+            and function.spec.graph.engine
+        ):
+            engine = function.spec.graph.engine
+        if mlrun.mlconf.is_explicit_ack_enabled() and engine == "async":
+            extra_arguments["explicit_ack_mode"] = extra_arguments.get(
+                "explicit_ack_mode", "explicitOnly"
+            )
+            extra_arguments["worker_allocation_mode"] = extra_arguments.get(
+                "worker_allocation_mode", "static"
+            )
+        return extra_arguments
+
 
 class HttpSource(OnlineSource):
     kind = "http"
@@ -1028,15 +1049,7 @@ class StreamSource(OnlineSource):
             raise_for_status=v3io.dataplane.RaiseForStatus.never,
         )
         res.raise_for_status([409, 204])
-
-        kwargs = {}
-        engine = "async"
-        if hasattr(function.spec, "graph") and function.spec.graph.engine:
-            engine = function.spec.graph.engine
-
-        if mlrun.mlconf.is_explicit_ack_enabled() and engine == "async":
-            kwargs["explicit_ack_mode"] = "explicitOnly"
-            kwargs["worker_allocation_mode"] = "static"
+        kwargs = self.set_explicit_ack_mode(function=function)
 
         function.add_v3io_stream_trigger(
             url,
@@ -1118,20 +1131,12 @@ class KafkaSource(OnlineSource):
         else:
             extra_attributes = copy(self.attributes)
         partitions = extra_attributes.pop("partitions", None)
-        explicit_ack_mode = None
-        engine = "async"
-        if hasattr(function.spec, "graph") and function.spec.graph.engine:
-            engine = function.spec.graph.engine
 
-        if mlrun.mlconf.is_explicit_ack_enabled() and engine == "async":
-            explicit_ack_mode = "explicitOnly"
-            extra_attributes["workerAllocationMode"] = extra_attributes.get(
-                "worker_allocation_mode", "static"
-            )
-        else:
-            extra_attributes["workerAllocationMode"] = extra_attributes.get(
-                "worker_allocation_mode", "pool"
-            )
+        extra_attributes = self.set_explicit_ack_mode(function, **extra_attributes)
+        explicit_ack_mode = extra_attributes.get("explicit_ack_mode")
+        extra_attributes["workerAllocationMode"] = extra_attributes.get(
+            "worker_allocation_mode", "pool"
+        )
 
         trigger_kwargs = {}
 

mlrun/db/base.py

@@ -102,7 +102,6 @@ class RunDBInterface(ABC):
         ] = None,  # Backward compatibility
         states: Optional[list[mlrun.common.runtimes.constants.RunStates]] = None,
         sort: bool = True,
-        last: int = 0,
         iter: bool = False,
         start_time_from: Optional[datetime.datetime] = None,
         start_time_to: Optional[datetime.datetime] = None,
@@ -149,7 +148,13 @@ class RunDBInterface(ABC):
 
     @abstractmethod
     def store_artifact(
-        self, key, artifact, uid=None, iter=None, tag="", project="", tree=None
+        self,
+        key,
+        artifact,
+        iter=None,
+        tag="",
+        project="",
+        tree=None,
     ):
         pass
 

mlrun/db/httpdb.py

@@ -350,17 +350,10 @@ class HTTPRunDB(RunDBInterface):
                 version=version,
             )
 
-        page_params = deepcopy(params) or {}
-
-        if page_params.get("page-token") is None and page_params.get("page") is None:
-            page_params["page"] = 1
-
-        if page_params.get("page-size") is None:
-            page_params["page-size"] = config.httpdb.pagination.default_page_size
-
+        page_params = self._resolve_page_params(params)
         response = _api_call(page_params)
 
-        # Yield only a single page of results
+        # yields a single page of results
         yield response
 
         if return_all:
@@ -899,7 +892,6 @@ class HTTPRunDB(RunDBInterface):
         ] = None,  # Backward compatibility
         states: typing.Optional[list[mlrun.common.runtimes.constants.RunStates]] = None,
         sort: bool = True,
-        last: int = 0,
         iter: bool = False,
         start_time_from: Optional[datetime] = None,
         start_time_to: Optional[datetime] = None,
@@ -946,7 +938,6 @@ class HTTPRunDB(RunDBInterface):
         :param states: List only runs whose state is one of the provided states.
         :param sort: Whether to sort the result according to their start time. Otherwise, results will be
             returned by their internal order in the DB (order will not be guaranteed).
-        :param last: Deprecated - currently not used (will be removed in 1.8.0).
         :param iter: If ``True`` return runs from all iterations. Otherwise, return only runs whose ``iter`` is 0.
         :param start_time_from: Filter by run start time in ``[start_time_from, start_time_to]``.
         :param start_time_to: Filter by run start time in ``[start_time_from, start_time_to]``.
@@ -974,7 +965,6 @@ class HTTPRunDB(RunDBInterface):
             state=state,
             states=states,
             sort=sort,
-            last=last,
             iter=iter,
             start_time_from=start_time_from,
             start_time_to=start_time_to,
@@ -1094,8 +1084,6 @@ class HTTPRunDB(RunDBInterface):
         self,
         key,
         artifact,
-        # TODO: deprecated, remove in 1.8.0
-        uid=None,
         iter=None,
         tag=None,
         project="",
@@ -1105,8 +1093,6 @@ class HTTPRunDB(RunDBInterface):
 
         :param key: Identifying key of the artifact.
         :param artifact: The :py:class:`~mlrun.artifacts.Artifact` to store.
-        :param uid: A unique ID for this specific version of the artifact
-                    (deprecated, artifact uid is generated in the backend use `tree` instead)
         :param iter: The task iteration which generated this artifact. If ``iter`` is not ``None`` the iteration will
             be added to the key provided to generate a unique key for the artifact of the specific iteration.
         :param tag: Tag of the artifact.
@@ -1114,15 +1100,6 @@ class HTTPRunDB(RunDBInterface):
         :param tree: The tree (producer id) which generated this artifact.
         :returns: The stored artifact dictionary.
         """
-        if uid:
-            warnings.warn(
-                "'uid' is deprecated in 1.6.0 and will be removed in 1.8.0, use 'tree' instead.",
-                # TODO: Remove this in 1.8.0
-                FutureWarning,
-            )
-
-        # we do this because previously the 'uid' name was used for the 'tree' parameter
-        tree = tree or uid
         project = project or mlrun.mlconf.default_project
         endpoint_path = f"projects/{project}/artifacts/{key}"
 
@@ -1295,7 +1272,7 @@ class HTTPRunDB(RunDBInterface):
         :param rows_per_partition: How many top rows (per sorting defined by `partition_sort_by` and `partition_order`)
             to return per group. Default value is 1.
         :param partition_sort_by: What field to sort the results by, within each partition defined by `partition_by`.
-            Currently the only allowed values are `created` and `updated`.
+            Currently, the only allowed values are `created` and `updated`.
         :param partition_order: Order of sorting within partitions - `asc` or `desc`. Default is `desc`.
         """
 
@@ -1318,7 +1295,7 @@ class HTTPRunDB(RunDBInterface):
             rows_per_partition=rows_per_partition,
             partition_sort_by=partition_sort_by,
             partition_order=partition_order,
-            return_all=True,
+            return_all=not limit,
         )
         return artifacts
 
@@ -5202,7 +5179,6 @@ class HTTPRunDB(RunDBInterface):
         ] = None,  # Backward compatibility
         states: typing.Optional[list[mlrun.common.runtimes.constants.RunStates]] = None,
         sort: bool = True,
-        last: int = 0,
         iter: bool = False,
         start_time_from: Optional[datetime] = None,
         start_time_to: Optional[datetime] = None,
@@ -5234,13 +5210,6 @@ class HTTPRunDB(RunDBInterface):
                 "using the `with_notifications` flag."
             )
 
-        if last:
-            # TODO: Remove this in 1.8.0
-            warnings.warn(
-                "'last' is deprecated and will be removed in 1.8.0.",
-                FutureWarning,
-            )
-
         if state:
             # TODO: Remove this in 1.9.0
             warnings.warn(
@@ -5256,7 +5225,6 @@ class HTTPRunDB(RunDBInterface):
             and not labels
             and not state
             and not states
-            and not last
             and not start_time_from
             and not start_time_to
             and not last_update_time_from
@@ -5378,6 +5346,33 @@ class HTTPRunDB(RunDBInterface):
             )
         return None
 
+    def _resolve_page_params(self, params: typing.Optional[dict]) -> dict:
+        """
+        Resolve the page parameters, setting defaults where necessary.
+        """
+        page_params = deepcopy(params) or {}
+        if page_params.get("page-token") is None and page_params.get("page") is None:
+            page_params["page"] = 1
+        if page_params.get("page-size") is None:
+            page_size = config.httpdb.pagination.default_page_size
+
+            if page_params.get("limit") is not None:
+                page_size = page_params["limit"]
+
+                # limit and page/page size are conflicting
+                page_params.pop("limit")
+            page_params["page-size"] = page_size
+
+        # this may happen only when page-size was explicitly set along with limit
+        # this is to ensure we will not get stopped by API on similar below validation
+        # but rather simply fallback to use page-size.
+        if page_params.get("page-size") and page_params.get("limit"):
+            logger.warning(
+                "Both 'limit' and 'page-size' are provided, using 'page-size'."
+            )
+            page_params.pop("limit")
+        return page_params
+
 
 def _as_json(obj):
     fn = getattr(obj, "to_json", None)

mlrun/db/nopdb.py

@@ -132,7 +132,6 @@ class NopDB(RunDBInterface):
         ] = None,  # Backward compatibility
         states: Optional[list[mlrun.common.runtimes.constants.RunStates]] = None,
         sort: bool = True,
-        last: int = 0,
         iter: bool = False,
         start_time_from: Optional[datetime.datetime] = None,
         start_time_to: Optional[datetime.datetime] = None,
@@ -175,7 +174,13 @@ class NopDB(RunDBInterface):
         pass
 
     def store_artifact(
-        self, key, artifact, uid=None, iter=None, tag="", project="", tree=None
+        self,
+        key,
+        artifact,
+        iter=None,
+        tag="",
+        project="",
+        tree=None,
     ):
         pass
 

mlrun/feature_store/__init__.py

@@ -13,11 +13,6 @@
 # limitations under the License.
 
 __all__ = [
-    "get_offline_features",
-    "get_online_feature_service",
-    "ingest",
-    "preview",
-    "deploy_ingestion_service_v2",
     "delete_feature_set",
     "delete_feature_vector",
     "get_feature_set",
@@ -38,13 +33,8 @@ from ..features import Entity, Feature
 from .api import (
     delete_feature_set,
     delete_feature_vector,
-    deploy_ingestion_service_v2,
     get_feature_set,
     get_feature_vector,
-    get_offline_features,
-    get_online_feature_service,
-    ingest,
-    preview,
 )
 from .common import RunConfig
 from .feature_set import FeatureSet