mlrun 1.10.0rc13__py3-none-any.whl → 1.10.0rc42__py3-none-any.whl

mlrun/common/schemas/hub.py

@@ -15,6 +15,7 @@
 from datetime import datetime, timezone
 from typing import Optional
 
+import deepdiff
 from pydantic.v1 import BaseModel, Extra, Field
 
 import mlrun.common.types
@@ -36,9 +37,9 @@ class HubObjectMetadata(BaseModel):
         extra = Extra.allow
 
 
-# Currently only functions are supported. Will add more in the future.
 class HubSourceType(mlrun.common.types.StrEnum):
     functions = "functions"
+    modules = "modules"
 
 
 # Sources-related objects
@@ -46,7 +47,6 @@ class HubSourceSpec(ObjectSpec):
     path: str  # URL to base directory, should include schema (s3://, etc...)
     channel: str
     credentials: Optional[dict] = {}
-    object_type: HubSourceType = Field(HubSourceType.functions, const=True)
 
 
 class HubSource(BaseModel):
@@ -55,11 +55,11 @@ class HubSource(BaseModel):
     spec: HubSourceSpec
     status: Optional[ObjectStatus] = ObjectStatus(state="created")
 
-    def get_full_uri(self, relative_path):
-        return f"{self.spec.path}/{self.spec.object_type}/{self.spec.channel}/{relative_path}"
+    def get_full_uri(self, relative_path, object_type):
+        return f"{self.spec.path}/{object_type}/{self.spec.channel}/{relative_path}"
 
-    def get_catalog_uri(self):
-        return self.get_full_uri(mlrun.mlconf.hub.catalog_filename)
+    def get_catalog_uri(self, object_type):
+        return self.get_full_uri(mlrun.mlconf.hub.catalog_filename, object_type)
 
     @classmethod
     def generate_default_source(cls):
@@ -78,11 +78,23 @@ class HubSource(BaseModel):
             spec=HubSourceSpec(
                 path=mlrun.mlconf.hub.default_source.url,
                 channel=mlrun.mlconf.hub.default_source.channel,
-                object_type=HubSourceType(mlrun.mlconf.hub.default_source.object_type),
             ),
             status=ObjectStatus(state="created"),
         )
 
+    def diff(self, another_source: "HubSource") -> dict:
+        """
+        Compare this HubSource with another one.
+        Returns a dict of differences (metadata, spec, status).
+        """
+        exclude_paths = [
+            "root['metadata']['updated']",
+            "root['metadata']['created']",
+        ]
+        return deepdiff.DeepDiff(
+            self.dict(), another_source.dict(), exclude_paths=exclude_paths
+        )
+
 
 last_source_index = -1
 
@@ -94,21 +106,16 @@ class IndexedHubSource(BaseModel):
 
 # Item-related objects
 class HubItemMetadata(HubObjectMetadata):
-    source: HubSourceType = Field(HubSourceType.functions, const=True)
+    source: HubSourceType = HubSourceType.functions
     version: str
     tag: Optional[str]
 
     def get_relative_path(self) -> str:
-        if self.source == HubSourceType.functions:
-            # This is needed since the hub deployment script modifies the paths to use _ instead of -.
-            modified_name = self.name.replace("-", "_")
-            # Prefer using the tag if exists. Otherwise, use version.
-            version = self.tag or self.version
-            return f"{modified_name}/{version}/"
-        else:
-            raise mlrun.errors.MLRunInvalidArgumentError(
-                f"Bad source for hub item - {self.source}"
-            )
+        # This is needed since the hub deployment script modifies the paths to use _ instead of -.
+        modified_name = self.name.replace("-", "_")
+        # Prefer using the tag if exists. Otherwise, use version.
+        version = self.tag or self.version
+        return f"{modified_name}/{version}/"
 
 
 class HubItemSpec(ObjectSpec):
@@ -127,3 +134,8 @@ class HubCatalog(BaseModel):
     kind: ObjectKind = Field(ObjectKind.hub_catalog, const=True)
     channel: str
     catalog: list[HubItem]
+
+
+class HubModuleType(mlrun.common.types.StrEnum):
+    generic = "generic"
+    monitoring_app = "monitoring_application"

mlrun/common/schemas/model_monitoring/__init__.py

@@ -16,6 +16,7 @@ from .constants import (
     INTERSECT_DICT_KEYS,
     ApplicationEvent,
     DriftStatus,
+    EndpointMode,
     EndpointType,
     EndpointUID,
     EventFieldType,
@@ -29,6 +30,7 @@ from .constants import (
     ModelEndpointMonitoringMetricType,
     ModelEndpointSchema,
     ModelMonitoringAppLabel,
+    ModelMonitoringInfraLabel,
     ModelMonitoringMode,
     MonitoringFunctionNames,
     PredictionsQueryConstants,
@@ -59,6 +61,7 @@ from .model_endpoints import (
     Features,
     FeatureValues,
     ModelEndpoint,
+    ModelEndpointDriftValues,
     ModelEndpointList,
     ModelEndpointMetadata,
     ModelEndpointMonitoringMetric,

mlrun/common/schemas/model_monitoring/constants.py

@@ -34,6 +34,7 @@ class ModelEndpointSchema(MonitoringStrEnum):
     UID = "uid"
     PROJECT = "project"
     ENDPOINT_TYPE = "endpoint_type"
+    MODE = "mode"
     NAME = "name"
     CREATED = "created"
     UPDATED = "updated"
@@ -195,6 +196,10 @@ class WriterEventKind(MonitoringStrEnum):
     RESULT = "result"
     STATS = "stats"
 
+    @classmethod
+    def user_app_outputs(cls):
+        return [cls.METRIC, cls.RESULT]
+
 
 class ControllerEvent(MonitoringStrEnum):
     KIND = "kind"
@@ -205,6 +210,11 @@ class ControllerEvent(MonitoringStrEnum):
     FIRST_REQUEST = "first_request"
     FEATURE_SET_URI = "feature_set_uri"
     ENDPOINT_TYPE = "endpoint_type"
+
+    # first_timestamp and last_timestamp are used to batch completed events
+    FIRST_TIMESTAMP = "first_timestamp"
+    LAST_TIMESTAMP = "last_timestamp"
+
     ENDPOINT_POLICY = "endpoint_policy"
     # Note: currently under endpoint policy we will have a dictionary including the keys: "application_names"
     # "base_period", and "updated_endpoint" stand for when the MEP was updated
@@ -219,6 +229,7 @@ class ControllerEventEndpointPolicy(MonitoringStrEnum):
 class ControllerEventKind(MonitoringStrEnum):
     NOP_EVENT = "nop_event"
     REGULAR_EVENT = "regular_event"
+    BATCH_COMPLETE = "batch_complete"
 
 
 class MetricData(MonitoringStrEnum):
@@ -297,6 +308,7 @@ class FileTargetKind:
     MONITORING_APPLICATION = "monitoring_application"
     ERRORS = "errors"
     STATS = "stats"
+    PARQUET_STATS = "parquet_stats"
     LAST_REQUEST = "last_request"
 
 
@@ -321,6 +333,12 @@ class EndpointType(IntEnum):
         return [cls.NODE_EP, cls.ROUTER, cls.BATCH_EP]
 
 
+class EndpointMode(IntEnum):
+    REAL_TIME = 0
+    BATCH = 1
+    BATCH_LEGACY = 2  # legacy batch mode, used for endpoints created through the batch inference job
+
+
 class MonitoringFunctionNames(MonitoringStrEnum):
     STREAM = "model-monitoring-stream"
     APPLICATION_CONTROLLER = "model-monitoring-controller"
@@ -474,19 +492,25 @@ class ModelEndpointMonitoringMetricType(StrEnum):
     METRIC = "metric"
 
 
+# refer to `mlrun.utils.regex.project_name`
+_INNER_PROJECT_PATTERN = r"[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?"
+PROJECT_PATTERN = rf"^{_INNER_PROJECT_PATTERN}$"
+
+MODEL_ENDPOINT_ID_PATTERN = r"^[a-zA-Z0-9_-]+$"
+
 _FQN_PART_PATTERN = r"[a-zA-Z0-9_-]+"
+_RESULT_NAME_PATTERN = r"[a-zA-Z_][a-zA-Z0-9_]*"
+
 FQN_PATTERN = (
-    rf"^(?P<project>{_FQN_PART_PATTERN})\."
+    rf"^(?P<project>{_INNER_PROJECT_PATTERN})\."
     rf"(?P<app>{_FQN_PART_PATTERN})\."
     rf"(?P<type>{ModelEndpointMonitoringMetricType.RESULT}|{ModelEndpointMonitoringMetricType.METRIC})\."
-    rf"(?P<name>{_FQN_PART_PATTERN})$"
+    rf"(?P<name>{_RESULT_NAME_PATTERN})$"
 )
 FQN_REGEX = re.compile(FQN_PATTERN)
+APP_NAME_REGEX = re.compile(_FQN_PART_PATTERN)
+RESULT_NAME_REGEX = re.compile(_RESULT_NAME_PATTERN)
 
-# refer to `mlrun.utils.regex.project_name`
-PROJECT_PATTERN = r"^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$"
-MODEL_ENDPOINT_ID_PATTERN = r"^[a-zA-Z0-9_-]+$"
-RESULT_NAME_PATTERN = r"[a-zA-Z_][a-zA-Z0-9_]*"
 
 INTERSECT_DICT_KEYS = {
     ModelEndpointMonitoringMetricType.METRIC: "intersect_metrics",

mlrun/common/schemas/model_monitoring/functions.py

@@ -54,15 +54,24 @@ class FunctionSummary(BaseModel):
 
         return cls(
             type=func_type,
-            name=func_dict["metadata"]["name"],
+            name=func_dict["metadata"]["name"]
+            if func_type != FunctionsType.APPLICATION
+            else func_dict["spec"]
+            .get("graph", {})
+            .get("steps", {})
+            .get("PrepareMonitoringEvent", {})
+            .get("class_args", {})
+            .get("application_name"),
             application_class=""
             if func_type != FunctionsType.APPLICATION
-            else func_dict["spec"]["graph"]["steps"]["PushToMonitoringWriter"]["after"][
-                0
-            ],
+            else func_dict["spec"]
+            .get("graph", {})
+            .get("steps", {})
+            .get("PushToMonitoringWriter", {})
+            .get("after", [None])[0],
             project_name=func_dict["metadata"]["project"],
             updated_time=func_dict["metadata"].get("updated"),
             status=func_dict["status"].get("state"),
             base_period=base_period,
-            stats=stats,
+            stats=stats or {},
         )

mlrun/common/schemas/model_monitoring/model_endpoints.py

@@ -28,6 +28,7 @@ from .constants import (
     FQN_REGEX,
     MODEL_ENDPOINT_ID_PATTERN,
     PROJECT_PATTERN,
+    EndpointMode,
     EndpointType,
     ModelEndpointMonitoringMetricType,
     ModelMonitoringMode,
@@ -118,6 +119,7 @@ class ModelEndpointMetadata(ObjectMetadata, ModelEndpointParser):
     project: constr(regex=PROJECT_PATTERN)
     endpoint_type: EndpointType = EndpointType.NODE_EP
     uid: Optional[constr(regex=MODEL_ENDPOINT_ID_PATTERN)]
+    mode: Optional[EndpointMode] = None
 
     @classmethod
     def mutable_fields(cls):
@@ -129,6 +131,15 @@ class ModelEndpointMetadata(ObjectMetadata, ModelEndpointParser):
             return str(v)
         return v
 
+    @validator("mode", pre=True, always=True)
+    def _set_mode_based_on_endpoint_type(cls, v, values):  # noqa: N805
+        if v is None:
+            if values.get("endpoint_type") == EndpointType.BATCH_EP:
+                return EndpointMode.BATCH_LEGACY
+            else:
+                return EndpointMode.REAL_TIME
+        return v
+
 
 class ModelEndpointSpec(ObjectSpec, ModelEndpointParser):
     model_class: Optional[str] = ""
@@ -352,6 +363,16 @@ class ApplicationMetricRecord(ApplicationBaseRecord):
     type: Literal["metric"] = "metric"
 
 
+class _DriftBin(NamedTuple):
+    timestamp: datetime
+    count_suspected: int
+    count_detected: int
+
+
+class ModelEndpointDriftValues(BaseModel):
+    values: list[_DriftBin]
+
+
 def _mapping_attributes(
     model_class: type[Model],
     flattened_dictionary: dict,

mlrun/common/schemas/pipeline.py

@@ -18,7 +18,7 @@ import pydantic.v1
 
 
 class PipelinesPagination(str):
-    default_page_size = 20
+    default_page_size = 200
     # https://github.com/kubeflow/pipelines/blob/master/backend/src/apiserver/list/list.go#L363
     max_page_size = 200
 

mlrun/common/schemas/serving.py

@@ -47,3 +47,6 @@ class MonitoringData(StrEnum):
 class ModelsData(enum.Enum):
     MODEL_CLASS = 0
     MODEL_PARAMETERS = 1
+
+
+MAX_BATCH_JOB_DURATION = "1w"

mlrun/common/schemas/workflow.py

@@ -49,9 +49,11 @@ class WorkflowRequest(pydantic.v1.BaseModel):
 class RerunWorkflowRequest(pydantic.v1.BaseModel):
     run_name: typing.Optional[str] = None
     run_id: typing.Optional[str] = None
-    original_workflow_id: typing.Optional[str] = None
     notifications: typing.Optional[list[Notification]] = None
     workflow_runner_node_selector: typing.Optional[dict[str, str]] = None
+    original_workflow_runner_uid: typing.Optional[str] = None
+    original_workflow_name: typing.Optional[str] = None
+    rerun_index: typing.Optional[int] = None
 
 
 class WorkflowResponse(pydantic.v1.BaseModel):

mlrun/common/secrets.py

@@ -11,10 +11,31 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import re
 from abc import ABC, abstractmethod
 
 import mlrun.common.schemas
+from mlrun.config import config as mlconf
+
+_AUTH_SECRET_NAME_TEMPLATE = re.escape(
+    mlconf.secret_stores.kubernetes.auth_secret_name.format(
+        hashed_access_key="",
+    )
+)
+AUTH_SECRET_PATTERN = re.compile(f"^{_AUTH_SECRET_NAME_TEMPLATE}.*")
+
+
+def validate_not_forbidden_secret(secret_name: str) -> None:
+    """
+    Forbid client-supplied references to internal MLRun auth/project secrets.
+    No-op when running inside the API server (API enrichments are allowed).
+    """
+    if not secret_name or mlrun.config.is_running_as_api():
+        return
+    if AUTH_SECRET_PATTERN.match(secret_name):
+        raise mlrun.errors.MLRunInvalidArgumentError(
+            f"Forbidden secret '{secret_name}' matches MLRun auth-secret pattern."
+        )
 
 
 class SecretProviderInterface(ABC):

mlrun/config.py

@@ -107,7 +107,11 @@ default_config = {
     "submit_timeout": "280",  # timeout when submitting a new k8s resource
     # runtimes cleanup interval in seconds
     "runtimes_cleanup_interval": "300",
-    "background_task_cleanup_interval": "86400",  # 24 hours in seconds
+    # disabled by default due to an internal bug in serving functions
+    # relying on a background task to hold the status for its model endpoints
+    # TODO: need to refine what/when we can delete the background tasks
+    # e.g: use labels or naming convention.
+    "background_task_cleanup_interval": "0",
     "background_task_max_age": "21600",  # 6 hours in seconds
     "monitoring": {
         "runs": {
@@ -193,7 +197,8 @@ default_config = {
     },
     "v3io_framesd": "http://framesd:8080",
     "model_providers": {
-        "openai_default_model": "gpt-4",
+        "openai_default_model": "gpt-4o",
+        "huggingface_default_model": "microsoft/Phi-3-mini-4k-instruct",
     },
     # default node selector to be applied to all functions - json string base64 encoded format
     "default_function_node_selector": "e30=",
@@ -250,7 +255,8 @@ default_config = {
             },
             "runtimes": {
                 "dask": "600",
-                "dask_cluster_start": "300",
+                # cluster start might take some time in case k8s needs to spin up new nodes
+                "dask_cluster_start": "600",
             },
             "push_notifications": "60",
         },
@@ -298,6 +304,7 @@ default_config = {
         "application": {
             "default_sidecar_internal_port": 8050,
             "default_authentication_mode": mlrun.common.schemas.APIGatewayAuthenticationMode.none,
+            "default_worker_number": 10000,
         },
     },
     # TODO: function defaults should be moved to the function spec config above
@@ -406,11 +413,7 @@ default_config = {
                 #
                 # if set to "nil" or "none", nothing would be set
                 "modes": (
-                    "STRICT_TRANS_TABLES"
-                    ",NO_ZERO_IN_DATE"
-                    ",NO_ZERO_DATE"
-                    ",ERROR_FOR_DIVISION_BY_ZERO"
-                    ",NO_ENGINE_SUBSTITUTION",
+                    "STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION"
                 )
             },
         },
@@ -647,6 +650,13 @@ default_config = {
                 "max_replicas": 1,
             },
         },
+        "writer_graph": {
+            "max_events": 1000,
+            "flush_after_seconds": 30,
+            "writer_version": "v1",  # v1 is the sync version while v2 is async
+            "parquet_batching_max_events": 10,
+            "parquet_batching_timeout_secs": 30,
+        },
         # Store prefixes are used to handle model monitoring storing policies based on project and kind, such as events,
         # stream, and endpoints.
         "store_prefixes": {
@@ -717,7 +727,6 @@ default_config = {
             "name": "default",
             "description": "MLRun global function hub",
             "url": "https://mlrun.github.io/marketplace",
-            "object_type": "functions",
             "channel": "master",
         },
     },
@@ -999,9 +1008,9 @@ class Config:
         )
 
     @staticmethod
-    def get_default_hub_source() -> str:
+    def get_default_hub_source_url_prefix(object_type) -> str:
         default_source = config.hub.default_source
-        return f"{default_source.url}/{default_source.object_type}/{default_source.channel}/"
+        return f"{default_source.url}/{object_type}/{default_source.channel}/"
 
     @staticmethod
     def decode_base64_config_and_load_to_object(
@@ -1242,6 +1251,19 @@ class Config:
         """
         return self.is_running_on_iguazio()
 
+    @staticmethod
+    def get_run_retry_staleness_threshold_timedelta() -> timedelta:
+        """
+        Get the staleness threshold in timedelta for run retries.
+        This is used to determine if a run is stale and should be retried.
+
+        :return: The staleness threshold in timedelta.
+        """
+        staleness_threshold = int(
+            mlrun.mlconf.monitoring.runs.retry.staleness_threshold
+        )
+        return timedelta(minutes=staleness_threshold)
+
     def to_dict(self):
         return copy.deepcopy(self._cfg)
 

mlrun/datastore/__init__.py

@@ -39,10 +39,11 @@ __all__ = [
 from urllib.parse import urlparse
 
 import fsspec
+import storey
 
 import mlrun.datastore.wasbfs
 from mlrun.datastore.datastore_profile import (
-    DatastoreProfileKafkaSource,
+    DatastoreProfileKafkaStream,
     DatastoreProfileKafkaTarget,
     DatastoreProfileV3io,
 )
@@ -122,7 +123,7 @@ def get_stream_pusher(stream_path: str, **kwargs):
         )
         if isinstance(
             datastore_profile,
-            (DatastoreProfileKafkaSource, DatastoreProfileKafkaTarget),
+            (DatastoreProfileKafkaStream, DatastoreProfileKafkaTarget),
         ):
             attributes = datastore_profile.attributes()
             brokers = attributes.pop("brokers", None)
@@ -168,11 +169,12 @@ def get_stream_pusher(stream_path: str, **kwargs):
             raise ValueError(f"unsupported stream path {stream_path}")
 
 
-class _DummyStream:
+class _DummyStream(storey.MapClass):
     """stream emulator for tests and debug"""
 
     def __init__(self, event_list=None, **kwargs):
         self.event_list = event_list or []
+        super().__init__(**kwargs)
 
     def push(self, data, **kwargs):
         if not isinstance(data, list):
@@ -180,3 +182,9 @@ class _DummyStream:
         for item in data:
             logger.info(f"dummy stream got event: {item}, kwargs={kwargs}")
             self.event_list.append(item)
+
+    def do(self, event):
+        if not isinstance(event, list):
+            event = [event]
+        for item in event:
+            self.event_list.append(item)