mlrun 1.10.0rc6__py3-none-any.whl → 1.10.0rc8__py3-none-any.whl

mlrun/__init__.py

@@ -61,6 +61,7 @@ from .run import (
     import_function,
     new_function,
     retry_pipeline,
+    terminate_pipeline,
     wait_for_pipeline_completion,
 )
 from .runtimes import mounts, new_model_server
@@ -217,5 +218,6 @@ def set_env_from_file(env_file: str, return_dict: bool = False) -> Optional[dict
     for key, value in env_vars.items():
         environ[key] = value
 
-    mlconf.reload()  # reload mlrun configuration
+    # reload mlrun configuration
+    mlconf.reload()
     return env_vars if return_dict else None

mlrun/__main__.py

@@ -23,6 +23,7 @@ from ast import literal_eval
 from base64 import b64decode
 from os import environ, path, remove
 from pprint import pprint
+from typing import Optional
 
 import click
 import dotenv
@@ -199,6 +200,13 @@ def main():
     multiple=True,
     help="Logging configurations for the handler's returning values",
 )
+@click.option(
+    "--allow-cross-project",
+    is_flag=True,
+    default=True,  # TODO: remove this default in 1.11
+    help="Override the loaded project name. This flag ensures awareness of loading an existing project yaml "
+    "as a baseline for a new project with a different name",
+)
 def run(
     url,
     param,
@@ -242,6 +250,7 @@ def run(
     run_args,
     ensure_project,
     returns,
+    allow_cross_project,
 ):
     """Execute a task and inject parameters."""
 
@@ -293,10 +302,11 @@ def run(
         mlrun.get_or_create_project(
             name=project,
             context="./",
+            allow_cross_project=allow_cross_project,
         )
     if func_url or kind:
         if func_url:
-            runtime = func_url_to_runtime(func_url, ensure_project)
+            runtime = func_url_to_runtime(func_url, ensure_project, allow_cross_project)
             kind = get_in(runtime, "kind", kind or "job")
             if runtime is None:
                 exit(1)
@@ -494,6 +504,13 @@ def run(
     default="/tmp/fullimage",
     help="path to file with full image data",
 )
+@click.option(
+    "--allow-cross-project",
+    is_flag=True,
+    default=True,  # TODO: remove this default in 1.11
+    help="Override the loaded project name. This flag ensures awareness of loading an existing project yaml "
+    "as a baseline for a new project with a different name",
+)
 def build(
     func_url,
     name,
@@ -516,6 +533,7 @@ def build(
     state_file_path,
     image_file_path,
     full_image_file_path,
+    allow_cross_project,
 ):
     """Build a container image from code and requirements."""
 
@@ -591,6 +609,7 @@ def build(
         mlrun.get_or_create_project(
             name=project,
             context="./",
+            allow_cross_project=allow_cross_project,
         )
 
     if hasattr(func, "deploy"):
@@ -644,6 +663,13 @@ def build(
     is_flag=True,
     help="ensure the project exists, if not, create project",
 )
+@click.option(
+    "--allow-cross-project",
+    is_flag=True,
+    default=True,  # TODO: remove this default in 1.11
+    help="Override the loaded project name. This flag ensures awareness of loading an existing project yaml "
+    "as a baseline for a new project with a different name",
+)
 def deploy(
     spec,
     source,
@@ -656,6 +682,7 @@ def deploy(
     verbose,
     env_file,
     ensure_project,
+    allow_cross_project,
 ):
     """Deploy model or function"""
     if env_file:
@@ -665,10 +692,11 @@ def deploy(
         mlrun.get_or_create_project(
             name=project,
             context="./",
+            allow_cross_project=allow_cross_project,
         )
 
     if func_url:
-        runtime = func_url_to_runtime(func_url, ensure_project)
+        runtime = func_url_to_runtime(func_url, ensure_project, allow_cross_project)
         if runtime is None:
             exit(1)
     elif spec:
@@ -971,6 +999,13 @@ def logs(uid, project, offset, db):
     "destination define: file=notification.json or a "
     'dictionary configuration e.g \'{"slack":{"webhook":"<webhook>"}}\'',
 )
+@click.option(
+    "--allow-cross-project",
+    is_flag=True,
+    default=True,  # TODO: remove this default in 1.11
+    help="Override the loaded project name. This flag ensures awareness of loading an existing project yaml "
+    "as a baseline for a new project with a different name",
+)
 def project(
     context,
     name,
@@ -998,6 +1033,7 @@ def project(
     notifications,
     save_secrets,
     save,
+    allow_cross_project,
 ):
     """load and/or run a project"""
     if env_file:
@@ -1024,6 +1060,7 @@ def project(
         clone=clone,
         save=save,
         parameters=parameters,
+        allow_cross_project=allow_cross_project,
     )
     url_str = " from " + url if url else ""
     print(f"Loading project {proj.name}{url_str} into {context}:\n")
@@ -1337,7 +1374,11 @@ def dict_to_str(struct: dict):
     return ",".join([f"{k}={v}" for k, v in struct.items()])
 
 
-def func_url_to_runtime(func_url, ensure_project: bool = False):
+def func_url_to_runtime(
+    func_url,
+    ensure_project: bool = False,
+    allow_cross_project: Optional[bool] = None,
+):
     try:
         if func_url.startswith("db://"):
             func_url = func_url[5:]
@@ -1348,7 +1389,9 @@ def func_url_to_runtime(func_url, ensure_project: bool = False):
             func_url = "function.yaml" if func_url == "." else func_url
             runtime = import_function_to_dict(func_url, {})
         else:
-            mlrun_project = load_project(".", save=ensure_project)
+            mlrun_project = load_project(
+                ".", save=ensure_project, allow_cross_project=allow_cross_project
+            )
             function = mlrun_project.get_function(func_url, enrich=True)
             if function.kind == "local":
                 command, function = load_func_code(function)

mlrun/artifacts/base.py

@@ -223,28 +223,9 @@ class Artifact(ModelObj):
         target_path=None,
         project=None,
         src_path: typing.Optional[str] = None,
-        # All params up until here are legacy params for compatibility with legacy artifacts.
-        # TODO: remove them in 1.10.0.
         metadata: ArtifactMetadata = None,
         spec: ArtifactSpec = None,
     ):
-        if (
-            key
-            or body
-            or viewer
-            or is_inline
-            or format
-            or size
-            or target_path
-            or project
-            or src_path
-        ):
-            warnings.warn(
-                "Artifact constructor parameters are deprecated in 1.7.0 and will be removed in 1.10.0. "
-                "Use the metadata and spec parameters instead.",
-                DeprecationWarning,
-            )
-
         self._metadata = None
         self.metadata = metadata
         self._spec = None
@@ -769,17 +750,9 @@ class LinkArtifact(Artifact):
         link_key=None,
         link_tree=None,
         project=None,
-        # All params up until here are legacy params for compatibility with legacy artifacts.
-        # TODO: remove them in 1.10.0.
         metadata: ArtifactMetadata = None,
         spec: LinkArtifactSpec = None,
     ):
-        if key or target_path or link_iteration or link_key or link_tree or project:
-            warnings.warn(
-                "Artifact constructor parameters are deprecated in 1.7.0 and will be removed in 1.10.0. "
-                "Use the metadata and spec parameters instead.",
-                DeprecationWarning,
-            )
         super().__init__(
             key, target_path=target_path, project=project, metadata=metadata, spec=spec
         )

mlrun/artifacts/dataset.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 import os
 import pathlib
-import warnings
 from io import StringIO
 from typing import Optional
 
@@ -161,13 +160,6 @@ class DatasetArtifact(Artifact):
         label_column: Optional[str] = None,
         **kwargs,
     ):
-        if key or format or target_path:
-            warnings.warn(
-                "Artifact constructor parameters are deprecated in 1.7.0 and will be removed in 1.10.0. "
-                "Use the metadata and spec parameters instead.",
-                DeprecationWarning,
-            )
-
         format = (format or "").lower()
         super().__init__(key, None, format=format, target_path=target_path)
         if format and format not in self.SUPPORTED_FORMATS:

mlrun/artifacts/model.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 
 import tempfile
-import warnings
 from os import path
 from typing import Any, Optional, Union
 
@@ -189,12 +188,6 @@ class ModelArtifact(Artifact):
                                 Saved as a sub-dictionary under the parameter.
         :param kwargs:          Arguments to pass to the artifact class.
         """
-        if key or body or format or target_path:
-            warnings.warn(
-                "Artifact constructor parameters are deprecated in 1.7.0 and will be removed in 1.10.0. "
-                "Use the metadata and spec parameters instead.",
-                DeprecationWarning,
-            )
         super().__init__(key, body, format=format, target_path=target_path, **kwargs)
         model_file = str(model_file or "")
         if model_file and model_url:

mlrun/artifacts/plots.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 import base64
 import typing
-import warnings
 from io import BytesIO
 
 import mlrun
@@ -35,12 +34,6 @@ class PlotArtifact(Artifact):
     def __init__(
         self, key=None, body=None, is_inline=False, target_path=None, title=None
     ):
-        if key or body or is_inline or target_path:
-            warnings.warn(
-                "Artifact constructor parameters are deprecated in 1.7.0 and will be removed in 1.10.0. "
-                "Use the metadata and spec parameters instead.",
-                DeprecationWarning,
-            )
         super().__init__(key, body, format="html", target_path=target_path)
         self.metadata.description = title
 
@@ -94,12 +87,6 @@ class PlotlyArtifact(Artifact):
         :param key:         Key for the artifact to be stored in the database.
         :param target_path: Path to save the artifact.
         """
-        if key or target_path:
-            warnings.warn(
-                "Artifact constructor parameters are deprecated in 1.7.0 and will be removed in 1.10.0. "
-                "Use the metadata and spec parameters instead.",
-                DeprecationWarning,
-            )
         # Validate the plotly package:
         try:
             from plotly.graph_objs import Figure

mlrun/common/schemas/background_task.py

@@ -22,6 +22,10 @@ import mlrun.common.types
 from .object import ObjectKind
 
 
+class BackGroundTaskLabel(mlrun.common.types.StrEnum):
+    pipeline = "pipeline"
+
+
 class BackgroundTaskState(mlrun.common.types.StrEnum):
     succeeded = "succeeded"
     failed = "failed"
@@ -37,6 +41,7 @@ class BackgroundTaskState(mlrun.common.types.StrEnum):
 
 class BackgroundTaskMetadata(pydantic.v1.BaseModel):
     name: str
+    id: typing.Optional[int]
     kind: typing.Optional[str]
     project: typing.Optional[str]
     created: typing.Optional[datetime.datetime]

mlrun/common/schemas/model_monitoring/__init__.py

@@ -28,6 +28,7 @@ from .constants import (
     ModelEndpointCreationStrategy,
     ModelEndpointMonitoringMetricType,
     ModelEndpointSchema,
+    ModelMonitoringAppLabel,
     ModelMonitoringMode,
     MonitoringFunctionNames,
     PredictionsQueryConstants,
@@ -36,6 +37,7 @@ from .constants import (
     ResultKindApp,
     ResultStatusApp,
     SpecialApps,
+    StreamProcessingEvent,
     TDEngineSuperTables,
     TSDBTarget,
     V3IOTSDBTables,

mlrun/common/schemas/model_monitoring/constants.py

@@ -142,6 +142,22 @@ class EventFieldType:
     EFFECTIVE_SAMPLE_COUNT = "effective_sample_count"
 
 
+class StreamProcessingEvent:
+    MODEL = "model"
+    MODEL_CLASS = "model_class"
+    MICROSEC = "microsec"
+    WHEN = "when"
+    ERROR = "error"
+    ENDPOINT_ID = "endpoint_id"
+    SAMPLING_PERCENTAGE = "sampling_percentage"
+    EFFECTIVE_SAMPLE_COUNT = "effective_sample_count"
+    LABELS = "labels"
+    FUNCTION_URI = "function_uri"
+    REQUEST = "request"
+    RESPONSE = "resp"
+    METRICS = "metrics"
+
+
 class FeatureSetFeatures(MonitoringStrEnum):
     LATENCY = EventFieldType.LATENCY
     METRICS = EventFieldType.METRICS

mlrun/common/schemas/project.py

@@ -148,6 +148,10 @@ class ProjectSummary(pydantic.v1.BaseModel):
     datasets_count: int = 0
     documents_count: int = 0
     llm_prompts_count: int = 0
+    running_model_monitoring_functions: int = 0
+    failed_model_monitoring_functions: int = 0
+    real_time_model_endpoint_count: int = 0
+    batch_model_endpoint_count: int = 0
 
 
 class IguazioProject(pydantic.v1.BaseModel):

mlrun/common/schemas/serving.py

@@ -33,7 +33,9 @@ class MonitoringData(StrEnum):
     INPUTS = "inputs"
     OUTPUTS = "outputs"
     INPUT_PATH = "input_path"
+    RESULT_PATH = "result_path"
     CREATION_STRATEGY = "creation_strategy"
     LABELS = "labels"
     MODEL_PATH = "model_path"
     MODEL_ENDPOINT_UID = "model_endpoint_uid"
+    MODEL_CLASS = "model_class"

mlrun/config.py

@@ -107,6 +107,8 @@ default_config = {
     "submit_timeout": "280",  # timeout when submitting a new k8s resource
     # runtimes cleanup interval in seconds
     "runtimes_cleanup_interval": "300",
+    "background_task_cleanup_interval": "86400",  # 24 hours in seconds
+    "background_task_max_age": "21600",  # 6 hours in seconds
     "monitoring": {
         "runs": {
             # runs monitoring interval in seconds
@@ -233,6 +235,7 @@ default_config = {
                 "delete_function": "900",
                 "model_endpoint_creation": "600",
                 "model_endpoint_tsdb_leftovers": "900",
+                "terminate_pipeline": "300",
             },
             "runtimes": {
                 "dask": "600",
@@ -638,6 +641,7 @@ default_config = {
         "offline_storage_path": "model-endpoints/{kind}",
         "parquet_batching_max_events": 10_000,
         "parquet_batching_timeout_secs": timedelta(minutes=1).total_seconds(),
+        "model_endpoint_creation_check_period": "15",
     },
     "secret_stores": {
         # Use only in testing scenarios (such as integration tests) to avoid using k8s for secrets (will use in-memory
@@ -896,11 +900,7 @@ class Config:
         return result
 
     def __setattr__(self, attr, value):
-        # in order for the dbpath setter to work
-        if attr == "dbpath":
-            super().__setattr__(attr, value)
-        else:
-            self._cfg[attr] = value
+        self._cfg[attr] = value
 
     def __dir__(self):
         return list(self._cfg) + dir(self.__class__)
@@ -1244,23 +1244,6 @@ class Config:
         # since the property will need to be url, which exists in other structs as well
         return config.ui.url or config.ui_url
 
-    @property
-    def dbpath(self):
-        return self._dbpath
-
-    @dbpath.setter
-    def dbpath(self, value):
-        self._dbpath = value
-        if value:
-            # importing here to avoid circular dependency
-            import mlrun.db
-
-            # It ensures that SSL verification is set before establishing a connection
-            _configure_ssl_verification(self.httpdb.http.verify)
-
-            # when dbpath is set we want to connect to it which will sync configuration from it to the client
-            mlrun.db.get_run_db(value, force_reconnect=True)
-
     def is_api_running_on_k8s(self):
         # determine if the API service is attached to K8s cluster
         # when there is a cluster the .namespace is set
@@ -1436,6 +1419,12 @@ def _do_populate(env=None, skip_errors=False):
     _configure_ssl_verification(config.httpdb.http.verify)
     _validate_config(config)
 
+    if config.dbpath:
+        from mlrun.db import get_run_db
+
+        # when dbpath is set we want to connect to it which will sync configuration from it to the client
+        get_run_db(config.dbpath, force_reconnect=True)
+
 
 def _validate_config(config):
     try:

mlrun/datastore/utils.py

@@ -236,9 +236,11 @@ class KafkaParameters:
             "partitions": "",
             "sasl": "",
             "worker_allocation_mode": "",
-            "tls_enable": "",  # for Nuclio with Confluent Kafka (Sarama client)
+            # for Nuclio with Confluent Kafka
+            "tls_enable": "",
             "tls": "",
             "new_topic": "",
+            "nuclio_annotations": "",
         }
         self._reference_dicts = (
             self._custom_attributes,

mlrun/db/base.py

@@ -97,9 +97,6 @@ class RunDBInterface(ABC):
         uid: Optional[Union[str, list[str]]] = None,
         project: Optional[str] = None,
         labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
-        state: Optional[
-            mlrun.common.runtimes.constants.RunStates
-        ] = None,  # Backward compatibility
         states: Optional[list[mlrun.common.runtimes.constants.RunStates]] = None,
         sort: bool = True,
         iter: bool = False,
@@ -470,22 +467,6 @@ class RunDBInterface(ABC):
     ) -> mlrun.common.schemas.FeaturesOutputV2:
         pass
 
-    # TODO: remove in 1.10.0
-    @deprecated(
-        version="1.7.0",
-        reason="'list_entities' will be removed in 1.10.0, use 'list_entities_v2' instead",
-        category=FutureWarning,
-    )
-    @abstractmethod
-    def list_entities(
-        self,
-        project: str,
-        name: Optional[str] = None,
-        tag: Optional[str] = None,
-        labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
-    ) -> mlrun.common.schemas.EntitiesOutput:
-        pass
-
     @abstractmethod
     def list_entities_v2(
         self,