mlrun 1.10.0rc9__tar.gz → 1.10.0rc10__tar.gz

{mlrun-1.10.0rc9 → mlrun-1.10.0rc10}/LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright 2020 Iguazio
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

{mlrun-1.10.0rc9 → mlrun-1.10.0rc10}/Makefile

@@ -35,7 +35,7 @@ PYTHON_VERSION ?= $(shell python --version)
 MLRUN_SKIP_COMPILE_SCHEMAS ?=
 INCLUDE_PYTHON_VERSION_SUFFIX ?=
 MLRUN_PIP_VERSION ?= 25.0.0
-MLRUN_UV_VERSION ?= 0.7.12
+MLRUN_UV_VERSION ?= 0.7.14
 MLRUN_UV_IMAGE ?= ghcr.io/astral-sh/uv:$(MLRUN_UV_VERSION)
 MLRUN_CACHE_DATE ?= $(shell date +%s)
 # empty by default, can be set to something like "tag-name" which will cause to:
@@ -936,19 +936,9 @@ ifdef MLRUN_DOCKER_CACHE_FROM_TAG
 	done;
 endif
 
-.PHONY: verify-uv-version
-verify-uv-version:
-	@{ \
-	uv_version=$$(uv self version | cut -d' ' -f2); \
-	result=$$(python -m semver compare $$uv_version $(MLRUN_UV_VERSION)); \
-	if [ "$$result" -eq -1 ]; then \
-	  echo "Error: The running uv version ($$uv_version) is outdated. Upgrade uv to version $(MLRUN_UV_VERSION)."; \
-	  exit 1; \
-	fi; \
-	}
 
 .PHONY: upgrade-mlrun-api-deps-lock
-upgrade-mlrun-api-deps-lock: verify-uv-version ## Upgrade mlrun-api locked requirements file
+upgrade-mlrun-api-deps-lock: ## Upgrade mlrun-api locked requirements file
 	uv pip compile \
 		requirements.txt \
 		extras-requirements.txt \
@@ -958,7 +948,7 @@ upgrade-mlrun-api-deps-lock: verify-uv-version ## Upgrade mlrun-api locked requi
 		--output-file dockerfiles/mlrun-api/locked-requirements.txt
 
 .PHONY: upgrade-mlrun-mlrun-deps-lock
-upgrade-mlrun-mlrun-deps-lock: verify-uv-version ## Upgrade mlrun-mlrun locked requirements file
+upgrade-mlrun-mlrun-deps-lock: ## Upgrade mlrun-mlrun locked requirements file
 	uv pip compile \
 		requirements.txt \
 		extras-requirements.txt \
@@ -967,7 +957,7 @@ upgrade-mlrun-mlrun-deps-lock: verify-uv-version ## Upgrade mlrun-mlrun locked r
 		--output-file dockerfiles/mlrun/locked-requirements.txt
 
 .PHONY: upgrade-mlrun-gpu-deps-lock
-upgrade-mlrun-gpu-deps-lock: verify-uv-version ## Upgrade mlrun-gpu locked requirements file
+upgrade-mlrun-gpu-deps-lock: ## Upgrade mlrun-gpu locked requirements file
 	uv pip compile \
 		requirements.txt \
 		extras-requirements.txt \
@@ -976,7 +966,7 @@ upgrade-mlrun-gpu-deps-lock: verify-uv-version ## Upgrade mlrun-gpu locked requi
 		--output-file dockerfiles/gpu/locked-requirements.txt
 
 .PHONY: upgrade-mlrun-jupyter-deps-lock
-upgrade-mlrun-jupyter-deps-lock: verify-uv-version ## Upgrade mlrun-jupyter locked requirements file
+upgrade-mlrun-jupyter-deps-lock: ## Upgrade mlrun-jupyter locked requirements file
 	uv pip compile \
 		requirements.txt \
 		extras-requirements.txt \
@@ -986,7 +976,7 @@ upgrade-mlrun-jupyter-deps-lock: verify-uv-version ## Upgrade mlrun-jupyter lock
 		--output-file dockerfiles/jupyter/locked-requirements.txt
 
 .PHONY: upgrade-mlrun-test-deps-lock
-upgrade-mlrun-test-deps-lock: verify-uv-version ## Upgrade mlrun test locked requirements file
+upgrade-mlrun-test-deps-lock: ## Upgrade mlrun test locked requirements file
 	uv pip compile \
 		requirements.txt \
 		extras-requirements.txt \
@@ -997,7 +987,7 @@ upgrade-mlrun-test-deps-lock: verify-uv-version ## Upgrade mlrun test locked req
 		--output-file dockerfiles/test/locked-requirements.txt
 
 .PHONY: upgrade-mlrun-system-test-deps-lock
-upgrade-mlrun-system-test-deps-lock: verify-uv-version ## Upgrade mlrun system test locked requirements file
+upgrade-mlrun-system-test-deps-lock: ## Upgrade mlrun system test locked requirements file
 	uv pip compile \
 		requirements.txt \
 		extras-requirements.txt \
@@ -1007,8 +997,7 @@ upgrade-mlrun-system-test-deps-lock: verify-uv-version ## Upgrade mlrun system t
 		$(MLRUN_UV_UPGRADE_FLAG) \
 		--output-file dockerfiles/test-system/locked-requirements.txt
 
-
-upgrade-mlrun-kfp-deps-lock: verify-uv-version ## Upgrade mlrun-kfp locked requirements file
+upgrade-mlrun-kfp-deps-lock: ## Upgrade mlrun-kfp locked requirements file
 	uv pip compile \
 		requirements.txt \
 		dockerfiles/mlrun-kfp/requirements.txt \
@@ -1017,7 +1006,7 @@ upgrade-mlrun-kfp-deps-lock: verify-uv-version ## Upgrade mlrun-kfp locked requi
 		--output-file dockerfiles/mlrun-kfp/locked-requirements.txt
 
 .PHONY: upgrade-mlrun-deps-lock
-upgrade-mlrun-deps-lock: verify-uv-version ## Upgrade mlrun-* locked requirements file
+upgrade-mlrun-deps-lock: ## Upgrade mlrun-* locked requirements file
 	@$(MAKE) -j \
 		upgrade-mlrun-mlrun-deps-lock \
 		upgrade-mlrun-api-deps-lock \
@@ -1027,6 +1016,7 @@ upgrade-mlrun-deps-lock: verify-uv-version ## Upgrade mlrun-* locked requirement
 		upgrade-mlrun-test-deps-lock \
 		upgrade-mlrun-system-test-deps-lock
 
+
 .PHONY: coverage-combine
 coverage-combine: ## Combine all coverage reports, ignoring errors like missing or corrupted source files
 	rm -f tests/coverage_reports/combined.coverage; \

{mlrun-1.10.0rc9/mlrun.egg-info → mlrun-1.10.0rc10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mlrun
-Version: 1.10.0rc9
+Version: 1.10.0rc10
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv
@@ -9,7 +9,6 @@ License: Apache License 2.0
 Keywords: mlrun,mlops,data-science,machine-learning,experiment-tracking
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
-Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: POSIX :: Linux
 Classifier: Operating System :: Microsoft :: Windows
 Classifier: Operating System :: MacOS

{mlrun-1.10.0rc9 → mlrun-1.10.0rc10}/dependencies.py

@@ -19,10 +19,6 @@ def base_requirements() -> list[str]:
     return list(_load_dependencies_from_file("requirements.txt"))
 
 
-def dev_requirements() -> list[str]:
-    return list(_load_dependencies_from_file("dev-requirements.txt"))
-
-
 def extra_requirements() -> dict[str, list[str]]:
     # NOTE:
     #     - These are tested in `automation/package_test/test.py`. If you modify these, make sure to change the

{mlrun-1.10.0rc9 → mlrun-1.10.0rc10}/mlrun/common/constants.py

@@ -76,6 +76,7 @@ class MLRunInternalLabels:
     kind = "kind"
     component = "component"
     mlrun_type = "mlrun__type"
+    original_workflow_id = "original-workflow-id"
 
     owner = "owner"
     v3io_user = "v3io_user"

{mlrun-1.10.0rc9 → mlrun-1.10.0rc10}/mlrun/common/schemas/__init__.py

@@ -218,6 +218,7 @@ from .serving import ModelRunnerStepData, MonitoringData
 from .tag import Tag, TagObjects
 from .workflow import (
     GetWorkflowResponse,
+    RerunWorkflowRequest,
     WorkflowRequest,
     WorkflowResponse,
     WorkflowSpec,

{mlrun-1.10.0rc9 → mlrun-1.10.0rc10}/mlrun/common/schemas/workflow.py

@@ -46,6 +46,13 @@ class WorkflowRequest(pydantic.v1.BaseModel):
     notifications: typing.Optional[list[Notification]] = None
 
 
+class RerunWorkflowRequest(pydantic.v1.BaseModel):
+    run_name: typing.Optional[str] = (None,)
+    run_id: typing.Optional[str] = (None,)
+    notifications: typing.Optional[list[Notification]] = None
+    workflow_runner_node_selector: typing.Optional[dict[str, str]] = None
+
+
 class WorkflowResponse(pydantic.v1.BaseModel):
     project: str = None
     name: str = None

{mlrun-1.10.0rc9 → mlrun-1.10.0rc10}/mlrun/db/base.py

@@ -638,6 +638,16 @@ class RunDBInterface(ABC):
     ):
         pass
 
+    @abstractmethod
+    def retry_pipeline(
+        self,
+        run_id: str,
+        project: str,
+        namespace: Optional[str] = None,
+        timeout: int = 30,
+    ):
+        pass
+
     @abstractmethod
     def list_project_secrets(
         self,
@@ -1034,6 +1044,13 @@ class RunDBInterface(ABC):
     ):
         pass
 
+    def get_project_background_task(
+        self,
+        project: str,
+        name: str,
+    ) -> mlrun.common.schemas.BackgroundTask:
+        pass
+
     @abstractmethod
     def submit_workflow(
         self,

{mlrun-1.10.0rc9 → mlrun-1.10.0rc10}/mlrun/db/nopdb.py

@@ -524,6 +524,15 @@ class NopDB(RunDBInterface):
     ):
         pass
 
+    def retry_pipeline(
+        self,
+        run_id: str,
+        project: str,
+        namespace: Optional[str] = None,
+        timeout: int = 30,
+    ):
+        pass
+
     def list_pipelines(
         self,
         project: str,

{mlrun-1.10.0rc9 → mlrun-1.10.0rc10}/mlrun/frameworks/tf_keras/mlrun_interface.py

@@ -280,7 +280,7 @@ class TFKerasMLRunInterface(MLRunInterface, ABC):
             print(f"Horovod worker #{self._hvd.rank()} is using CPU")
 
         # Adjust learning rate based on the number of GPUs:
-        if hasattr(self.optimizer, "lr"):
+        if hasattr(optimizer, "lr"):
             optimizer.lr *= self._hvd.size()
         else:
             optimizer.learning_rate *= self._hvd.size()

{mlrun-1.10.0rc9 → mlrun-1.10.0rc10}/mlrun/frameworks/tf_keras/model_handler.py

@@ -518,7 +518,6 @@ class TFKerasModelHandler(DLModelHandler):
         )
 
         # Read additional files according to the model format used:
-        # # ModelFormats.SAVED_MODEL - Unzip the SavedModel archive:
         if self._model_format == TFKerasModelHandler.ModelFormats.SAVED_MODEL:
             # Unzip the SavedModel directory:
             with zipfile.ZipFile(self._model_file, "r") as zip_file:
@@ -528,21 +527,17 @@ class TFKerasModelHandler(DLModelHandler):
                 os.path.dirname(self._model_file), self._model_name
             )
         elif self._model_format == TFKerasModelHandler.ModelFormats.KERAS:
-            # When keras tried to load it, it validates the suffix. The `artifacts.model.get_model` function is
-            # downloading the keras file to a temp file with a `pkl` suffix, so it needs to be replaced:
-            self._model_file = self._model_file.rsplit(".pkl", 1)[0] + ".keras"
+            # Rename the model file suffix:
+            self._rename_model_file_suffix(suffix="keras")
         elif self._model_format == TFKerasModelHandler.ModelFormats.H5:
-            # When keras tried to load it, it validates the suffix. The `artifacts.model.get_model` function is
-            # downloading the keras file to a temp file with a `pkl` suffix, so it needs to be replaced:
-            self._model_file = self._model_file.rsplit(".pkl", 1)[0] + ".h5"
-        # # ModelFormats.JSON_ARCHITECTURE_H5_WEIGHTS - Get the weights file:
-        elif (
+            # Rename the model file suffix:
+            self._rename_model_file_suffix(suffix="h5")
+        elif (  # ModelFormats.JSON_ARCHITECTURE_H5_WEIGHTS
             self._model_format
             == TFKerasModelHandler.ModelFormats.JSON_ARCHITECTURE_H5_WEIGHTS
         ):
-            # When keras tried to load it, it validates the suffix. The `artifacts.model.get_model` function is
-            # downloading the keras file to a temp file with a `pkl` suffix, so it needs to be replaced:
-            self._model_file = self._model_file.rsplit(".pkl", 1)[0] + ".json"
+            # Rename the model file suffix:
+            self._rename_model_file_suffix(suffix="json")
             # Get the weights file:
             self._weights_file = self._extra_data[
                 self._get_weights_file_artifact_name()
@@ -551,6 +546,20 @@ class TFKerasModelHandler(DLModelHandler):
         # Continue collecting from abstract class:
         super()._collect_files_from_store_object()
 
+    def _rename_model_file_suffix(self, suffix: str):
+        """
+        Rename the model file suffix to the given one.
+
+        This is used for the case of loading a model from a store object that was saved with a different suffix as when
+        keras tries to load it, it validates the suffix. The `artifacts.model.get_model` function is downloading the
+        file to a temp file with a `pkl` suffix, so it needs to be replaced:than the one keras expects.
+
+        :param suffix: The suffix to rename the model file to (without the trailing dot).
+        """
+        new_name = self._model_file.rsplit(".", 1)[0] + f".{suffix}"
+        os.rename(self._model_file, new_name)
+        self._model_file = new_name
+
     def _collect_files_from_local_path(self):
         """
         If the model path given is of a local path, search for the needed model files and collect them into this handler

{mlrun-1.10.0rc9 → mlrun-1.10.0rc10}/mlrun/projects/__init__.py

@@ -32,6 +32,7 @@ from .pipelines import (
     load_and_run_workflow,
     load_and_run,
     pipeline_context,
+    rerun_workflow,
 )  # noqa
 from .project import (
     MlrunProject,

{mlrun-1.10.0rc9 → mlrun-1.10.0rc10}/mlrun/projects/pipelines.py

@@ -1070,6 +1070,40 @@ def github_webhook(request):
     return {"msg": "pushed"}
 
 
+def rerun_workflow(
+    context: mlrun.execution.MLClientCtx, run_uid: str, project_name: str
+):
+    """
+    Re-run a workflow by retrying a previously failed KFP pipeline.
+
+    :param context:      MLRun context.
+    :param run_uid:      The run UID of the original workflow to retry.
+    :param project_name: The project name.
+    """
+
+    try:
+        # TODO in followups: handle start and running notifications
+
+        # Retry the pipeline  - TODO: add submit-direct flag when created
+        db = mlrun.get_run_db()
+        new_pipeline_id = db.retry_pipeline(run_uid, project_name)
+
+        # Store result for observability
+        context.set_label("workflow-id", new_pipeline_id)
+        context.log_result("workflow_id", new_pipeline_id)
+
+        # wait for pipeline completion so monitor will push terminal notifications
+        wait_for_pipeline_completion(
+            new_pipeline_id,
+            project=project_name,
+        )
+
+    # Temporary exception
+    except Exception as exc:
+        context.logger.error("Failed to rerun workflow", exc=err_to_str(exc))
+        raise
+
+
 def load_and_run(context, *args, **kwargs):
     """
     This function serves as an alias to `load_and_run_workflow`,

{mlrun-1.10.0rc9 → mlrun-1.10.0rc10}/mlrun/serving/routers.py

@@ -80,10 +80,16 @@ class BaseModelRouter(RouterToDict):
         self._input_path = input_path
         self._result_path = result_path
         self._background_task_check_timestamp = None
-        self._background_task_terminate = False
         self._background_task_current_state = None
         self.kwargs = kwargs
 
+    @property
+    def background_task_reached_terminal_state(self):
+        return (
+            self._background_task_current_state
+            and self._background_task_current_state != "running"
+        )
+
     def parse_event(self, event):
         parsed_event = {}
         try:
@@ -185,35 +191,33 @@ class BaseModelRouter(RouterToDict):
                     background_task.status.state
                     in mlrun.common.schemas.BackgroundTaskState.terminal_states()
                 ):
-                    logger.debug(
+                    logger.info(
                         f"Model endpoint creation task completed with state {background_task.status.state}"
                     )
-                    self._background_task_terminate = True
                 else:  # in progress
-                    logger.debug(
+                    logger.info(
                         f"Model endpoint creation task is still in progress with the current state: "
-                        f"{background_task.status.state}. Events will not be monitored for the next 15 seconds",
+                        f"{background_task.status.state}. Events will not be monitored for the next "
+                        f"{mlrun.mlconf.model_endpoint_monitoring.model_endpoint_creation_check_period} seconds",
                         name=self.name,
                         background_task_check_timestamp=self._background_task_check_timestamp.isoformat(),
                     )
                 return background_task.status.state
             else:
-                logger.debug(
-                    "Model endpoint creation task name not provided",
+                logger.error(
+                    "Model endpoint creation task name not provided. This function is not being monitored.",
                 )
         elif self.context.monitoring_mock:
-            self._background_task_terminate = (
-                True  # If mock monitoring we return success and terminate task check.
-            )
             return mlrun.common.schemas.BackgroundTaskState.succeeded
-        self._background_task_terminate = True  # If mock without monitoring we return failed and terminate task check.
         return mlrun.common.schemas.BackgroundTaskState.failed
 
     def _update_background_task_state(self, event):
-        if not self._background_task_terminate and (
+        if not self.background_task_reached_terminal_state and (
             self._background_task_check_timestamp is None
             or now_date() - self._background_task_check_timestamp
-            >= timedelta(seconds=15)
+            >= timedelta(
+                seconds=mlrun.mlconf.model_endpoint_monitoring.model_endpoint_creation_check_period
+            )
         ):
             self._background_task_current_state = self._get_background_task_status()
         if event.body:

{mlrun-1.10.0rc9 → mlrun-1.10.0rc10}/mlrun/serving/system_steps.py

@@ -314,7 +314,8 @@ class BackgroundTaskStatus(storey.MapClass):
         else:  # in progress
             logger.info(
                 f"Model endpoint creation task is still in progress with the current state: "
-                f"{background_task_state}. Events will not be monitored for the next 15 seconds",
+                f"{background_task_state}. Events will not be monitored for the next "
+                f"{mlrun.mlconf.model_endpoint_monitoring.model_endpoint_creation_check_period} seconds",
                 name=self.name,
                 background_task_check_timestamp=self._background_task_check_timestamp.isoformat(),
             )

{mlrun-1.10.0rc9 → mlrun-1.10.0rc10}/mlrun/serving/v2_serving.py

@@ -508,8 +508,8 @@ class V2ModelServer(StepToDict):
                         name=self.name,
                     )
             else:
-                logger.debug(
-                    "Model endpoint creation task name not provided",
+                logger.error(
+                    "Model endpoint creation task name not provided. This function is not being monitored.",
                 )
 
 

mlrun-1.10.0rc10/mlrun/utils/version/version.json

@@ -0,0 +1,4 @@
+{
+  "git_commit": "7711525d2af418d6991128a0d3253094584ddedc",
+  "version": "1.10.0-rc10"
+}

{mlrun-1.10.0rc9 → mlrun-1.10.0rc10/mlrun.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mlrun
-Version: 1.10.0rc9
+Version: 1.10.0rc10
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv
@@ -9,7 +9,6 @@ License: Apache License 2.0
 Keywords: mlrun,mlops,data-science,machine-learning,experiment-tracking
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
-Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: POSIX :: Linux
 Classifier: Operating System :: Microsoft :: Windows
 Classifier: Operating System :: MacOS

{mlrun-1.10.0rc9 → mlrun-1.10.0rc10}/pyproject.toml

@@ -39,6 +39,9 @@ author = "Iguazio"
 [tool.ruff.format]
 docstring-code-format = true
 
+[tool.uv]
+required-version = ">=0.7.14"
+
 [tool.uv.pip]
 python-version = "3.9"  # TODO: change to 3.11 when 3.9 is no longer supported
 generate-hashes = true

{mlrun-1.10.0rc9 → mlrun-1.10.0rc10}/setup.py

@@ -69,12 +69,10 @@ setup(
     ],
     python_requires=">=3.9, <3.12",
     install_requires=dependencies.base_requirements(),
-    tests_require=dependencies.dev_requirements(),
     extras_require=dependencies.extra_requirements(),
     classifiers=[
         "Development Status :: 4 - Beta",
         "Intended Audience :: Developers",
-        "License :: OSI Approved :: Apache Software License",
         "Operating System :: POSIX :: Linux",
         "Operating System :: Microsoft :: Windows",
         "Operating System :: MacOS",

mlrun-1.10.0rc9/mlrun/utils/version/version.json

@@ -1,4 +0,0 @@
-{
-  "git_commit": "15a29118a80d3da0d4438ad82c31f14af981fccc",
-  "version": "1.10.0-rc9"
-}