supervisely 6.73.407__py3-none-any.whl → 6.73.408__py3-none-any.whl

supervisely/nn/inference/inference.py

@@ -182,6 +182,9 @@ class Inference:
         self.classes: List[str] = None
         self._model_dir = model_dir
         self._model_served = False
+        self._freeze_timer = None
+        self._model_frozen = False
+        self._inactivity_timeout = 3600  # 1 hour
         self._deploy_params: dict = None
         self._model_meta = None
         self._confidence = "confidence"
@@ -225,6 +228,7 @@ class Inference:
                 deploy_params["model_files"] = local_model_files
             logger.debug("Loading model...")
             self._load_model_headless(**deploy_params)
+            self._schedule_freeze_on_inactivity()
 
         if self._use_gui:
             initialize_custom_gui_method = getattr(self, "initialize_custom_gui", None)
@@ -273,6 +277,7 @@ class Inference:
                     self.device = device
                     self.load_on_device(self._model_dir, device)
                     gui.show_deployed_model_info(self)
+                self._schedule_freeze_on_inactivity()
 
             def on_change_model_callback(
                 gui: Union[GUI.InferenceGUI, GUI.ServingGUI, GUI.ServingGUITemplate],
@@ -407,6 +412,7 @@ class Inference:
         deploy_params = self._get_deploy_parameters_from_custom_checkpoint(checkpoint_path, device, runtime)
         logger.debug(f"Deploying custom model '{checkpoint_name}'...")
         self._load_model_headless(**deploy_params)
+        self._schedule_freeze_on_inactivity()
         return self
 
     def get_batch_size(self):
@@ -1228,6 +1234,10 @@ class Inference:
 
     def shutdown_model(self):
         self._model_served = False
+        self._model_frozen = False
+        if self._freeze_timer is not None:
+            self._freeze_timer.cancel()
+            self._freeze_timer = None
         self.device = None
         self.runtime = None
         self.model_precision = None
@@ -1437,12 +1447,13 @@ class Inference:
         if api is None:
             api = self.api
         return api
-
+        
     def _inference_auto(
         self,
         source: List[Union[str, np.ndarray]],
         settings: Dict[str, Any],
     ) -> Tuple[List[Annotation], List[dict]]:
+        self._unfreeze_model()
         inference_mode = settings.get("inference_mode", "full_image")
         use_raw = (
             inference_mode == "sliding_window" and settings["sliding_window_mode"] == "advanced"
@@ -1453,9 +1464,11 @@ class Inference:
         if (not use_raw and self.is_batch_inference_supported()) or (
             use_raw and is_predict_batch_raw_implemented
         ):
-            return self._inference_batched_wrapper(source, settings)
+            result = self._inference_batched_wrapper(source, settings)
         else:
-            return self._inference_one_by_one_wrapper(source, settings)
+            result = self._inference_one_by_one_wrapper(source, settings)
+        self._schedule_freeze_on_inactivity()
+        return result
 
     def inference(
         self,
@@ -2432,9 +2445,7 @@ class Inference:
     def _check_serve_before_call(self, func):
         @wraps(func)
         def wrapper(*args, **kwargs):
-            if self._model_served is True:
-                return func(*args, **kwargs)
-            else:
+            if self._model_served is False:
                 msg = (
                     "The model has not yet been deployed. "
                     "Please select the appropriate model in the UI and press the 'Serve' button. "
@@ -2442,9 +2453,53 @@ class Inference:
                 )
                 # raise DialogWindowError(title="Call undeployed model.", description=msg)
                 raise RuntimeError(msg)
-
+            return func(*args, **kwargs)
         return wrapper
 
+    def _freeze_model(self):
+        if self._model_frozen or not self._model_served:
+            return
+        logger.debug("Freezing model...")
+        runtime = self._deploy_params.get("runtime")
+        if runtime and runtime.lower() != RuntimeType.PYTORCH.lower():
+            logger.debug("Model is not running in PyTorch runtime, cannot freeze.")
+            return
+        previous_device = self._deploy_params.get("device")
+        if previous_device == "cpu":
+            logger.debug("Model is already running on CPU, cannot freeze.")
+            return
+
+        deploy_params = self._deploy_params.copy()
+        deploy_params["device"] = "cpu"
+        try:
+            self._load_model(deploy_params)
+            self._model_frozen = True
+            logger.info(
+                "Model has been re-deployed to CPU for resource optimization. "
+                "It will be loaded back to the original device on the next inference request."
+            )
+        finally:
+            self._deploy_params["device"] = previous_device
+            clean_up_cuda()
+
+    def _unfreeze_model(self):
+        if not self._model_frozen:
+            return
+        logger.debug("Unfreezing model...")
+        self._model_frozen = False
+        self._load_model(self._deploy_params)
+        clean_up_cuda()
+        logger.debug("Model is unfrozen and ready for inference.")
+
+    def _schedule_freeze_on_inactivity(self):
+        if self._freeze_timer is not None:
+            self._freeze_timer.cancel()
+        timer = threading.Timer(self._inactivity_timeout, self._freeze_model)
+        timer.daemon = True
+        timer.start()
+        self._freeze_timer = timer
+        logger.debug("Model will be frozen in %s seconds due to inactivity.", self._inactivity_timeout)
+
     def _set_served_callback(self):
         self._model_served = True
 
@@ -2506,6 +2561,7 @@ class Inference:
             # update to set correct device
             device = deploy_params.get("device", "cpu")
             self.gui.set_deployed(device)
+            self._schedule_freeze_on_inactivity()
             return {"result": "model was successfully deployed"}
         except Exception as e:
             self.gui._success_label.hide()
@@ -3400,6 +3456,8 @@ class Inference:
                 if self.gui is not None:
                     self.gui._success_label.hide()
                 raise e
+            finally:
+                self._schedule_freeze_on_inactivity()
 
         @server.post("/list_pretrained_models")
         def _list_pretrained_models():
@@ -3479,6 +3537,16 @@ class Inference:
                 },
             }
 
+        @server.post("/freeze_model")
+        def _freeze_model(request: Request):
+            if self._model_frozen:
+                return {"message": "Model is already frozen."}
+
+            self._freeze_model()
+            if not self._model_frozen:
+                return {"message": "Failed to freeze model. Check the logs for details."}
+            return {"message": "Model is frozen."}
+
         # Local deploy without predict args
         if self._is_cli_deploy:
             self._run_server()

supervisely/nn/model/model_api.py

@@ -111,6 +111,12 @@ class ModelAPI:
         response = self._post("tasks.stop", {ApiField.ID: id})
         return TaskApi.Status(response[ApiField.STATUS])
 
+    def freeze_model(self):
+        """Freeze the model to free up resources."""
+        if self.task_id is not None:
+            return self.api.task.send_request(self.task_id, "freeze_model", {})
+        return self._post("freeze_model", {})
+
     # --------------------- #
 
     # region Load

{supervisely-6.73.407.dist-info → supervisely-6.73.408.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: supervisely
-Version: 6.73.407
+Version: 6.73.408
 Summary: Supervisely Python SDK.
 Home-page: https://github.com/supervisely/supervisely
 Author: Supervisely

{supervisely-6.73.407.dist-info → supervisely-6.73.408.dist-info}/RECORD

@@ -893,7 +893,7 @@ supervisely/nn/benchmark/visualization/widgets/table/__init__.py,sha256=47DEQpj8
 supervisely/nn/benchmark/visualization/widgets/table/table.py,sha256=atmDnF1Af6qLQBUjLhK18RMDKAYlxnsuVHMSEa5a-e8,4319
 supervisely/nn/inference/__init__.py,sha256=QFukX2ip-U7263aEPCF_UCFwj6EujbMnsgrXp5Bbt8I,1623
 supervisely/nn/inference/cache.py,sha256=rfmb1teJ9lNDfisUSh6bwDCVkPZocn8GMvDgLQktnbo,35023
-supervisely/nn/inference/inference.py,sha256=L14M8qGofz6EGrgJvGMcz0h3vsu-AnHPeL_iUO8Kf3Y,195721
+supervisely/nn/inference/inference.py,sha256=7c2-GuG3MgI5H0muTyxoR-XgUzWpbP9if37CyRewobA,198528
 supervisely/nn/inference/inference_request.py,sha256=y6yw0vbaRRcEBS27nq3y0sL6Gmq2qLA_Bm0GrnJGegE,14267
 supervisely/nn/inference/session.py,sha256=dIg2F-OBl68pUzcmtmcI0YQIp1WWNnrJTVMjwFN91Q4,35824
 supervisely/nn/inference/uploader.py,sha256=21a9coOimCHhEqAbV-llZWcp12847DEMoQp3N16bpK0,5425
@@ -968,7 +968,7 @@ supervisely/nn/legacy/pytorch/weights.py,sha256=Zb9kcpUCg6ykr7seO53CkKSQa2K44wo8
 supervisely/nn/legacy/training/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 supervisely/nn/legacy/training/eval_planner.py,sha256=zN9b0_CX7sWGdC8e6riTvD-NOUc3_Xduyhj00S7PEIo,1311
 supervisely/nn/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-supervisely/nn/model/model_api.py,sha256=rIne6ymvL2aVwzSTJMGxAbevE0U-f-WKhDciqaneqRc,9666
+supervisely/nn/model/model_api.py,sha256=rq-08BrQmWKqUxWnBB5yKBfnjxw8Lg88mhva848Ae2I,9911
 supervisely/nn/model/prediction.py,sha256=N3oO9s3NDiC5CFvW8utfU8rz3bfpCl37Sk4VEBH94Bc,11307
 supervisely/nn/model/prediction_session.py,sha256=sy0FSQaWSmT8i0RkR4J8oIn3Ek4IDVJNBR1Tg4mulkM,25523
 supervisely/nn/tracker/__init__.py,sha256=LiojByb5kGsTQ49lWuboEh7B4JUwM1vfz81J8kJlLYo,337
@@ -1114,9 +1114,9 @@ supervisely/worker_proto/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZ
 supervisely/worker_proto/worker_api_pb2.py,sha256=VQfi5JRBHs2pFCK1snec3JECgGnua3Xjqw_-b3aFxuM,59142
 supervisely/worker_proto/worker_api_pb2_grpc.py,sha256=3BwQXOaP9qpdi0Dt9EKG--Lm8KGN0C5AgmUfRv77_Jk,28940
 supervisely_lib/__init__.py,sha256=7-3QnN8Zf0wj8NCr2oJmqoQWMKKPKTECvjH9pd2S5vY,159
-supervisely-6.73.407.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-supervisely-6.73.407.dist-info/METADATA,sha256=fIR2z5kCfeQzldtPPODSuq8z4QQhOoTVLgeZRZokLtU,35254
-supervisely-6.73.407.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
-supervisely-6.73.407.dist-info/entry_points.txt,sha256=U96-5Hxrp2ApRjnCoUiUhWMqijqh8zLR03sEhWtAcms,102
-supervisely-6.73.407.dist-info/top_level.txt,sha256=kcFVwb7SXtfqZifrZaSE3owHExX4gcNYe7Q2uoby084,28
-supervisely-6.73.407.dist-info/RECORD,,
+supervisely-6.73.408.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+supervisely-6.73.408.dist-info/METADATA,sha256=C6yRG5xZiYpAXpe3RyAIimpbPRggSn--0fqKT00sSKI,35254
+supervisely-6.73.408.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
+supervisely-6.73.408.dist-info/entry_points.txt,sha256=U96-5Hxrp2ApRjnCoUiUhWMqijqh8zLR03sEhWtAcms,102
+supervisely-6.73.408.dist-info/top_level.txt,sha256=kcFVwb7SXtfqZifrZaSE3owHExX4gcNYe7Q2uoby084,28
+supervisely-6.73.408.dist-info/RECORD,,