quadra 2.4.0a0__tar.gz → 2.5.0__tar.gz

{quadra-2.4.0a0 → quadra-2.5.0}/PKG-INFO

@@ -1,8 +1,9 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: quadra
-Version: 2.4.0a0
+Version: 2.5.0
 Summary: Deep Learning experiment orchestration library
 License: Apache-2.0
+License-File: LICENSE
 Keywords: deep learning,experiment,lightning,hydra-core
 Author: Federico Belotti
 Author-email: federico.belotti@orobix.com

{quadra-2.4.0a0 → quadra-2.5.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "quadra"
-version = "2.4.0a0"
+version = "2.5.0"
 description = "Deep Learning experiment orchestration library"
 authors = [
 	"Federico Belotti <federico.belotti@orobix.com>",

{quadra-2.4.0a0 → quadra-2.5.0}/quadra/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "2.4.0a0"
+__version__ = "2.5.0"
 
 
 def get_version():

{quadra-2.4.0a0 → quadra-2.5.0}/quadra/configs/core/default.yaml

@@ -9,3 +9,4 @@ experiment_path: null
 upload_artifacts: False
 upload_models: ${export.types} # Default behavior in quadra <= 1.5.6
 log_level: info
+mlflow_zip_models: False

{quadra-2.4.0a0 → quadra-2.5.0}/quadra/tasks/classification.py

@@ -307,6 +307,14 @@ class Classification(Generic[ClassificationDataModuleT], LightningTask[Classific
         # TODO: What happens if we have 64 precision?
         half_precision = "16" in self.trainer.precision
 
+        example_input: torch.Tensor | None = None
+
+        if hasattr(self.trainer, "datamodule") and hasattr(self.trainer.datamodule, "val_dataset"):
+            # Retrieve a better input to evaluate fp16 performance or efficientnetb0 does not sometimes export properly
+            example_input = self.trainer.datamodule.val_dataset[0][0]
+
+        # Selected rtol and atol are quite high, this is mostly done for efficientnetb0 that seems to be
+        # quite unstable in fp16
         self.model_json, export_paths = export_model(
             config=self.config,
             model=module.model,
@@ -314,6 +322,9 @@ class Classification(Generic[ClassificationDataModuleT], LightningTask[Classific
             half_precision=half_precision,
             input_shapes=input_shapes,
             idx_to_class=idx_to_class,
+            example_inputs=example_input,
+            rtol=0.05,
+            atol=0.01,
         )
 
         if len(export_paths) == 0:
@@ -1136,7 +1147,7 @@ class ClassificationEvaluation(Evaluation[ClassificationDataModuleT]):
             return
 
         if isinstance(self.deployment_model.model.features_extractor, timm.models.resnet.ResNet):
-            target_layers = [cast(BaseNetworkBuilder, self.deployment_model.model).features_extractor.layer4[-1]]
+            target_layers = [cast(BaseNetworkBuilder, self.deployment_model.model).features_extractor.layer4[-1]]  # type: ignore[index]
             self.cam = GradCAM(
                 model=self.deployment_model.model,
                 target_layers=target_layers,

{quadra-2.4.0a0 → quadra-2.5.0}/quadra/utils/export.py

@@ -119,6 +119,7 @@ def export_torchscript_model(
     input_shapes: list[Any] | None = None,
     half_precision: bool = False,
     model_name: str = "model.pt",
+    example_inputs: list[torch.Tensor] | tuple[torch.Tensor, ...] | torch.Tensor | None = None,
 ) -> tuple[str, Any] | None:
     """Export a PyTorch model with TorchScript.
 
@@ -128,6 +129,8 @@ def export_torchscript_model(
         output_path: Path to save the model
         half_precision: If True, the model will be exported with half precision
         model_name: Name of the exported model
+        example_inputs: If provided use this to evaluate the model instead of generating random inputs, it's expected to
+            be a list of tensors or a single tensor without batch dimension
 
     Returns:
         If the model is exported successfully, the path to the model and the input shape are returned.
@@ -144,7 +147,32 @@ def export_torchscript_model(
     else:
         model.cpu()
 
-    model_inputs = extract_torch_model_inputs(model, input_shapes, half_precision)
+    batch_size = 1
+    model_inputs: tuple[list[Any] | tuple[Any, ...] | torch.Tensor, list[Any]] | None
+    if example_inputs is not None:
+        if isinstance(example_inputs, Sequence):
+            model_input_tensors = []
+            model_input_shapes = []
+
+            for example_input in example_inputs:
+                new_inp = example_input.to(
+                    device="cuda:0" if half_precision else "cpu",
+                    dtype=torch.float16 if half_precision else torch.float32,
+                )
+                new_inp = new_inp.unsqueeze(0).repeat(batch_size, *(1 for x in new_inp.shape))
+                model_input_tensors.append(new_inp)
+                model_input_shapes.append(new_inp[0].shape)
+
+            model_inputs = (model_input_tensors, [model_input_shapes])
+        else:
+            new_inp = example_inputs.to(
+                device="cuda:0" if half_precision else "cpu",
+                dtype=torch.float16 if half_precision else torch.float32,
+            )
+            new_inp = new_inp.unsqueeze(0).repeat(batch_size, *(1 for x in new_inp.shape))
+            model_inputs = (new_inp, [new_inp[0].shape])
+    else:
+        model_inputs = extract_torch_model_inputs(model, input_shapes, half_precision)
 
     if model_inputs is None:
         return None
@@ -182,6 +210,9 @@ def export_onnx_model(
     input_shapes: list[Any] | None = None,
     half_precision: bool = False,
     model_name: str = "model.onnx",
+    example_inputs: list[torch.Tensor] | tuple[torch.Tensor, ...] | torch.Tensor | None = None,
+    rtol: float = 0.01,
+    atol: float = 5e-3,
 ) -> tuple[str, Any] | None:
     """Export a PyTorch model with ONNX.
 
@@ -192,6 +223,10 @@ def export_onnx_model(
         onnx_config: ONNX export configuration
         half_precision: If True, the model will be exported with half precision
         model_name: Name of the exported model
+        example_inputs: If provided use this to evaluate the model instead of generating random inputs, it's expected to
+            be a list of tensors or a single tensor without batch dimension
+        rtol: Relative tolerance for the ONNX safe export in fp16
+        atol: Absolute tolerance for the ONNX safe export in fp16
     """
     if not ONNX_AVAILABLE:
         log.warning("ONNX is not installed, can not export model in this format.")
@@ -210,9 +245,32 @@ def export_onnx_model(
     else:
         batch_size = 1
 
-    model_inputs = extract_torch_model_inputs(
-        model=model, input_shapes=input_shapes, half_precision=half_precision, batch_size=batch_size
-    )
+    model_inputs: tuple[list[Any] | tuple[Any, ...] | torch.Tensor, list[Any]] | None
+    if example_inputs is not None:
+        if isinstance(example_inputs, Sequence):
+            model_input_tensors = []
+            model_input_shapes = []
+
+            for example_input in example_inputs:
+                new_inp = example_input.to(
+                    device="cuda:0" if half_precision else "cpu",
+                    dtype=torch.float16 if half_precision else torch.float32,
+                )
+                new_inp = new_inp.unsqueeze(0).repeat(batch_size, *(1 for x in new_inp.shape))
+                model_input_tensors.append(new_inp)
+                model_input_shapes.append(new_inp[0].shape)
+
+            model_inputs = (model_input_tensors, [model_input_shapes])
+        else:
+            new_inp = example_inputs.to(
+                device="cuda:0" if half_precision else "cpu",
+                dtype=torch.float16 if half_precision else torch.float32,
+            )
+            new_inp = new_inp.unsqueeze(0).repeat(batch_size, *(1 for x in new_inp.shape))
+            model_inputs = ([new_inp], [new_inp[0].shape])
+    else:
+        model_inputs = extract_torch_model_inputs(model, input_shapes, half_precision)
+
     if model_inputs is None:
         return None
 
@@ -266,6 +324,8 @@ def export_onnx_model(
 
     if isinstance(inp, list):
         inp = tuple(inp)  # onnx doesn't like lists representing tuples of inputs
+    elif isinstance(inp, torch.Tensor):
+        inp = (inp,)
 
     if isinstance(inp, dict):
         raise ValueError("ONNX export does not support model with dict inputs")
@@ -290,6 +350,8 @@ def export_onnx_model(
             onnx_config=onnx_config,
             input_shapes=input_shapes,
             input_names=input_names,
+            rtol=rtol,
+            atol=atol,
         )
 
         if not is_export_ok:
@@ -324,6 +386,8 @@ def _safe_export_half_precision_onnx(
     onnx_config: DictConfig,
     input_shapes: list[Any],
     input_names: list[str],
+    rtol: float = 0.01,
+    atol: float = 5e-3,
 ) -> bool:
     """Check that the exported half precision ONNX model does not contain NaN values. If it does, attempt to export
     the model with a more stable export and overwrite the original model.
@@ -335,6 +399,8 @@ def _safe_export_half_precision_onnx(
         onnx_config: ONNX export configuration
         input_shapes: Input shapes for the model
         input_names: Input names for the model
+        rtol: Relative tolerance to evaluate the model
+        atol: Absolute tolerance to evaluate the model
 
     Returns:
         True if the model is stable or it was possible to export a more stable model, False otherwise.
@@ -364,16 +430,15 @@ def _safe_export_half_precision_onnx(
             export_output = export_onnx_model(
                 model=model,
                 output_path=os.path.dirname(export_model_path),
-                onnx_config=onnx_config,
+                # Force to not simplify fp32 model
+                onnx_config=DictConfig({**onnx_config, "simplify": False}),
                 input_shapes=input_shapes,
                 half_precision=False,
                 model_name=os.path.basename(export_model_path),
             )
-            if export_output is not None:
-                export_model_path, _ = export_output
-            else:
-                log.warning("Failed to export model")
-                return False
+            if export_output is None:
+                # This should not happen
+                raise RuntimeError("Failed to export model")
 
             model_fp32 = onnx.load(export_model_path)
             test_data = {input_names[i]: inp[i].float().cpu().numpy() for i in range(len(inp))}
@@ -381,7 +446,7 @@ def _safe_export_half_precision_onnx(
             with open(os.devnull, "w") as f, contextlib.redirect_stdout(f):
                 # This function prints a lot of information that is not useful for the user
                 model_fp16 = auto_convert_mixed_precision(
-                    model_fp32, test_data, rtol=0.01, atol=5e-3, keep_io_types=False
+                    model_fp32, test_data, rtol=rtol, atol=atol, keep_io_types=False
                 )
             onnx.save(model_fp16, export_model_path)
 
@@ -431,6 +496,9 @@ def export_model(
     input_shapes: list[Any] | None = None,
     idx_to_class: dict[int, str] | None = None,
     pytorch_model_type: Literal["backbone", "model"] = "model",
+    example_inputs: list[Any] | tuple[Any, ...] | torch.Tensor | None = None,
+    rtol: float = 0.01,
+    atol: float = 5e-3,
 ) -> tuple[dict[str, Any], dict[str, str]]:
     """Generate deployment models for the task.
 
@@ -443,6 +511,9 @@ def export_model(
         idx_to_class: Mapping from class index to class name
         pytorch_model_type: Type of the pytorch model config to be exported, if it's backbone on disk we will save the
             config.backbone config, otherwise we will save the config.model
+        example_inputs: If provided use this to evaluate the model instead of generating random inputs
+        rtol: Relative tolerance for the ONNX safe export in fp16
+        atol: Absolute tolerance for the ONNX safe export in fp16
 
     Returns:
         If the model is exported successfully, return a dictionary containing information about the exported model and
@@ -468,6 +539,7 @@ def export_model(
                 input_shapes=input_shapes,
                 output_path=export_folder,
                 half_precision=half_precision,
+                example_inputs=example_inputs,
             )
 
             if out is None:
@@ -495,6 +567,9 @@ def export_model(
                 onnx_config=config.export.onnx,
                 input_shapes=input_shapes,
                 half_precision=half_precision,
+                example_inputs=example_inputs,
+                rtol=rtol,
+                atol=atol,
             )
 
             if out is None:

{quadra-2.4.0a0 → quadra-2.5.0}/quadra/utils/utils.py

@@ -8,10 +8,12 @@ import glob
 import json
 import logging
 import os
+import shutil
 import subprocess
 import sys
 import warnings
 from collections.abc import Iterable, Iterator, Sequence
+from tempfile import TemporaryDirectory
 from typing import Any, cast
 
 import cv2
@@ -299,45 +301,78 @@ def finish(
                     quadra_export.generate_torch_inputs(input_size, device=device, half_precision=half_precision),
                 )
                 types_to_upload = config.core.get("upload_models")
-                for model_path in deployed_models:
-                    model_type = model_type_from_path(model_path)
-                    if model_type is None:
-                        logging.warning("%s model type not supported", model_path)
-                        continue
-                    if model_type is not None and model_type in types_to_upload:
-                        if model_type == "pytorch":
-                            logging.warning("Pytorch format still not supported for mlflow upload")
+                mlflow_zip_models = config.core.get("mlflow_zip_models", False)
+                model_uploaded = False
+                with mlflow.start_run(run_id=mlflow_logger.run_id) as _:
+                    for model_path in deployed_models:
+                        model_type = model_type_from_path(model_path)
+                        model_name = os.path.basename(model_path)
+
+                        if model_type is None:
+                            logging.warning("%s model type not supported", model_path)
                             continue
-
-                        model = quadra_export.import_deployment_model(
-                            model_path,
-                            device=device,
-                            inference_config=config.inference,
-                        )
-
-                        if model_type in ["torchscript", "pytorch"]:
-                            signature = infer_signature_model(model.model, inputs)
-                            with mlflow.start_run(run_id=mlflow_logger.run_id) as _:
-                                mlflow.pytorch.log_model(
-                                    model.model,
-                                    artifact_path=model_path,
-                                    signature=signature,
-                                )
-                        elif model_type in ["onnx", "simplified_onnx"] and ONNX_AVAILABLE:
-                            signature = infer_signature_model(model, inputs)
-                            with mlflow.start_run(run_id=mlflow_logger.run_id) as _:
-                                if model.model_path is None:
-                                    logging.warning(
-                                        "Cannot log onnx model on mlflow, \
-                                        BaseEvaluationModel 'model_path' attribute is None"
+                        if model_type is not None and model_type in types_to_upload:
+                            if model_type == "pytorch" and not mlflow_zip_models:
+                                logging.warning("Pytorch format still not supported for mlflow upload")
+                                continue
+
+                            if mlflow_zip_models:
+                                with TemporaryDirectory() as temp_dir:
+                                    if model_type == "pytorch" and os.path.isfile(
+                                        os.path.join(export_folder, "model_config.yaml")
+                                    ):
+                                        shutil.copy(model_path, temp_dir)
+                                        shutil.copy(os.path.join(export_folder, "model_config.yaml"), temp_dir)
+                                        shutil.make_archive("assets", "zip", root_dir=temp_dir)
+                                    else:
+                                        shutil.make_archive(
+                                            "assets",
+                                            "zip",
+                                            root_dir=os.path.dirname(model_path),
+                                            base_dir=model_name,
+                                        )
+                                    shutil.move("assets.zip", temp_dir)
+                                    mlflow.pyfunc.log_model(
+                                        artifact_path=model_path,
+                                        loader_module="not.used",
+                                        data_path=os.path.join(temp_dir, "assets.zip"),
+                                        pip_requirements=[""],
                                     )
-                                else:
-                                    model_proto = onnx.load(model.model_path)
-                                    mlflow.onnx.log_model(
-                                        model_proto,
+                                    model_uploaded = True
+                            else:
+                                model = quadra_export.import_deployment_model(
+                                    model_path,
+                                    device=device,
+                                    inference_config=config.inference,
+                                )
+
+                                if model_type in ["torchscript", "pytorch"]:
+                                    signature = infer_signature_model(model.model, inputs)
+                                    mlflow.pytorch.log_model(
+                                        model.model,
                                         artifact_path=model_path,
                                         signature=signature,
                                     )
+                                    model_uploaded = True
+
+                                elif model_type in ["onnx", "simplified_onnx"] and ONNX_AVAILABLE:
+                                    if model.model_path is None:
+                                        logging.warning(
+                                            "Cannot log onnx model on mlflow, \
+                                            BaseEvaluationModel 'model_path' attribute is None"
+                                        )
+                                    else:
+                                        signature = infer_signature_model(model, inputs)
+                                        model_proto = onnx.load(model.model_path)
+                                        mlflow.onnx.log_model(
+                                            model_proto,
+                                            artifact_path=model_path,
+                                            signature=signature,
+                                        )
+                                        model_uploaded = True
+
+                    if model_uploaded:
+                        mlflow.log_artifact(os.path.join(export_folder, "model.json"), export_folder)
 
         if tensorboard_logger is not None:
             config_paths = []
@@ -376,7 +411,7 @@ def model_type_from_path(model_path: str) -> str | None:
       - "pytorch" if the model has a '.pth' extension (PyTorch).
       - "simplified_onnx" if the model file ends with 'simplified.onnx' (Simplified ONNX).
       - "onnx" if the model has a '.onnx' extension (ONNX).
-      - "json" id the model has a '.json' extension (JSON).
+      - "json" if the model has a '.json' extension (JSON).
       - None if model extension is not supported.
 
     Example: