mlonmcu 0.5.dev0__tar.gz → 0.6.0__tar.gz

{mlonmcu-0.5.dev0 → mlonmcu-0.6.0}/PKG-INFO

@@ -1,12 +1,13 @@
 Metadata-Version: 2.1
 Name: mlonmcu
-Version: 0.5.dev0
+Version: 0.6.0
 Summary: This project contains research code related to the deployment of inferenceor learning applications on tiny micro-controllers.
 Home-page: https://github.com/tum-ei-eda/mlonmcu
 Author: TUM Department of Electrical and Computer Engineering - Chair of Electronic Design Automation
 Author-email: philipp.van-kempen@tum.de
 License: Apache License 2.0
 Keywords: mlonmcu
+Platform: UNKNOWN
 Classifier: Development Status :: 2 - Pre-Alpha
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: Apache Software License
@@ -20,10 +21,12 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
+Provides-Extra: all-prod
 Provides-Extra: espidf
 Provides-Extra: etiss
 Provides-Extra: gvsoc_pulp
 Provides-Extra: microtvm
+Provides-Extra: microtvm_gvsoc
 Provides-Extra: moiopt
 Provides-Extra: onnx
 Provides-Extra: relay-visualization
@@ -34,7 +37,6 @@ Provides-Extra: tvm-autotuning
 Provides-Extra: visualize
 Provides-Extra: visualize_tflite
 Provides-Extra: zephyr
-Provides-Extra: all-prod
 License-File: LICENSE
 License-File: NOTICE
 
@@ -259,3 +261,5 @@ the projects [Scale4Edge](https://www.edacentrum.de/scale4edge/) (grant number 1
 ## 0.1.0 (2021-11-12)
 
 * First closed-source release.
+
+

{mlonmcu-0.5.dev0 → mlonmcu-0.6.0}/mlonmcu/feature/feature.py

@@ -48,7 +48,7 @@ class FeatureBase(ABC):
     @property
     def enabled(self):
         value = self.config["enabled"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     def remove_config_prefix(self, config):  # TODO: move to different place
         def helper(key):
@@ -229,3 +229,9 @@ class RunFeature(FeatureBase):
 
     def add_run_config(self, config):
         config.update(self.get_run_config())
+
+    # def get_postprocesses(self):
+    #     return []
+
+    # def add_postprocesses(self, postprocesses):
+    #     postprocesses.extend(self.get_postprocesses())

{mlonmcu-0.5.dev0 → mlonmcu-0.6.0}/mlonmcu/feature/features.py

@@ -127,7 +127,7 @@ class Validate(FrontendFeature, PlatformFeature):
     @property
     def allow_missing(self):
         value = self.config["allow_missing"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     @property
     def fail_on_error(self):
@@ -172,8 +172,7 @@ class Muriscvnn(SetupFeature, FrameworkFeature, PlatformFeature):
         value = self.config["use_vext"]
         if value == "AUTO" or value is None:
             return value
-        if not isinstance(value, (bool, int)):
-            value = str2bool(value)
+        value = str2bool(value)
         return "ON" if value else "OFF"
 
     @property
@@ -181,8 +180,7 @@ class Muriscvnn(SetupFeature, FrameworkFeature, PlatformFeature):
         value = self.config["use_pext"]
         if value == "AUTO" or value is None:
             return value
-        if not isinstance(value, (bool, int)):
-            value = str2bool(value)
+        value = str2bool(value)
         return "ON" if value else "OFF"
 
     def add_framework_config(self, framework, config):
@@ -370,8 +368,7 @@ class MuriscvnnByoc(SetupFeature, BackendFeature, PlatformFeature):
         value = self.config["use_vext"]
         if value == "AUTO" or value is None:
             return value
-        if not isinstance(value, (bool, int)):
-            value = str2bool(value)
+        value = str2bool(value)
         return "ON" if value else "OFF"
 
     @property
@@ -379,8 +376,7 @@ class MuriscvnnByoc(SetupFeature, BackendFeature, PlatformFeature):
         value = self.config["use_pext"]
         if value == "AUTO" or value is None:
             return value
-        if not isinstance(value, (bool, int)):
-            value = str2bool(value)
+        value = str2bool(value)
         return "ON" if value else "OFF"
 
     def add_backend_config(self, backend, config):
@@ -563,22 +559,22 @@ class Bext(SetupFeature, TargetFeature, PlatformFeature):
     @property
     def zba(self):
         value = self.config["zba"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     @property
     def zbb(self):
         value = self.config["zbb"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     @property
     def zbc(self):
         value = self.config["zbc"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     @property
     def zbs(self):
         value = self.config["zbs"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     def get_target_config(self, target):
         return filter_none(
@@ -636,7 +632,7 @@ class GdbServer(TargetFeature):
     @property
     def attach(self):
         value = self.config["attach"]
-        return str2bool(value, allow_none=True) if not isinstance(value, (bool, int)) else value
+        return str2bool(value, allow_none=True)
 
     @property
     def port(self):
@@ -677,7 +673,7 @@ class Trace(TargetFeature):
     @property
     def to_file(self):
         value = self.config["to_file"]
-        return str2bool(value, allow_none=True) if not isinstance(value, (bool, int)) else value
+        return str2bool(value, allow_none=True)
 
     def __init__(self, features=None, config=None):
         super().__init__("trace", features=features, config=config)
@@ -765,7 +761,7 @@ class Usmp(BackendFeature):
     @property
     def use_workspace_io(self):
         value = self.config["use_workspace_io"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     def add_backend_config(self, backend, config):
         assert backend in ["tvmaot"], f"Unsupported feature '{self.name}' for backend '{backend}'"
@@ -1286,7 +1282,7 @@ class CacheSim(TargetFeature):
     @property
     def ic_enable(self):
         value = self.config["ic_enable"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     @property
     def ic_config(self):
@@ -1295,7 +1291,7 @@ class CacheSim(TargetFeature):
     @property
     def dc_enable(self):
         value = self.config["dc_enable"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     @property
     def dc_config(self):
@@ -1304,7 +1300,7 @@ class CacheSim(TargetFeature):
     @property
     def l2_enable(self):
         value = self.config["l2_enable"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     @property
     def l2_config(self):
@@ -1313,12 +1309,12 @@ class CacheSim(TargetFeature):
     @property
     def log_misses(self):
         value = self.config["log_misses"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     @property
     def detailed(self):
         value = self.config["detailed"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     # def add_target_config(self, target, config, directory=None):
     def add_target_config(self, target, config):
@@ -1379,7 +1375,7 @@ class LogInstructions(TargetFeature):
     @property
     def to_file(self):
         value = self.config["to_file"]
-        return str2bool(value, allow_none=True) if not isinstance(value, (bool, int)) else value
+        return str2bool(value, allow_none=True)
 
     # def add_target_config(self, target, config, directory=None):
     def add_target_config(self, target, config):
@@ -1527,12 +1523,12 @@ class TargetOptimized(RunFeature):
     @property
     def layouts(self):
         value = self.config["layouts"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     @property
     def schedules(self):
         value = self.config["schedules"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     def __init__(self, features=None, config=None):
         super().__init__("target_optimized", features=features, config=config)
@@ -1571,7 +1567,7 @@ class AutoVectorize(PlatformFeature):
     @property
     def verbose(self):
         value = self.config["verbose"]
-        # return str2bool(value) if not isinstance(value, (bool, int)) else value
+        # return str2bool(value)
         if value is None or not value:
             return "OFF"
         assert isinstance(value, str)
@@ -1582,12 +1578,12 @@ class AutoVectorize(PlatformFeature):
     @property
     def loop(self):
         value = self.config["loop"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     @property
     def slp(self):
         value = self.config["slp"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     @property
     def force_vector_width(self):
@@ -1618,7 +1614,7 @@ class AutoVectorize(PlatformFeature):
     @property
     def custom_unroll(self):
         value = self.config["custom_unroll"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     def get_platform_defs(self, platform):
         return {
@@ -1660,7 +1656,7 @@ class Benchmark(PlatformFeature, TargetFeature):
     @property
     def total(self):
         value = self.config["total"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     @property
     def aggregate(self):
@@ -1848,37 +1844,37 @@ class XCoreV(TargetFeature, PlatformFeature, SetupFeature):
     @property
     def mac(self):
         value = self.config["mac"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     @property
     def mem(self):
         value = self.config["mem"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     @property
     def bi(self):
         value = self.config["bi"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     @property
     def alu(self):
         value = self.config["alu"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     @property
     def bitmanip(self):
         value = self.config["bitmanip"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     @property
     def simd(self):
         value = self.config["simd"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     @property
     def hwlp(self):
         value = self.config["hwlp"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     # def add_target_config(self, target, config, directory=None):
     def add_target_config(self, target, config):
@@ -1940,8 +1936,8 @@ class Xpulp(TargetFeature, PlatformFeature, SetupFeature):
 
     # Default getter functions:
     @staticmethod
-    def generalized_str2bool(input: Union[str, bool, int]) -> bool:
-        return str2bool(input) if isinstance(input, str) else input
+    def generalized_str2bool(inp: Union[str, bool, int]) -> bool:
+        return str2bool(inp)
 
     def getter_bool(self, key_name: Union[str, bool, int]) -> bool:
         return self.generalized_str2bool(self.config[key_name])
@@ -2227,3 +2223,212 @@ class VanillaAccelerator(TargetFeature):
             assert self.name not in extra_plugin_config
             extra_plugin_config[self.name]["baseaddr"] = self.base_addr
             config.update({f"{target}.extra_plugin_config": extra_plugin_config})
+
+
+@register_feature("gen_data")
+class GenData(FrontendFeature):  # TODO: use custom stage instead of LOAD
+    """Generate input data for validation."""
+
+    DEFAULTS = {
+        **FeatureBase.DEFAULTS,
+        "fill_mode": "file",  # Allowed: random, ones, zeros, file, dataset
+        "file": "auto",  # Only relevant if fill_mode=file
+        "number": 10,  # generate up to number samples (may be less if file has less inputs)
+        "fmt": "npy",  # Allowed: npy, npz
+    }
+
+    def __init__(self, features=None, config=None):
+        super().__init__("gen_data", features=features, config=config)
+
+    @property
+    def fill_mode(self):
+        value = self.config["fill_mode"]
+        assert value in ["random", "ones", "zeros", "file", "dataset"]
+        return value
+
+    @property
+    def file(self):
+        value = self.config["file"]
+        return value
+
+    @property
+    def number(self):
+        return int(self.config["number"])
+
+    @property
+    def fmt(self):
+        value = self.config["fmt"]
+        assert value in ["npy", "npz"]
+        return value
+
+    def get_frontend_config(self, frontend):
+        assert frontend in ["tflite"]
+        return {
+            f"{frontend}.gen_data": self.enabled,
+            f"{frontend}.gen_data_fill_mode": self.fill_mode,
+            f"{frontend}.gen_data_file": self.file,
+            f"{frontend}.gen_data_number": self.number,
+            f"{frontend}.gen_data_fmt": self.fmt,
+        }
+
+
+@register_feature("gen_ref_data", depends=["gen_data"])
+class GenRefData(FrontendFeature):  # TODO: use custom stage instead of LOAD
+    """Generate reference outputs for validation."""
+
+    DEFAULTS = {
+        **FeatureBase.DEFAULTS,
+        "mode": "file",  # Allowed: file, model
+        "file": "auto",  # Only relevant if mode=file
+        "fmt": "npy",  # Allowed: npy, npz
+    }
+
+    def __init__(self, features=None, config=None):
+        super().__init__("gen_ref_data", features=features, config=config)
+
+    @property
+    def mode(self):
+        value = self.config["mode"]
+        assert value in ["file", "model"]
+        return value
+
+    @property
+    def file(self):
+        value = self.config["file"]
+        return value
+
+    @property
+    def fmt(self):
+        value = self.config["fmt"]
+        assert value in ["npy", "npz"]
+        return value
+
+    def get_frontend_config(self, frontend):
+        assert frontend in ["tflite"]
+        return {
+            f"{frontend}.gen_ref_data": self.enabled,
+            f"{frontend}.gen_ref_data_mode": self.mode,
+            f"{frontend}.gen_ref_data_file": self.file,
+            f"{frontend}.gen_ref_data_fmt": self.fmt,
+        }
+
+
+@register_feature("gen_ref_labels", depends=["gen_data"])
+class GenRefLabels(FrontendFeature):  # TODO: use custom stage instead of LOAD
+    """Generate reference labels for classification."""
+
+    DEFAULTS = {
+        **FeatureBase.DEFAULTS,
+        "mode": "file",  # Allowed: file, model
+        "file": "auto",  # Only relevant if mode=file
+        "fmt": "npy",  # Allowed: npy, npz, txt
+    }
+
+    def __init__(self, features=None, config=None):
+        super().__init__("gen_ref_labels", features=features, config=config)
+
+    @property
+    def mode(self):
+        value = self.config["mode"]
+        assert value in ["file", "model"]
+        return value
+
+    @property
+    def file(self):
+        value = self.config["file"]
+        return value
+
+    @property
+    def fmt(self):
+        value = self.config["fmt"]
+        assert value in ["npy", "npz"]
+        return value
+
+    def get_frontend_config(self, frontend):
+        assert frontend in ["tflite"]
+        return {
+            f"{frontend}.gen_ref_labels": self.enabled,
+            f"{frontend}.gen_ref_labels_mode": self.mode,
+            f"{frontend}.gen_ref_labels_file": self.file,
+            f"{frontend}.gen_ref_labels_fmt": self.fmt,
+        }
+
+
+@register_feature("set_inputs")
+class SetInputs(PlatformFeature):  # TODO: use custom stage instead of LOAD
+    """Apply test inputs to model."""
+
+    DEFAULTS = {
+        **FeatureBase.DEFAULTS,
+        "interface": "auto",  # Allowed: auto, rom, filesystem, stdin, stdin_raw, uart
+    }
+
+    def __init__(self, features=None, config=None):
+        super().__init__("set_inputs", features=features, config=config)
+
+    @property
+    def interface(self):
+        value = self.config["interface"]
+        assert value in ["auto", "rom", "filesystem", "stdin", "stdin_raw", "uart"]
+        return value
+
+    def get_platform_config(self, platform):
+        assert platform in ["mlif", "tvm", "microtvm"]
+        # if tvm/microtvm: allow using --fill-mode provided by tvmc run
+        return {
+            f"{platform}.set_inputs": self.enabled,
+            f"{platform}.set_inputs_interface": self.interface,
+        }
+
+
+@register_feature("get_outputs")
+class GetOutputs(PlatformFeature):  # TODO: use custom stage instead of LOAD
+    """Extract resulting outputs from model."""
+
+    DEFAULTS = {
+        **FeatureBase.DEFAULTS,
+        "interface": "auto",  # Allowed: auto, filesystem, stdout, stdout_raw, uart
+        "fmt": "npy",  # Allowed: npz, npz
+    }
+
+    def __init__(self, features=None, config=None):
+        super().__init__("get_outputs", features=features, config=config)
+
+    @property
+    def interface(self):
+        value = self.config["interface"]
+        assert value in ["auto", "filesystem", "stdout", "stdout_raw", "uart"]
+        return value
+
+    @property
+    def fmt(self):
+        value = self.config["fmt"]
+        assert value in ["npy", "npz"]
+        return value
+
+    def get_platform_config(self, platform):
+        assert platform in ["mlif", "tvm", "microtvm"]
+        return {
+            f"{platform}.get_outputs": self.enabled,
+            f"{platform}.get_outputs_interface": self.interface,
+            f"{platform}.get_outputs_fmt": self.fmt,
+        }
+
+
+@register_feature("validate_new", depends=["gen_data", "gen_ref_data", "set_inputs", "get_outputs"])
+class ValidateNew(RunFeature):
+    """Wrapper feature for enabling all validatioon related features at once."""
+
+    DEFAULTS = {
+        **FeatureBase.DEFAULTS,
+    }
+
+    def __init__(self, features=None, config=None):
+        super().__init__("validate_new", features=features, config=config)
+
+    # def get_postprocesses(self):
+    #     # config = {}
+    #     # from mlonmcu.session.postprocess import ValidateOutputsPostprocess
+    #     # validate_outputs_postprocess = ValidateOutputsPostprocess(features=[], config=config)
+    #     # return [validate_outputs_postprocess]
+    #     return ["validate_outputs"]

{mlonmcu-0.5.dev0 → mlonmcu-0.6.0}/mlonmcu/flow/tflm/backend/tflmc.py

@@ -50,7 +50,7 @@ class TFLMCBackend(TFLMBackend):
     @property
     def print_outputs(self):
         value = self.config["print_outputs"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     def __init__(self, features=None, config=None):
         super().__init__(features=features, config=config)

{mlonmcu-0.5.dev0 → mlonmcu-0.6.0}/mlonmcu/flow/tflm/backend/tflmi.py

@@ -20,7 +20,7 @@ import sys
 from typing import Tuple
 
 from .backend import TFLMBackend
-from mlonmcu.config import str2bool, str2list
+from mlonmcu.config import str2bool, str2list, str2dict
 from mlonmcu.flow.backend import main
 from mlonmcu.artifact import Artifact, ArtifactFormat
 
@@ -435,12 +435,12 @@ class TFLMIBackend(TFLMBackend):
     @property
     def legacy(self):
         value = self.config["legacy"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     @property
     def debug_arena(self):
         value = self.config["debug_arena"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     @property
     def arena_size(self):
@@ -449,17 +449,17 @@ class TFLMIBackend(TFLMBackend):
     @property
     def ops(self):
         value = self.config["ops"]
-        return str2list(value) if isinstance(value, str) else value
+        return str2list(value)
 
     @property
     def custom_ops(self):
         value = self.config["custom_ops"]
-        return str2list(value) if isinstance(value, str) else value
+        return str2list(value)
 
     @property
     def registrations(self):
         value = self.config["registrations"]
-        return str2list(value) if isinstance(value, str) else value
+        return str2dict(value)
 
     @property
     def ops_resolver(self):
@@ -468,7 +468,7 @@ class TFLMIBackend(TFLMBackend):
     @property
     def reporter(self):
         value = self.config["reporter"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     def generate(self) -> Tuple[dict, dict]:
         artifacts = []

{mlonmcu-0.5.dev0 → mlonmcu-0.6.0}/mlonmcu/flow/tvm/backend/backend.py

@@ -160,6 +160,7 @@ class TVMBackend(Backend):
             extra = {}
         if isinstance(extra, str):
             import ast
+
             extra = ast.literal_eval(extra)
         assert isinstance(extra, dict)
         return extra
@@ -251,7 +252,7 @@ class TVMBackend(Backend):
     @property
     def use_tuning_results(self):
         value = self.config["use_tuning_results"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     @property
     def tvmc_extra_args(self):
@@ -264,7 +265,7 @@ class TVMBackend(Backend):
     @property
     def disabled_passes(self):
         value = self.config["disabled_passes"]
-        return str2list(value) if isinstance(value, str) else value
+        return str2list(value)
 
     @property
     def tvm_pythonpath(self):
@@ -285,17 +286,17 @@ class TVMBackend(Backend):
     @property
     def print_outputs(self):
         value = self.config["print_outputs"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     @property
     def use_tlcpack(self):
         value = self.config["tvm.use_tlcpack"]
-        return str2bool(value, allow_none=True) if not isinstance(value, (bool, int)) else value
+        return str2bool(value, allow_none=True)
 
     @property
     def custom_unroll(self):
         value = self.config["custom_unroll"]
-        return str2bool(value, allow_none=True) if not isinstance(value, (bool, int)) else value
+        return str2bool(value, allow_none=True)
 
     @property
     def dump(self):
@@ -317,7 +318,7 @@ class TVMBackend(Backend):
     @property
     def refresh_model_info(self):
         value = self.config["refresh_model_info"]
-        return str2bool(value, allow_none=True) if not isinstance(value, (bool, int)) else value
+        return str2bool(value, allow_none=True)
 
     @property
     def generate_wrapper(self):

{mlonmcu-0.5.dev0 → mlonmcu-0.6.0}/mlonmcu/flow/tvm/backend/tvmaot.py

@@ -67,12 +67,12 @@ class TVMAOTBackend(TVMBackend):
     @property
     def unpacked_api(self):
         value = self.config["unpacked_api"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     @property
     def debug_arena(self):
         value = self.config["debug_arena"]
-        return str2bool(value) if not isinstance(value, (bool, int)) else value
+        return str2bool(value)
 
     @property
     def alignment_bytes(self):

{mlonmcu-0.5.dev0 → mlonmcu-0.6.0}/mlonmcu/flow/tvm/backend/tvmc_utils.py

@@ -164,7 +164,7 @@ def get_tvmrt_tvmc_args(runtime="crt", system_lib=True, link_params=True):
     return ret
 
 
-def get_data_tvmc_args(mode=None, ins_file=None, outs_file=None, print_top=10):
+def get_data_tvmc_args(mode=None, ins_file=None, outs_file=None, print_top=None):
     ret = []
     if ins_file is not None:
         ret.extend(["--inputs", ins_file])
@@ -176,7 +176,7 @@ def get_data_tvmc_args(mode=None, ins_file=None, outs_file=None, print_top=10):
         ret.extend(["--outputs", outs_file])
 
     if print_top is not None and print_top > 0:
-        ret.extend(["--print-top", print_top])
+        ret.extend(["--print-top", str(print_top)])
 
     return ret