mindspore 2.4.1__cp39-cp39-manylinux1_x86_64.whl → 2.4.10__cp39-cp39-manylinux1_x86_64.whl

mindspore/run_check/_check_version.py

@@ -258,7 +258,7 @@ class AscendEnvChecker(EnvChecker):
 
     def __init__(self, library_path):
         self.library_path = library_path
-        self.version = ["7.3", "7.5"]
+        self.version = ["7.5", "7.6"]
 
         # env
         self.path = os.getenv("PATH")
@@ -266,7 +266,7 @@ class AscendEnvChecker(EnvChecker):
         self.ld_lib_path = os.getenv("LD_LIBRARY_PATH")
         self.ascend_opp_path = os.getenv("ASCEND_OPP_PATH")
         self.ascend_aicpu_path = os.getenv("ASCEND_AICPU_PATH")
-        if not self.ascend_opp_path is None:
+        if self.ascend_opp_path is not None:
             self.compiler_version = self.ascend_opp_path.split("opp")[0] + "compiler/version.info"
         else:
             self.compiler_version = ""
@@ -287,7 +287,7 @@ class AscendEnvChecker(EnvChecker):
     def check_custom_version(self):
         """custom op version check"""
         custom_ascendc_soc_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),
-                                              "../lib/plugin/ascend/custom_ascendc_ops/op_impl/ai_core/tbe/kernel",
+                                              "../lib/plugin/ascend/custom_ascendc_910b/op_impl/ai_core/tbe/kernel",
                                               MSContext.get_instance().get_ascend_soc_version())
         if not os.path.exists(custom_ascendc_soc_dir):
             logger.debug(f"The path {custom_ascendc_soc_dir} of the custom ascend c operator does not exist.")
@@ -298,7 +298,7 @@ class AscendEnvChecker(EnvChecker):
 
         cur_version = self._read_version(self.compiler_version)
         custom_version_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
-                                           "../lib/plugin/ascend/custom_ascendc_ops/version.info")
+                                           "../lib/plugin/ascend/custom_ascendc_910/version.info")
         with open(custom_version_path, 'r') as f:
             all_info = f.readlines()
             for line in all_info:
@@ -372,12 +372,21 @@ class AscendEnvChecker(EnvChecker):
         curr_path = os.path.realpath(os.path.dirname(__file__))
         cust_aicpu_path = os.path.realpath(os.path.join(curr_path, "../lib/plugin/ascend/custom_aicpu_ops"))
         cust_aicore_path = os.path.realpath(os.path.join(curr_path, "../lib/plugin/ascend/custom_aicore_ops"))
-        cust_ascendc_path = os.path.realpath(os.path.join(curr_path, "../lib/plugin/ascend/custom_ascendc_ops"))
+        cust_ascendc_ascend910_path = os.path.realpath(
+            os.path.join(curr_path, "../lib/plugin/ascend/custom_ascendc_910"))
+        cust_ascendc_ascend910b_path = os.path.realpath(
+            os.path.join(curr_path, "../lib/plugin/ascend/custom_ascendc_910b"))
         if os.getenv('ASCEND_CUSTOM_OPP_PATH'):
             os.environ['ASCEND_CUSTOM_OPP_PATH'] = os.environ['ASCEND_CUSTOM_OPP_PATH'] + ":" + \
-                                                   cust_ascendc_path + ":" + cust_aicore_path + ":" + cust_aicpu_path
+                                                   cust_ascendc_ascend910_path + ":" + cust_ascendc_ascend910b_path + \
+                                                   ":" + cust_aicore_path + ":" + cust_aicpu_path
         else:
-            os.environ['ASCEND_CUSTOM_OPP_PATH'] = cust_ascendc_path + ":" + cust_aicore_path + ":" + cust_aicpu_path
+            os.environ['ASCEND_CUSTOM_OPP_PATH'] = cust_ascendc_ascend910_path + ":" + \
+                                                   cust_ascendc_ascend910b_path + ":" + cust_aicore_path + ":" + \
+                                                   cust_aicpu_path
+        # Ignore ge infer missing error. To be removed after infers are completed.
+        os.environ['FAST_IGNORE_INFER_ERROR'] = "1"
+        os.environ['IGNORE_INFER_ERROR'] = "1"
         plugin_dir = os.path.dirname(self.library_path)
         akg_dir = os.path.join(plugin_dir, "ascend")
         AscendEnvChecker._concat_variable('LD_LIBRARY_PATH', akg_dir)
@@ -400,7 +409,7 @@ class AscendEnvChecker(EnvChecker):
                 "Environment Variable PYTHONPATH is set. For details, refer to the installation guidelines: "
                 "https://www.mindspore.cn/install")
 
-        if self.ld_lib_path is None or not self.ld_lib_path_check_fwk in self.ld_lib_path:
+        if self.ld_lib_path is None or self.ld_lib_path_check_fwk not in self.ld_lib_path:
             logger.warning("Can not find driver so(need by mindspore-ascend). Please check whether the "
                            "Environment Variable LD_LIBRARY_PATH is set. For details, refer to the installation "
                            "guidelines: https://www.mindspore.cn/install")

mindspore/train/callback/_tft_register.py

@@ -109,7 +109,7 @@ or repair_info["repair_type"] == cb_ctx.tft.RepairType.RT_UCE_LOWLEVEL.value):
     logger.info("Finish _tft_repair_callback")
 
 
-def _tft_clean_callback(is_uce_error, ctx):
+def _tft_clean_callback(is_uce_error, args, ctx):
     """ Callback used for TFT clean function."""
     logger.info("Enter _tft_clean_callback")
     ret = 0
@@ -130,12 +130,13 @@ def _tft_clean_callback(is_uce_error, ctx):
     return ret
 
 
-def _tft_stop_callback(cb_ctx):
+def _tft_stop_callback(args, cb_ctx):
     """ Callback used for TFT stop function."""
     logger.info("Enter _tft_stop_callback device_id: {}".format(cb_ctx.device_id))
     _stop_device(cb_ctx.device_id)
-    if not cb_ctx._is_params_consistent():    # pylint: disable=W0212
+    if (not cb_ctx.is_uce_rank) and (not cb_ctx._is_params_consistent()):    # pylint: disable=W0212
         raise RuntimeError("Can't stop device, because training parameters are left in inconsistent state!")
+    cb_ctx.is_uce_rank = False
     logger.info("Finish _tft_stop_callback")
 
 
@@ -261,6 +262,7 @@ class TFTRegister(Callback):
         # let it raise errors if not install mindio_tft package
         from mindio_ttp import framework_ttp as tft
         self.tft = tft
+        self.is_uce_rank = False
         self.global_step = 0
         Validator.check_non_negative_int(ctrl_port)
         self.has_init_replica = False
@@ -300,7 +302,7 @@ class TFTRegister(Callback):
         replica_info = [
             {
                 "type": 1,
-                "rank_list": dp,
+                "rank_list": list(dp),
                 "replica_cnt": len(dp),
                 "replica_shift": 0
             }
@@ -321,13 +323,12 @@ class TFTRegister(Callback):
         cur_rank = get_rank()
         enable_local_copy = False
         enable_arf = False
-        enable_zit = False
         enable_tls = False
         tls_key_dir = ""
 
         if cur_rank == self._controller_rank_id:
             logger.info(f"Begin to start tft controller on rank_id:{cur_rank}")
-            self.tft.tft_init_controller(cur_rank, world_size, enable_local_copy, enable_arf, enable_zit)
+            self.tft.tft_init_controller(cur_rank, world_size, enable_local_copy, enable_arf)
             self.tft.tft_start_controller(self._controller_ip, self._controller_port, enable_tls, tls_key_dir)
             logger.info("Finish start tft controller.")
 

mindspore/train/model.py

@@ -149,6 +149,7 @@ def _handle_tft(func):
                     e_str = str(e)
                     logger.info("uce wrapper caught RuntimeError e_str:{}".format(e_str))
                     if "UCEError" in e_str:
+                        obj.is_uce_rank = True
                         logger.info("uce wrapper report UCEError")
                         tft.tft_report_error(tft.ReportState.RS_UCE.value)
                     elif "ForceStopError" in e_str:

mindspore/train/serialization.py

@@ -81,6 +81,7 @@ from safetensors.numpy import save_file
 from safetensors import safe_open
 from ..ops.operations._opaque_predicate_registry import add_opaque_predicate, clean_funcs
 
+
 tensor_to_ms_type = {"Int8": mstype.int8, "UInt8": mstype.uint8, "Int16": mstype.int16, "UInt16": mstype.uint16,
                      "Int32": mstype.int32, "UInt32": mstype.uint32, "Int64": mstype.int64, "UInt64": mstype.uint64,
                      "Float16": mstype.float16, "Float32": mstype.float32, "Float64": mstype.float64,
@@ -328,6 +329,8 @@ def _exec_save(ckpt_file_name, data_list, enc_key=None, enc_mode="AES-GCM", map_
             file_name_list = list(os.path.splitext(ckpt_file_name))
             file_name_list[1] = file_name_list[1].replace(f".{format}", ".tmp")
             tmp_name = ''.join(file_name_list)
+            if _ckpt_fs.backend == "mindio":
+                tmp_name = ckpt_file_name
             if os.path.exists(ckpt_file_name):
                 os.chmod(ckpt_file_name, stat.S_IWUSR)
                 os.remove(ckpt_file_name)
@@ -383,7 +386,7 @@ def _exec_save(ckpt_file_name, data_list, enc_key=None, enc_mode="AES-GCM", map_
             if not os.path.exists(tmp_name):
                 logger.warning(f"Rename failed, can't find {tmp_name}, it is possible that multiple processes have "
                                f"simultaneously modified a file.")
-            else:
+            elif _ckpt_fs.backend != "mindio":
                 os.rename(tmp_name, ckpt_file_name)
             os.chmod(ckpt_file_name, stat.S_IRUSR)
     except BaseException as e:

mindspore/version.py

@@ -1 +1 @@
-__version__ = '2.4.1'
+__version__ = '2.4.10'

{mindspore-2.4.1.dist-info → mindspore-2.4.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mindspore
-Version: 2.4.1
+Version: 2.4.10
 Summary: MindSpore is a new open source deep learning training/inference framework that could be used for mobile, edge and cloud scenarios.
 Home-page: https://www.mindspore.cn
 Download-URL: https://github.com/mindspore-ai/mindspore/tags
@@ -118,7 +118,7 @@ MindSpore offers build options across multiple backends:
 
 | Hardware Platform | Operating System | Status |
 | :---------------- | :--------------- | :----- |
-| Ascend910 | Ubuntu-x86 | ✔️ |
+| Ascend | Ubuntu-x86 | ✔️ |
 |  | Ubuntu-aarch64 | ✔️ |
 |  | EulerOS-aarch64 | ✔️ |
 |  | CentOS-x86 | ✔️ |