pwact 0.3.2__tar.gz → 0.3.4__tar.gz

{pwact-0.3.2 → pwact-0.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pwact
-Version: 0.3.2
+Version: 0.3.4
 Summary: PWACT is an open-source automated active learning platform based on MatPL for efficient data sampling.
 Home-page: https://github.com/LonxunQuantum/PWact
 Author: LonxunQuantum

{pwact-0.3.2 → pwact-0.3.4}/pwact/active_learning/explore/run_model_md.py

@@ -80,7 +80,8 @@ class Explore(object):
     def back_explore(self):
         slurm_remain, slurm_success = get_slurm_job_run_info(self.real_md_dir, \
         job_patten="*-{}".format(EXPLORE_FILE_STRUCTURE.md_job), \
-        tag_patten="*-{}".format(EXPLORE_FILE_STRUCTURE.md_tag))
+        tag_patten="*-{}".format(EXPLORE_FILE_STRUCTURE.md_tag),
+        for_back = True)
         slurm_done = True if len(slurm_remain) == 0 and len(slurm_success) > 0 else False
         if slurm_done:
             # bk and do new job
@@ -180,9 +181,6 @@ class Explore(object):
             job_patten="*-{}".format(EXPLORE_FILE_STRUCTURE.md_job), \
             tag_patten="*-{}".format(EXPLORE_FILE_STRUCTURE.md_tag))
         # for slurm remain, check if tags done
-        slurm_done = True if len(slurm_remain) == 0 and len(slurm_success) > 0 else False
-        if slurm_done is False:
-            slurm_remain = recheck_slurm_by_jobtag(slurm_remain, EXPLORE_FILE_STRUCTURE.md_tag)
         if len(slurm_remain) > 0:
             #recover slurm jobs
             if len(slurm_remain) > 0:

{pwact-0.3.2 → pwact-0.3.4}/pwact/active_learning/init_bulk/init_bulk_run.py

@@ -164,15 +164,7 @@ def do_collection(resource: Resource, input_param:InitBulkParam):
         copy_dir(bigmodel_dir, os.path.join(collection_dir, INIT_BULK.bigmodel))        
     
     if len(result_save_path) > 0:
-        _path_path = []
-        for _data_path in result_save_path:
-            if input_param.data_format == PWDATA.extxyz:
-                _path_path.append(_data_path)
-            elif input_param.data_format == PWDATA.pwmlff_npy: # */PWdata/*.npy
-                tmp = search_files(_data_path, "*/position.npy")
-                _path_path.extend([os.path.dirname(_) for _ in tmp])
-            
-        result_lines = ["\"{}\",".format(_) for _ in _path_path]
+        result_lines = ["\"{}\",".format(_) for _ in result_save_path]
         result_lines = "\n".join(result_lines)
         # result_lines = result_lines[:-1] # Filter the last ','
         result_save_path = os.path.join(collection_dir, INIT_BULK.npy_format_name)

{pwact-0.3.2 → pwact-0.3.4}/pwact/active_learning/label/labeling.py

@@ -149,7 +149,8 @@ class Labeling(object):
         if self.input_param.scf.dft_style == DFT_STYLE.bigmodel:
             slurm_remain, slurm_success = get_slurm_job_run_info(self.real_bigmodel_dir, \
                 job_patten="*-{}".format(LABEL_FILE_STRUCTURE.bigmodel_job), \
-                tag_patten="*-{}".format(LABEL_FILE_STRUCTURE.bigmodel_tag))
+                tag_patten="*-{}".format(LABEL_FILE_STRUCTURE.bigmodel_tag),
+                for_back = True)
             slurm_done = True if len(slurm_remain) == 0 and len(slurm_success) > 0 else False        
         else:
             slurm_remain, slurm_success = get_slurm_job_run_info(self.real_scf_dir, \

{pwact-0.3.2 → pwact-0.3.4}/pwact/active_learning/train/train_model.py

@@ -50,7 +50,8 @@ class ModelTrian(object):
     def back_train(self):
         slurm_remain, slurm_success = get_slurm_job_run_info(self.real_train_dir, \
             job_patten="*-{}".format(TRAIN_FILE_STRUCTUR.train_job), \
-            tag_patten="*-{}".format(TRAIN_FILE_STRUCTUR.train_tag))
+            tag_patten="*-{}".format(TRAIN_FILE_STRUCTUR.train_tag),
+            for_back = True)
         slurm_done = True if len(slurm_remain) == 0 and len(slurm_success) > 0 else False # len(slurm_remain) > 0 exist slurm jobs need to do
         if slurm_done:
             # bk and do new job

{pwact-0.3.2 → pwact-0.3.4}/pwact/active_learning/user_input/resource.py

@@ -6,7 +6,7 @@ class Resource(object):
     # scf_style for init_bulk relabel
     def __init__(self, json_dict:dict, job_type:str=AL_WORK.run_iter, dft_style:str=None, scf_style:str=None) -> None:
         if job_type == AL_WORK.run_iter:
-            self.train_resource = self.get_resource(get_required_parameter("train", json_dict))
+            self.train_resource = self.get_resource(get_required_parameter("train", json_dict), default_groupsize=1)
             if self.train_resource.number_node > 1:
                 self.train_resource.number_node = 1
             if self.train_resource.gpu_per_node > 1:
@@ -16,28 +16,28 @@ class Resource(object):
             print("Warining: the resouce of node, gpu per node and cpu per node  in training automatically adjust to [1, 1, 1]")
             self.train_resource.command = self.train_resource.command.upper()
              
-            self.explore_resource = self.get_resource(get_required_parameter("explore", json_dict))
+            self.explore_resource = self.get_resource(get_required_parameter("explore", json_dict), default_groupsize=1)
             if "-in" in self.explore_resource.command:
                 self.explore_resource.command = self.explore_resource.command.split('-in')[0].strip()
             self.explore_resource.command = "{} -in {} > {}".format(self.explore_resource.command, LAMMPS.input_lammps, SLURM_OUT.md_out)
         else:
             if "explore" in json_dict.keys():
-                self.explore_resource = self.get_resource(get_required_parameter("explore", json_dict))
+                self.explore_resource = self.get_resource(get_required_parameter("explore", json_dict), default_groupsize=1)
             else:
                 self.explore_resource = None
         # check dft resource
         if "dft" in json_dict.keys():
-            self.dft_resource = self.get_resource(get_required_parameter("dft", json_dict))
+            self.dft_resource = self.get_resource(get_required_parameter("dft", json_dict), default_groupsize=-1)
         else:
             self.dft_resource = ResourceDetail("mpirun -np 1 PWmat", 1, 1, 1, 1, 1, None, None, None)
 
         if "direct" in json_dict.keys():
-            self.direct_resource = self.get_resource(get_required_parameter("direct", json_dict))
+            self.direct_resource = self.get_resource(get_required_parameter("direct", json_dict), default_groupsize=1)
         else:
             self.direct_resource = None
 
         if "scf" in json_dict.keys():
-            self.scf_resource = self.get_resource(get_parameter("scf", json_dict, None))
+            self.scf_resource = self.get_resource(get_parameter("scf", json_dict, None), default_groupsize=-1) 
         else:
             self.scf_resource = None
         # dftb_command = get_parameter("dftb_command", json_dict["dft"], None)
@@ -66,9 +66,9 @@ class Resource(object):
     #         cls._instance = cls(json_dict)
     #     return cls._instance
     
-    def get_resource(self, json_dict:dict):
+    def get_resource(self, json_dict:dict, default_groupsize=1):
         command = get_required_parameter("command", json_dict)
-        group_size = get_parameter("group_size", json_dict, 1)
+        group_size = get_parameter("group_size", json_dict, default_groupsize)
         parallel_num = get_parameter("parallel_num", json_dict, 1)
         number_node = get_parameter("number_node", json_dict, 1)
         gpu_per_node = get_parameter("gpu_per_node", json_dict, 0)

{pwact-0.3.2 → pwact-0.3.4}/pwact/utils/constant.py

@@ -10,9 +10,9 @@ class AL_WORK:
     run_iter = "run"
 
 class AL_STRUCTURE:
-    train = "train"
-    explore = "explore"
-    labeling = "label"
+    train = "00.train"
+    explore = "01.explore"
+    labeling = "02.label"
     pertub = "pertub"
     aimd = "aimd"
     collection = "collection"

{pwact-0.3.2 → pwact-0.3.4}/pwact/utils/slurm_script.py

@@ -35,11 +35,13 @@ Obtain the execution status of the slurm jobs under the 'dir'
 
         0-scf.job 1-scf.job 2-scf.job 3-scf.job 4-scf.job
         0-tag.scf.success 1-tag.scf.success 2-tag.scf.success 3-tag.scf.success 4-tag.scf.success
+    new change:
+        only jugt the slurm file is done by the tag under each subwork tag. so the tag of job file is nouse maybe
 param {*} dir
 Returns: 
 Author: WU Xingxing
 '''
-def get_slurm_job_run_info(dir:str, job_patten:str="*.job", tag_patten:str="tag.*.success"):
+def get_slurm_job_run_info(dir:str, job_patten:str="*.job", tag_patten:str="tag.*.success", for_back:bool=False):
     slurm_job_files = sorted(glob.glob(os.path.join(dir, job_patten)))
     slrum_job_dirs = [int(os.path.basename(_).split('-')[0]) for _ in slurm_job_files]
 
@@ -49,14 +51,39 @@ def get_slurm_job_run_info(dir:str, job_patten:str="*.job", tag_patten:str="tag.
     slurm_failed = []
     slurm_success = []
 
-    for i, d in enumerate(slrum_job_dirs):
-        if d in slrum_tag_sucess_dirs:
-            slurm_success.append(slurm_job_files[i])
-        else:
-            slurm_failed.append(slurm_job_files[i])
+    if for_back is False:
+        for slurm_file in slurm_job_files:
+            if slurm_job_done_by_jobtag(slurm_file):
+                slurm_success.append(slurm_file)
+            else:
+                slurm_failed.append(slurm_file)
+    else:
+        for i, d in enumerate(slrum_job_dirs):
+            if d in slrum_tag_sucess_dirs:
+                slurm_success.append(slurm_job_files[i])
+            else:
+                slurm_failed.append(slurm_job_files[i])
             
     return slurm_failed, slurm_success
 
+
+def slurm_job_done_by_jobtag(slurm_file):
+    with open(slurm_file, 'r') as f:
+        content = f.read()
+    cd_pattern = r'cd\s+([^\n]+)'
+    directories = re.findall(cd_pattern, content)        
+    if not directories:
+        raise Exception("Error! There is no task in the slurm.job file {}".format(slurm_file))
+    for directory in directories:
+        directory = directory.strip()
+        success_file = glob.glob(os.path.join(directory, "*.success"))
+        if len(success_file) > 0:
+            continue
+        else:
+            return False
+    return True
+
+
 def recheck_slurm_by_jobtag(slurm_files:list[str], tag):
     remain_job = []
     for slurm_file in slurm_files:
@@ -76,6 +103,27 @@ def recheck_slurm_by_jobtag(slurm_files:list[str], tag):
                 break
     return remain_job
 
+# def slurm_job_is_done_by_jobtag(dir:str, job_patten:str="*.job", tag_patten:str="tag.*.success"):
+#     slurm_job_files = sorted(glob.glob(os.path.join(dir, job_patten)))
+#     slurm_failed = []
+#     for slurm_file in slurm_job_files:
+#         with open(slurm_file, 'r') as f:
+#             content = f.read()
+#         cd_pattern = r'cd\s+([^\n]+)'
+#         directories = re.findall(cd_pattern, content)        
+#         if not directories:
+#             raise Exception("Error! There is no task in the slurm.job file {}".format(slurm_file))
+#         for directory in directories:
+#             directory = directory.strip()
+#             success_file = os.path.join(directory, tag_patten)
+#             if os.path.exists(success_file):
+#                 continue
+#             else:
+#                 slurm_failed.append(slurm_file)
+#                 break
+#     return slurm_failed
+
+
 '''
 description: 
     split the job_list with groupsize
@@ -85,7 +133,7 @@ return {*} [["job1","job2",...,"job_groupseze"], ..., [..., "job_N", "NONE",...,
 author: wuxingxing
 '''
 def split_job_for_group(groupsize:int , job_list:list[str], parallel_num=1):
-    groupsize = 1 if groupsize is None else groupsize
+    groupsize = len(job_list) if groupsize == -1 else groupsize
     if groupsize > 1:
         groupsize_adj = ceil(groupsize/parallel_num)
         if groupsize_adj*parallel_num > groupsize:

{pwact-0.3.2 → pwact-0.3.4}/pwact.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pwact
-Version: 0.3.2
+Version: 0.3.4
 Summary: PWACT is an open-source automated active learning platform based on MatPL for efficient data sampling.
 Home-page: https://github.com/LonxunQuantum/PWact
 Author: LonxunQuantum

{pwact-0.3.2 → pwact-0.3.4}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="pwact", 
-    version="0.3.2",
+    version="0.3.4",
     author="LonxunQuantum",
     author_email="lonxun@pwmat.com",
     description="PWACT is an open-source automated active learning platform based on MatPL for efficient data sampling.",