shancx 1.9.33.109__py3-none-any.whl → 1.9.33.218__py3-none-any.whl

shancx/__init__.py

@@ -33,10 +33,9 @@ print_handler = logging.StreamHandler()
 print_handler.setFormatter(formatter)   
 loggers.addHandler(print_handler)
 
-
-from pathlib import Path
+from pathlib import Path as PathlibPath
 def crDir(path):
-    path_obj = Path(path)
+    path_obj = PathlibPath(path)
     directory = path_obj.parent if path_obj.suffix else path_obj
     directory.mkdir(parents=True, exist_ok=True) 
 
@@ -73,21 +72,6 @@ def validate_param_list(param_list):
             raise ValueError("Sub-lists in param_list cannot be empty.")        
 
 from itertools import product
-from concurrent.futures import ProcessPoolExecutor as PoolExecutor
-def Mul_(map_fun,param_list,num=6):
-    print(f"Pro num {num}")
-    validate_param_list(param_list)
-    if len(param_list) == 1:
-        product_List = [(x,) for x in param_list[0]]
-    else:
-        product_List = list(product(*param_list))
-    with PoolExecutor(num) as p:
-        try:
-             P_data = [result for result in tqdm(p.map(map_fun, product_List), total=len(product_List), desc="Processing", unit="task")]
-        except KeyboardInterrupt:
-           sys.exit(1)  
-    return list(P_data)
-
 from concurrent.futures import ProcessPoolExecutor as PoolExecutor, as_completed 
 import sys
 from tqdm import tqdm 
@@ -102,83 +86,12 @@ def Mul_sub(task, param_list, num=6):
     with PoolExecutor(max_workers=num) as executor:
         try:           
             futures = [executor.submit(task, item) for item in product_list]
-            for future in tqdm(as_completed(futures), total=len(futures), desc="Processing tasks", unit="task"):  #results = [ future for future in tqdm(as_completed(futures), total=len(futures), desc="Processing", unit="task")]
+            for future in tqdm(as_completed(futures), total=len(futures), desc="Processing tasks", unit="task"):   
                 results.append(future.result())  
         except KeyboardInterrupt:
             sys.exit(1)    
     return results
-
-
-from concurrent.futures import ProcessPoolExecutor as PoolExecutor, as_completed, TimeoutError
-import sys
-from tqdm import tqdm
-from itertools import product
  
-def Mul_subT(task, param_list, num=3, timeout=3600):
-    print(f"Process num: {num}")
-    validate_param_list(param_list)
-    if len(param_list) == 1:
-        product_list = [(x,) for x in param_list[0]]
-    else:
-        product_list = list(product(*param_list))    
-    results = []
-    with PoolExecutor(max_workers=num) as executor:
-        try:
-            futures = {executor.submit(task, item): item for item in product_list}
-            for future in tqdm(as_completed(futures), 
-                             total=len(futures), 
-                             desc="Processing tasks", 
-                             unit="task"):
-                try:
-                    res = future.result(timeout=timeout)
-                    results.append(res)
-                except TimeoutError:
-                    print(f"\nTimeout on task {futures[future]}")
-                    future.cancel()
-                except Exception as e:
-                    print(f"\nTask failed: {futures[future]}, error: {str(e)}")                    
-        except KeyboardInterrupt:
-            print("\nReceived keyboard interrupt, terminating...")
-            executor.shutdown(wait=False)
-            sys.exit(1)    
-    return results
-
-
-from concurrent.futures import ProcessPoolExecutor as PoolExecutor, as_completed, TimeoutError
-import sys
-from tqdm import tqdm
-from itertools import product 
-
-def Mul_subT_S(task, param_list, num=3, timeout=3600):
-    print(f"Process num: {num}")
-    validate_param_list(param_list)
-    if len(param_list) == 1:
-        product_list = [(x,) for x in param_list[0]]
-    else:
-        product_list = list(product(*param_list))
-    results = [None] * len(product_list)    
-    with PoolExecutor(max_workers=num) as executor:
-        futures = {executor.submit(task, item): idx for idx, item in enumerate(product_list)}
-        
-        try:
-            for future in tqdm(as_completed(futures), 
-                             total=len(futures), 
-                             desc="Processing tasks", 
-                             unit="task"):
-                idx = futures[future]
-                try:
-                    results[idx] = future.result(timeout=timeout)
-                except TimeoutError:
-                    print(f"\nTimeout on task {product_list[idx]}")
-                    future.cancel()
-                except Exception as e:
-                    print(f"\nTask failed: {product_list[idx]}, error: {str(e)}")                    
-        except KeyboardInterrupt:
-            print("\nReceived keyboard interrupt, terminating...")
-            executor.shutdown(wait=False)
-            sys.exit(1)    
-    return results
-
 def Mul_sub_S(task, param_list, num=6):
     print(f"Pro num {num}")
     validate_param_list(param_list)
@@ -197,7 +110,6 @@ def Mul_sub_S(task, param_list, num=6):
             sys.exit(1)       
     return results
 
-
 from concurrent.futures import ProcessPoolExecutor as PoolExecutor, as_completed
 import sys
 from tqdm import tqdm
@@ -235,36 +147,6 @@ def Mul_sub_Splus(task, param_lists, num=6):
             sys.exit(1)    
     return results
 
-import multiprocessing
-from multiprocessing import Pool
-from tqdm import tqdm
-def parallel(add_numbers,tasks, num_processes=6):
-    if num_processes is None:
-        num_processes = 6   
-    with Pool(processes=num_processes) as pool:
-        results = list(tqdm(
-            pool.imap(add_numbers, tasks),
-            total=len(tasks),
-            desc="Processing",
-            unit="task"
-        ))    
-    return results
-
-from concurrent.futures import ThreadPoolExecutor
-def Mul_th(task, task_args, workers=2):
-    with ThreadPoolExecutor(workers) as ex:
-        try:
-            return list(ex.map(task, task_args))
-        except KeyboardInterrupt:
-            print("\n用户中断操作")
-            ex.shutdown(wait=False)
-            sys.exit(1) 
-
-# task_args = [(cr_f1[i],) for i in range(cr_f1.shape[0])]
-#task_args = [[chid,utcyear, utcmonth,latArr,lonArr]  
-#                for chid in allcchannels]  
-# results = Mul_th(func, task_args)
-
 from concurrent.futures import ThreadPoolExecutor
 from itertools import product
 def Mul_TH(task, param_list, max_workers=6):
@@ -285,121 +167,104 @@ def Mul_TH(task, param_list, max_workers=6):
             ex.shutdown(wait=False)
             sys.exit(1) 
 
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from itertools import product
-from tqdm import tqdm
-import sys
-
-def Mul_sub_th(func, param_list=None,workers=2,  show_progress=True):
-    print(f"Pro num {workers}")
-    validate_param_list(param_list)
-    iterable = [
-            (arg,) if len(param_list) == 1 else arg
-            for arg in (
-                param_list[0] if len(param_list) == 1 
-                else product(*param_list)
-            )
-        ]    
-    try:
-        if show_progress:
-            with ThreadPoolExecutor(max_workers=workers) as executor:
-                futures = [
-                    executor.submit(func, *args) if isinstance(args, tuple) 
-                    else executor.submit(func, args)
-                    for args in iterable
-                ]                
-                results = []
-                progress = tqdm(as_completed(futures), 
-                              total=len(futures),
-                              desc=f"Threading(workers={workers})",
-                              unit="task")                
-                try:
-                    for future in progress:
-                        results.append(future.result())
-                    return results
-                except KeyboardInterrupt:
-                    progress.close()
-                    print("\n用户中断操作，正在取消剩余任务...")
-                    for f in futures:
-                        f.cancel()
-                    executor.shutdown(wait=False)
-                    sys.exit(1)
-        with ThreadPoolExecutor(workers) as ex:
-            try:
-                return list(ex.map(func, iterable))
-            except KeyboardInterrupt:
-                print("\n用户中断操作")
-                ex.shutdown(wait=False)
-                sys.exit(1)                
-    except Exception as e:
-        print(f"执行出错: {str(e)}")
-        sys.exit(1)
- 
 from tqdm import tqdm  
 def _tqdm(train_loader,desc="Training Progress",epoch=None):
     epoch = epoch +1 if epoch is not None else None
     descdict = {"val":"Val Progress", "train":"Training Progress"}
     return tqdm(train_loader, desc=f"Epoch {epoch} {descdict.get(desc,'DESC')}", ncols=100)
 
+import os
+import datetime
+from pathlib import Path
+def lock_file(lock_file_path):
+    if os.path.exists(lock_file_path):
+        print(f" {lock_file_path} is existded ")
+        return False
+    try:
+        os.makedirs(os.path.dirname(lock_file_path), exist_ok=True)
+        with open(lock_file_path, 'w') as f:
+            f.write(f"process_id:{os.getpid()},create_time:{datetime.datetime.now()}")
+            return True 
+    except Exception as e:
+        print(f"创建锁文件失败 create lock failed : {e}")
+        return False
+"""
+lock_file = f"{output_base_path}{sat_code}/lock_files/MSP2_WTX_AIW_QPF_L88_GLOB_{utc_time_str}.lock"    
+if os.path.exists(outpath):
+    print(f" 目标文件已存在，跳过处理: {outpath}")
+    return True    
+if os.path.exists(lock_file):
+    print(f"锁文件存在，已有程序在处理时次 {utc_time_str}，跳过")
+    return False 
+"""
+import os
+import time
+def is_process_alive(pid):
+    try:
+        os.kill(pid, 0)  
+        return True
+    except OSError:
+        return False
+def check_lock(lock_file):
+    if not os.path.exists(lock_file):
+        return False    
+    try:
+        with open(lock_file, 'r') as f:
+            content = f.read().strip()        
+        if 'process_id:' in content and 'create_time:' in content:
+            pid_str = content.split('process_id:')[1].split(',')[0]
+            pid = int(pid_str)            
+            if not is_process_alive(pid):
+                print(f"进程 {pid} 已消亡，清理锁文件")
+                os.remove(lock_file)
+                return False 
+            else:
+                print(f"进程 {pid} 仍在运行，跳过执行")
+                return True                
+    except Exception as e:
+        print(f"锁文件解析错误，清理: {e}")
+        os.remove(lock_file)
+        return False    
+    return False
+"""
+if  check_lock(lock_file):
+    return False
+"""
 import multiprocessing
-import sys
-from typing import Optional, Literal
-def set_multiprocessing_method(
-    method: Optional[Literal['auto', 'fork', 'spawn', 'forkserver']] = 'auto',
-    verbose: bool = True
-) -> str:
-    """
-        method: 
-            - 'auto'   : 自动选择当前平台最优方式 (默认)
-            - 'fork'   : 强制使用 fork (仅Unix)
-            - 'spawn'  : 强制使用 spawn (所有平台)
-            - 'forkserver' : 强制使用 forkserver (仅Unix)
-        >>> set_multiprocessing_method('auto')  # 自动选择
-        >>> set_multiprocessing_method('spawn') # 强制spawn
-    """
-    available_methods = multiprocessing.get_all_start_methods()
-    current_method = multiprocessing.get_start_method()
-    if method == 'auto':
-        if sys.platform == 'linux' and 'fork' in available_methods:
-            method = 'fork'
-        elif 'forkserver' in available_methods:
-            method = 'forkserver'
-        else:
-            method = 'spawn'
-    if method not in available_methods:
-        raise ValueError(
-            f"启动方法 '{method}' 在当前平台不可用。可用方法: {available_methods}"
-        )
-    if current_method == method:
-        if verbose:
-            print(f"⚠️ 启动方法已是 '{method}'，无需更改")
-        return method
+def set_multiprocessing(method="spawn", verbose=True):  # 补充缺失的verbose参数定义
     try:
         multiprocessing.set_start_method(method, force=True)
         if verbose:
-            print(f"✅ 设置多进程启动方法为: '{method}'")
-            if method == 'fork':
-                print("   ⚠️ 注意: fork 可能引发线程安全问题 (详见文档说明)")
+            print(f"The startup method is: '{method}'")
         return method
-    except RuntimeError as e:
-        raise RuntimeError(
-            f"无法设置启动方法为 '{method}': {str(e)}\n"
-            f"当前方法: {current_method}, 可用方法: {available_methods}"
-        ) from e
-
-
+    except RuntimeError as e:   
+        if verbose:  
+            print(f"Setup failed: {e}")
+        return multiprocessing.get_start_method()
+    
+"""
+current_method = set_multiprocessing_method()
+# 输出：已设置多进程启动方式: 'spawn'
+print("当前启动方式：", current_method)  # 输出：当前启动方式：spawn
+"""
 """
 if __name__ == '__main__':
     set_multiprocessing_method('auto')
     print("当前启动方法:", multiprocessing.get_start_method())
 
 """
-
+        
 """
 zoom插值
 from scipy.ndimage import zoom
 d = zoom(d_clip, [4201/169,6201/249], order=1)[:-1, :-1]
 """
+"""
+torch.cuda.empty_cache()
+del 
+gc.collect()
+"""
+
 '''
 from multiprocessing import Pool
 '''
@@ -434,24 +299,6 @@ logger.addHandler(print_handler)
 ###解决方法  pip install torch==2.4.0  torchvision    torchaudio三个同时安装  python 3.12  解决cuda启动不了的问题
 Res网络
 '''
-'''
-import concurrent.futures
-from itertools import product
-def task(args):
-    args1,args2  = args
-    print( f"Task ({args1}, {args2}) , result")
-    return (args1,args2,5)
-
-def Mul_sub(task, pro):
-    product_list = product(*pro)
-    with concurrent.futures.ThreadPoolExecutor() as executor:
-        futures = [executor.submit(task, item) for item in product_list]
-    results = [future.result() for future in concurrent.futures.as_completed(futures)]   
-    return results
-res = Mul_sub(task, [[1, 23, 4, 5], ["n"]])
-print("res")
-print(res)
-'''
 
 """
 find /mnt/wtx_weather_forecast/scx/SpiderGLOBPNGSource -type f -name "*.png" -mtime +3 -exec rm {} \;

shancx/bak.py

@@ -109,6 +109,8 @@ if __name__ == '__main__':
     else: 
         print("cmdList is empty, skipping the call.")
         raise ValueError("cmdList is empty, cannot execute command.")
+
+
 CUDA_LAUNCH_BLOCKING=1 python makeHis.py --times 202410010048,202410110048 --gpu=0 --isDebug --sepSec 3600 --pool 5
 CUDA_LAUNCH_BLOCKING=1 python makeHis1.py --times 202410010048,202410110048 --gpu=0 --isDebug --sepSec 3600 --pool 5
 """
@@ -154,9 +156,11 @@ python makeDOC_newv2.py --times $end_date"0000",$end_date"2359" --tag $tag
 """
 frile name :launch.json
 args:
+
 {
     "version": "0.2.0",
     "configurations": [   
+
         {
             "name": "Python: Current File",
             "type": "debugpy",
@@ -166,32 +170,35 @@ args:
             "cwd": "${fileDirname}",
             "purpose": ["debug-in-terminal"],
             "justMyCode": false,
-            "args": [  
-                "--times", "202409160000,202409180000" 
+            "args": [
+            "--times", "202410010042,202410020042",
+            "--isDebug" ,
+            "--isOverwrite", 
+            "--sepSec", "3600",
+            "--gpu", "0"
             ]
         }
     ]
 }
 
+
 {
     "version": "0.2.0",
-    "configurations": [   
-
+    "configurations": [    
         {
-            "name": "Python: Current File",
+            "name": "VAE: Train SEVIR-LR",
             "type": "debugpy",
             "request": "launch",
-            "program": "${file}",
+            "program": "${workspaceFolder}/scripts/vae/sevirlr/train.py",
             "console": "integratedTerminal",
-            "cwd": "${fileDirname}",
+            "cwd": "${workspaceFolder}",
             "purpose": ["debug-in-terminal"],
             "justMyCode": false,
+            "python": "/home/scx/miniconda3/envs/mqpf/bin/python",
             "args": [
-            "--times", "202410010042,202410020042",
-            "--isDebug" ,
-            "--isOverwrite", 
-            "--sepSec", "3600",
-            "--gpu", "0"
+                "--save", "vae_sevirlr_train",
+                "--gpus", "1",
+                "--cfg", "${workspaceFolder}/scripts/vae/sevirlr/cfg.yaml"
             ]
         }
     ]
@@ -450,8 +457,6 @@ sudo chmod -R 777 /mnt/wtx_weather_forecast/scx/MSG/MSG_Data
 """
 
 """
-
-
 import os
 import numpy as np
 import pandas as pd
@@ -617,10 +622,8 @@ def map_fun(conf):
             logger.error(f"{UTC} error {e}")            
             logger.info(traceback.format_exc())
             print(traceback.format_exc())
-        return 
-        
+        return   
         
-
     def getCheckArea(self, eps):
         '''
         split area
@@ -678,46 +681,21 @@ class drawPng():
 """
 """
 conda install conda-forge::cudatoolkit==11.8.0
+""" 
 """
-"""
-# 删除 scx 用户（同时删除家目录）
-sudo userdel -r scx
-# 再删除 scx 组
-sudo groupdel scx
-# 重新创建组（GID=1015）
-sudo groupadd -g 1015 scx
-# 重新创建用户（UID=1015，GID=1015）
-sudo useradd -u 1015 -g scx -m -s /bin/bash scx
-# 设置密码
-sudo passwd scx
-# 赋予 sudo 权限
-sudo usermod -aG sudo scx
-2. 如果不想删除用户，可以修改用户的主组
-bash
-# 查看 scx 用户当前的主组
-id scx
-# 修改 scx 用户的主组为其他组（如 users）
-sudo usermod -g users scx
-# 现在可以删除 scx 组
-sudo groupdel scx
-# 重新创建 scx 组（GID=1015）
+sudo pkill -9 -u scx 2>/dev/null || true
+sudo groupdel scx 2>/dev/null; sudo userdel -r scx 2>/dev/null; sudo groupadd -g 1015 scx && sudo useradd -m -u 1015 -g 1015 -s /bin/bash scx && echo "scx:123456" | sudo chpasswd && sudo chown -R scx:scx /home/scx && id scx
+sudo pkill -9 -u scx 2>/dev/null || true
+sudo ps aux | grep scx | awk '{print $2}' | xargs -r sudo kill -9 2>/dev/null || true
+sudo userdel -rf scx 2>/dev/null || true
+sudo groupdel -f scx 2>/dev/null || true
+sleep 2
 sudo groupadd -g 1015 scx
-# 修改 scx 用户的主组回 scx
-sudo usermod -g scx scx
-验证
-bash
+sudo useradd -m -u 1015 -g 1015 -s /bin/bash scx
+echo "scx:123456" | sudo chpasswd
 id scx
-预期输出：
-text
-uid=1015(scx) gid=1015(scx) groups=1015(scx),27(sudo)
-
-groupmod -g 1016 scx
-sudo usermod -u 1015 scx
-# 修改用户的主组（GID）
-sudo usermod -g scx scx
-# 确保家目录权限正确
-sudo chown -R scx:scx /home/scx
 """
+
 """
     from hjnwtx.mkNCHJN import dataClass, mkNCCommonUni,envelope,timeSeq,mkDir
     env = envelope(35,10,108,125)
@@ -740,4 +718,51 @@ find  /mnt/wtx_weather_forecast/scx/GOES -type f -name "*.txt" -mmin +300 -delet
 """
 """
 mask = (mask_data == 0).to(device)
+"""
+
+"""
+import pdb
+pdb.set_trace()
+l 10 查看最近10行
+(Pdb) !a = 5  # 在当前作用域创建变量 a
+(Pdb) p a      
+for i in range(5): print(i)  
+n  执行下一行
+c  继续执行
+q  退出
+(Pdb) n          # Next line
+(Pdb) s          # Step into function
+(Pdb) c          # Continue execution
+(Pdb) b <line>   # Set breakpoint
+(Pdb) q          # Quit debugger
+(Pdb) !import os; os.listdir('.')  
+(Pdb) p locals()    # Show local variables
+(Pdb) p globals()   # Show global variables  
+(Pdb) where         # Show stack trace
+(Pdb) list          # Show current code context
+"""
+"""
+np.savez_compressed(output_path.replace('.npy', '.npz'), data=data)
+data = np.load(output_path.replace('.npy', '.npz'))['data']
+with np.load(output_path.replace('.npy', '.npz')) as npz_file:
+    data = npz_file['data']
+with np.load(output_path.replace('.npy', '.npz')) as npz_file:
+    data = npz_file[npz_file.files[0]]  
+"""
+"""
+lats = np.linspace(15, 60, h)
+lons = np.linspace(70, 140, w)
+
+# import cv2
+# [ cv2.resize(i, (6200, 4200),interpolation=cv2.INTER_LINEAR)  for i in dP.df_Mat ]
+B08_fixed = B08.astype(np.float32)
+print(f"转换后dtype: {B08_fixed.dtype}")     高位字节前后问题， cv2希望高阶字字节在后
+print(f"转换后范围: {B08_fixed.min()} ~ {B08_fixed.max()}")
+d_test = cv2.resize(B08_fixed, (100, 100), interpolation=cv2.INTER_CUBIC)         #双三次插值
+d_cv2 = cv2.resize(B08, (6200, 4200), interpolation=cv2.INTER_LINEAR)   #双线性插值
+print(f"测试插值范围: {d_test.min()} ~ {d_test.max()}")
+"""
+"""
+import torch
+torch.cuda.empty_cache()
 """

shancx/geosProj.py

@@ -17,7 +17,7 @@ class goesProj():
         self.CFAC=FAC[resolution]
         self.LFAC=FAC[resolution]
 
-    def transform(self,latD,lonDe,RO=0.5):
+    def transform(self,latD,lonDe,RO=22):
         lat=np.radians(latD)
         lon=np.radians(lonDe)
         ba2=np.square(self.eb/self.ea)
@@ -58,7 +58,7 @@ class goesProjMSG10():
         self.CFAC=FAC[resolution]
         self.LFAC=FAC[resolution]
 
-    def transform(self,latD,lonDe,ROC=5,ROL=-5):
+    def transform(self,latD,lonDe,ROC=-10,ROL=-30):
         lat=np.radians(latD)
         lon=np.radians(lonDe)
         ba2=np.square(self.eb/self.ea)

shancx/wait.py

@@ -157,6 +157,39 @@ def checkSize(pattern: str,size_mb: float = 50.0,timeout: int = 180,interval: in
 checkSize(pattern: str,size_mb: float = 50.0,timeout: int = 180,interval: int = 5)
 """ 
 
+import os
+import time
+def is_process_alive(pid):
+    try:
+        os.kill(pid, 0)  
+        return True
+    except OSError:
+        return False
+def check_lock(lock_file):
+    if not os.path.exists(lock_file):
+        return False    
+    try:
+        with open(lock_file, 'r') as f:
+            content = f.read().strip()        
+        if 'process_id:' in content and 'create_time:' in content:
+            pid_str = content.split('process_id:')[1].split(',')[0]
+            pid = int(pid_str)            
+            if not is_process_alive(pid):
+                print(f"进程 {pid} 已消亡，清理锁文件")
+                os.remove(lock_file)
+                return False 
+            else:
+                print(f"进程 {pid} 仍在运行，跳过执行")
+                return True                
+    except Exception as e:
+        print(f"锁文件解析错误，清理: {e}")
+        os.remove(lock_file)
+        return False    
+    return False
+"""
+if  check_lock(lock_file):
+    return False
+"""
 
 import numpy as np
 from typing import Union
@@ -209,4 +242,5 @@ def safe_delete(path_pattern):
 """
             if os.path.exists(zip_file):
                 safe_delete(zip_file)
-"""
+"""
+

{shancx-1.9.33.109.dist-info → shancx-1.9.33.218.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: shancx
-Version: 1.9.33.109
+Version: 1.9.33.218
 Summary: A simple timer decorator
 Home-page: https://gitee.com/shancx
 Author: shancx
@@ -13,13 +13,21 @@ Description-Content-Type: text/markdown
 Requires-Dist: tqdm
 Requires-Dist: pandas
 Requires-Dist: matplotlib
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
-# My Timer Decorator
+# Welecome to shancx
 
 A simple Python package that provides a timer decorator to measure the execution time of functions.
 
 ## Installation
-
  
 pip install shancx