shancx 1.9.33.109__py3-none-any.whl → 1.9.33.218__py3-none-any.whl

shancx/{Dsalgor → Algo}/__init__.py

@@ -82,4 +82,40 @@ if __name__ == "__main__":
                 Far=far_value
             )
             manager.log_progress(epoch, idx)
-    manager.close()
+    manager.close()
+    
+import psutil
+import os
+def get_memory():
+    process = psutil.Process(os.getpid())
+    return process.memory_info().rss / 1024 / 1024 
+
+'''
+initial_memory = get_memory()
+logger.info(f"内存使用前: {initial_memory:.2f} MB {sUTC}")
+load_memory = get_memory()
+logger.info(f"创建对象后: {load_memory:.2f} MB {sUTC}")
+final_memory = get_memory()
+logger.info(f"删除对象后: {final_memory:.2f} MB {sUTC}")
+logger.info(f"内存变化: 初始{initial_memory:.2f} -> 峰值{load_memory:.2f} -> 最终{final_memory:.2f}")
+'''
+
+import psutil
+import sys
+def check_memory_threshold(threshold=90):
+    mem = psutil.virtual_memory()
+    return mem.percent >= threshold
+try:
+    if check_memory_threshold(90):
+        raise RuntimeError("The system memory usage is too high. Terminating the task. error")   
+except RuntimeError as e:
+    print(f"Memory warning error: {e}")
+except MemoryError:
+    print("Insufficient memory, task failed. error")
+# except Exception as e:
+#     print(f"未知错误：{e}")
+'''
+if check_memory_threshold(90):
+    raise RuntimeError("The system memory usage is too high. Terminating the task. error")  
+
+'''

shancx/Calmetrics/__init__.py

@@ -1,18 +1,13 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
-import os
-def start():
-    print("import successful")
+import os 
 # constants
 __author__ = 'shancx'
  
-__author_email__ = 'shancx@126.com'
-
+__author_email__ = 'shanhe12@163.com'
 # @Time : 2025/08/19 下午11:31
 # @Author : shanchangxi
-# @File : Calmetrics.py
- 
- 
+# @File : Calmetrics.py 
 from sklearn.metrics import mean_squared_error, r2_score
 from scipy.stats import pearsonr
 import numpy as np
@@ -25,4 +20,78 @@ def calculate_metrics(y_true, y_pred):
     y_pred = y_pred[mask]
     mape = np.mean(np.abs((y_true - y_pred) / y_true)) * 100
     r2 = r2_score(y_true, y_pred)
-    
+    rmse = f"{rmse:0.4f}"
+    correlation = f"{correlation:0.4f}"
+    mape = f"{mape:0.4f}"
+    r2 = f"{r2:0.4f}"
+    return rmse,correlation,mape,r2
+import numpy as np
+class Metrics:
+    def __init__(self, true_values, forecast_values):
+        """
+        初始化Metrics类，传入真值和预测值
+        :param true_values: 实际观测值，二值化数据
+        :param forecast_values: 预测结果，二值化数据
+        """
+        self.true_values = true_values
+        self.forecast_values = forecast_values
+
+    def cal_confusion_matrix(self):
+        """
+        计算混淆矩阵的四个要素: TP, TN, FP, FN
+        :return: 返回TP, TN, FP, FN
+        """
+        TP = np.sum((self.true_values == 1) & (self.forecast_values == 1))  # True Positive
+        TN = np.sum((self.true_values == 0) & (self.forecast_values == 0))  # True Negative
+        FP = np.sum((self.true_values == 0) & (self.forecast_values == 1))  # False Positive
+        FN = np.sum((self.true_values == 1) & (self.forecast_values == 0))  # False Negative
+        
+        return TP, TN, FP, FN
+
+    def cal_ts(self):
+        """
+        计算TS评分
+        :return: TS评分
+        """
+        TP, TN, FP, FN = self.cal_confusion_matrix()
+        ts_score = TP / (TP + FP + FN) if (TP + FP + FN) > 0 else np.nan
+        return ts_score
+
+    def cal_acc(self):
+        """
+        计算准确率
+        :return: 准确率
+        """
+        TP, TN, FP, FN = self.cal_confusion_matrix()
+        accuracy = (TP + TN) / (TP + TN + FP + FN) if (TP + TN + FP + FN) > 0 else np.nan
+        return accuracy
+
+    def cal_pod(self):  
+        """  
+        计算命中率(Probability of Detection, POD)
+        :return: 命中率(POD)
+        """  
+        TP, TN, FP, FN = self.cal_confusion_matrix()  
+        pod = TP / (TP + FN) if (TP + FN) > 0 else np.nan  
+        return pod
+
+
+    def cal_fnr(self):
+            """
+            计算漏报率（False Negative Rate, FNR）
+            :return: 漏报率（FNR）
+            """
+            TP, TN, FP, FN = self.cal_confusion_matrix()
+            fnr = FN / (TP + FN) if (TP + FN) > 0 else np.nan
+            return fnr
+
+    def cal_far(self):
+        """
+        计算空报率（False Alarm Rate, FAR）
+        :return: 空报率（FAR）
+        """
+        TP, TN, FP, FN = self.cal_confusion_matrix()
+        far = FP / (TP + FP) if (TP + FP) > 0 else np.nan
+        return far
+
+        

shancx/Calmetrics/calmetrics.py

@@ -0,0 +1,14 @@
+import numpy as np
+from . import Metrics
+def cal_metrics(obs_v,pre_v):
+    metrics = Metrics(obs_v.ravel(), pre_v.ravel())
+    ts_score = np.around(metrics.cal_ts(),4)
+    accuracy = np.around(metrics.cal_acc(),4)
+    pod = np.around(metrics.cal_pod(),4)
+    fnr = np.around(metrics.cal_fnr(),4)
+    far = np.around(metrics.cal_far(),4)
+    return ts_score,accuracy,pod,fnr,far
+
+"""
+ts_score, accuracy, pod, fnr, far = cal_metrics(obs_v,pre_v)  两个是维度一致的二维数组
+"""

shancx/Calmetrics/rmseR2score.py

@@ -1,4 +1,3 @@
- 
 from sklearn.metrics import mean_squared_error, r2_score
 from scipy.stats import pearsonr
 import numpy as np
@@ -10,8 +9,15 @@ def calculate_metrics(y_true, y_pred):
     y_true = y_true[mask]
     y_pred = y_pred[mask]
     mape = np.mean(np.abs((y_true - y_pred) / y_true)) * 100
-    r2 = r2_score(y_true, y_pred)    
-    
+    r2 = r2_score(y_true, y_pred)
+    rmse = f"{rmse:0.4f}"
+    correlation = f"{correlation:0.4f}"
+    mape = f"{mape:0.4f}"
+    r2 = f"{r2:0.4f}"
+    return rmse,correlation,mape,r2
+"""
+calculate_metrics(data.flatten(),datanpy.flatten())
+"""    
 """
 dt_res[f'{flag}'] = dt_res[f'{flag}'].astype(float)
 dt_res[f'{flag}_p'] = dt_res[f'{flag}_p'].astype(float)
@@ -20,5 +26,10 @@ dt_ = dt_res[(dt_res[f'{flag}'] < 900000.0) & (dt_res[f'{flag}_p'] < 900000.0)]
 dt_.replace([np.inf, -np.inf], np.nan, inplace=True)
 dt_.dropna(inplace=True)
 correlation, rmse, mape, r2 = calculate_metrics(dt_[f"{flag}"], dt_[f"{flag}_p"])
+
+
+y_true = arry1.flatten()
+y_pred = arry2.flatten()
+correlation, rmse, mape, r2  =  calculate_metrics(y_true, y_pred)
 """
     

shancx/{Command.py → Cmd.py}

@@ -2,29 +2,34 @@
 
 import subprocess
 import logging
-
-def runcommand(cmd, timeout= 300) -> bool:
+from shancx.NN import setlogger
+logger = setlogger(level=logging.INFO)
+def runcommand(cmd, timeout=300):
     try:
         result = subprocess.run(
-            cmd,
-            shell=True,  # 关键修改
-            timeout=timeout,
-            check=True,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE
-        )
-        logging.info("List succeeded!")
-        return True
+                                cmd
+                                ,shell=True
+                                ,timeout=timeout
+                                ,check=True
+                                ,capture_output=True
+                                ,text=True
+                                )        
+        for output, label in [(result.stdout, "output"), (result.stderr, "error output")]:
+            if output:
+                logger.info(f"Command {label}:\n{output}")                
+        logger.info("Command succeeded!")
+        return True        
     except subprocess.TimeoutExpired:
-        logging.error(f"Timeout after {timeout} seconds! File may be too large or network is slow.")
+        logger.error(f"Timeout after {timeout} seconds!")
         return False
     except subprocess.CalledProcessError as e:
-        error_msg = e.stderr.decode().strip() if e.stderr else "Unknown error"
-        logging.error(f"Download failed! Return code: {e.returncode}, Error: {error_msg}")
+        error_msg = e.stderr.strip() if e.stderr else "Unknown error"
+        logger.error(f"Command failed! Code: {e.returncode}, Error: {error_msg}")
         return False
     except Exception as e:
-        logging.error(f"Unexpected error: {str(e)}")
+        logger.error(f"Unexpected error: {str(e)}")
         return False
+        
 """
 allDf.dtypes
 

shancx/Config_.py

@@ -0,0 +1,26 @@
+import pygrib
+import numpy as np
+import pandas as pd
+import yaml
+def _load_config(config_path: str) :
+    """加载YAML配置文件"""
+    print(f"load config file Get configuration parameters: {config_path}")
+    with open(config_path, 'r') as f:
+        config = yaml.safe_load(f)
+    return config
+
+import configparser
+import traceback
+def parse_config(path,section,option):
+    cp = configparser.ConfigParser()
+    try:
+        cp.read(path)
+        res = cp.get(section,option)
+    except Exception as e:
+        print(traceback.format_exc())
+        exit()
+    return res
+"""
+Path = "./application.conf"
+radar_path = parse_config(Path, "JY", "radar_path")  #JY是选择部分,radar_path配置路径
+"""

shancx/Df/__init__.py

@@ -19,5 +19,16 @@ def Dtype(df_):
     print(dftypes)
     return dftypes
 
+# 如果env_rang的定义是 (北界, 南界, 西界, 东界)
+def getrange(df,env_rang=None):
+    north, south, west, east = env_rang
+    filtered_data = df[
+        (df["Lat"] < north) & 
+        (df["Lat"] > south) & 
+        (df["Lon"] > west) & 
+        (df["Lon"] < east)
+    ]
+    return filtered_data
+
 #pd.concat(filter(None, results))
 #valid_results = [df for df in results if isinstance(df, pd.DataFrame) and not df.empty]

shancx/Df/tool.py

@@ -1 +0,0 @@
- 

shancx/NN/__init__.py

@@ -4,22 +4,19 @@ import os
 def start():
     print("import successful")
 # constants
-__author__ = 'shancx'
- 
-__author_email__ = 'shancx@126.com'
-
-# @Time : 2023/09/27 下午8:52
-# @Author : shanchangxi
-# @File : util_log.py
- 
-
+__author__ = 'shancx' 
+__author_email__ = 'shanhe12@163.com'
+__time__ = '20251028 21:16'
 import logging
 from logging.handlers import RotatingFileHandler
 import os
 from shancx import crDir 
-def _loggers(logger_name="loggers", root="./logs", phase="project", level=logging.INFO, screen=True, max_bytes=10*1024*1024, backup_count=5, overwrite=False):
+def _loggers(logger_name="loggers", root="./logs", phase="project", level=logging.INFO, screen=True, max_bytes=10*1024*1024, backup_count=5, overwrite=False,handlersflag=False):
     '''set up logger with rotating file handler'''
     l = logging.getLogger(logger_name)
+    if handlersflag:
+        if l.handlers:
+            return l
     formatter = logging.Formatter(
         "%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s", datefmt='%Y-%m-%d %H:%M:%S') 
     log_file = os.path.join(root, '{}.log'.format(phase))
@@ -36,7 +33,6 @@ def _loggers(logger_name="loggers", root="./logs", phase="project", level=loggin
         l.addHandler(sh)    
     return l
 
-
 """
 logger = _loggers(logger_name="test_logger", root=curpathplus, phase="test_log", overwrite=True, screen=True)
 # 测试日志输出
@@ -46,6 +42,48 @@ for i in range(5):
 """
 
 
+import logging
+from logging.handlers import RotatingFileHandler
+import os
+from shancx import crDir 
+import multiprocessing
+def _loggersPlus(logger_name="loggers", root="./logs", phase="project", level=logging.INFO, 
+             screen=True, max_bytes=10 * 1024 * 1024, backup_count=5, overwrite=False, 
+             handlersflag=False, enable_rotating=None):
+    l = logging.getLogger(logger_name)
+    if handlersflag:
+        if l.handlers:
+            return l    
+    formatter = logging.Formatter(
+        "%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s", 
+        datefmt='%Y-%m-%d %H:%M:%S'
+    )    
+    log_file = os.path.join(root, '{}.log'.format(phase))
+    crDir(log_file)
+    mode = 'w' if overwrite else 'a'    
+    if enable_rotating is None:
+        enable_rotating = (multiprocessing.current_process().name == 'MainProcess')
+    
+    if enable_rotating:
+        fh = RotatingFileHandler(
+            log_file, 
+            mode=mode, 
+            maxBytes=max_bytes, 
+            backupCount=backup_count
+        )
+    else:
+        fh = logging.FileHandler(log_file, mode=mode)
+    
+    fh.setFormatter(formatter)    
+    l.setLevel(level)
+    l.addHandler(fh)    
+    
+    if screen:
+        sh = logging.StreamHandler()
+        sh.setFormatter(formatter)
+        l.addHandler(sh)  
+    return l
+
 
 import logging
 def setlogger(level=logging.INFO):
@@ -64,4 +102,155 @@ if __name__ == "__main__":
     logger.debug("这条日志会显示")  # 默认情况下DEBUG不显示，但因为我们设置了级别，现在会显示
     logger.info("这是一条INFO日志")
 '''
+import os
+import glob
+import logging
+from datetime import datetime
+from shancx import loggers as logger
+def cleanupLogs(log_dir='/mnt/wtx_weather_forecast/scx/SATH9SEAStest/logs', keep_count=10, 
+                 pattern='*.log', recursive=False, dry_run=False):
+    stats = {'total_dirs': 0, 'deleted_files': 0, 'errors': []}    
+    def _cleanup_dir(directory):
+        stats['total_dirs'] += 1
+        if not os.path.exists(directory):
+            logging.warning(f"目录不存在: {directory}")
+            return
+        file_paths = glob.glob(os.path.join(directory, pattern))
+        log_files = [(path, os.path.getmtime(path)) for path in file_paths]
+        log_files.sort(key=lambda x: x[1], reverse=True)        
+        if len(log_files) <= keep_count:
+            logging.info(f"目录 {directory} 中的文件数量 ({len(log_files)}) 不超过保留数量 ({keep_count})，无需清理")
+            return
+        files_to_delete = log_files[keep_count:]
+        for file_path, mtime in files_to_delete:
+            try:
+                if dry_run:
+                    logging.info(f"[试运行] 将删除: {file_path} (修改时间: {datetime.fromtimestamp(mtime).strftime('%Y-%m-%d %H:%M:%S')})")
+                else:
+                    os.remove(file_path)
+                    logging.info(f"已删除: {file_path} (修改时间: {datetime.fromtimestamp(mtime).strftime('%Y-%m-%d %H:%M:%S')})")
+                stats['deleted_files'] += 1
+            except Exception as e:
+                error_msg = f"删除失败 {file_path}: {str(e)}"
+                logging.error(error_msg)
+                stats['errors'].append(error_msg)
+    if recursive:
+        for root, _, _ in os.walk(log_dir):
+            _cleanup_dir(root)
+    else:
+        _cleanup_dir(log_dir)    
+    return stats
+"""
+if __name__ == "__main__": 
+    dir = "/mnt/wtx_weather_forecast/scx/SATH9SEAStest/logs/H9SEAS/"
+    stats = cleanupLogs(dir,3, '*.log', False, False)    
+    logging.info(f"清理完成: 处理目录数={stats['total_dirs']}, 删除文件数={stats['deleted_files']}, 错误数={len(stats['errors'])}")
+    if stats['errors']:
+        logging.error(f"错误详情: {stats['errors']}")
+"""  
+
+from itertools import product
+from concurrent.futures import ProcessPoolExecutor as PoolExecutor, as_completed 
+import sys
+from tqdm import tqdm 
+def validate_param_list(param_list):
+    if len(param_list) == 0:
+        raise ValueError("param_list cannot be empty.")    
+    for sublist in param_list:
+        if len(sublist) == 0:
+            raise ValueError("Sub-lists in param_list cannot be empty.")     
+def Mul_sub(task, param_list, num=6):
+    print(f"Pro num {num}")
+    validate_param_list(param_list)
+    if len(param_list) == 1:
+        product_list = [(x,) for x in param_list[0]]
+    else:
+        product_list = list(product(*param_list))
+    with PoolExecutor(max_workers=num) as executor:
+        try:           
+            futures = [executor.submit(task, item) for item in product_list]
+            for future in tqdm(as_completed(futures), total=len(futures), desc="Processing tasks", unit="task"):  
+                future.result() 
+        except KeyboardInterrupt:
+            sys.exit(1)    
+    print("All tasks completed")
+
+from concurrent.futures import ThreadPoolExecutor
+from itertools import product
+def Mul_TH(task, param_list, max_workers=6):
+    print(f"Thread num: {max_workers}")
+    validate_param_list(param_list)
+    task_args = [
+        (arg,) if len(param_list) == 1 else arg
+        for arg in (
+            param_list[0] if len(param_list) == 1 
+            else product(*param_list)
+        )
+    ]    
+    with ThreadPoolExecutor(max_workers) as ex:
+        try:
+            list(ex.map(task, task_args))
+        except KeyboardInterrupt:
+            print("\n用户中断操作")
+            ex.shutdown(wait=False)
+            sys.exit(1) 
+
+import traceback
+import shutil, os
+def safe_del(path):
+    try:
+        shutil.rmtree(path) if os.path.isdir(path) else None
+        print(f"{path} deleted")
+    except Exception:
+        print(traceback.format_exc())
+
+"""
+safe_del("./data")
+"""
+
+import os, glob
+def clean_files(folder, keep=0):
+    if os.path.isdir(folder):
+        try:
+            files = [os.path.join(folder, f) for f in os.listdir(folder) 
+                    if os.path.isfile(os.path.join(folder, f))]
+            if keep > 0 and len(files) > keep:
+                files.sort(key=os.path.getmtime)
+                [os.remove(f) for f in files[:-keep]]
+            elif keep == 0:
+                [os.remove(f) for f in files]
+        except Exception as e:
+            print(traceback.format_exc())
+
+"""
+clean_files("./logs", keep=10)   
+clean_files("./temp") 
+"""        
 
+# import os
+# from datetime import datetime
+# from shancx.NN import _loggers
+# from shancx import lock_file
+# from shancx.wait import check_lock
+# from shancx import crDir
+# logger =_loggers()
+# def check_process_data(UTC, sat_cd,basepath ="/mnt/wtx_weather_forecast/scx/test/lock_files" ): 
+#     try:
+#         UTCStr = UTC.strftime("%Y%m%d%H%M")
+#         file = f"/mnt/wtx_weather_forecast/scx/test/lock_files/{sat_cd}/{UTCStr[:4]}/{UTCStr[:8]}/File_{UTCStr}.lock"
+#         crDir(file)
+#         if not lock_file(file):
+#             if check_lock(file):
+#                 logger.info("data is making or maked")
+#                 return True ,file       
+#         return False,file        
+#     except Exception as e:
+#         logger.error(f"Error in check_and_process_data: {str(e)}")
+#         return False,file  
+# """
+# flag1,file = check_process_data(UTC, "H9SEAS" )
+# if flag1:
+#     sys.exit() 
+# if os.path.exists(output_path): #配合使用
+#    sys.exit()    
+# """

shancx/{path.py → Path1.py}

@@ -2,14 +2,14 @@
     
 import os 
 from pathlib import Path
-def CurPath():
+def curPath_():
     current_file_path = os.path.abspath(__file__) 
     current_folder_path = os.path.dirname(current_file_path)
     parent = Path(__file__).parent
     return current_folder_path,parent
 
 import sys
-def CurPathplus():
+def curPath():
     # 获取当前执行文件的绝对路径
     current_file_path = os.path.abspath(sys.argv[0])
     current_folder_path = os.path.dirname(current_file_path)
@@ -157,6 +157,5 @@ if __name__ == '__main__':
         Mul_sub(map_data,[timeList1],31)
     except Exception as e:
         print(traceback.format_exc())
-
 """