shancx 1.8.92__py3-none-any.whl → 1.9.33.218__py3-none-any.whl

shancx/Hug/__init__.py

@@ -0,0 +1,81 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# gdown --folder https://drive.google.com/drive/folders/1D3bf2G2o4Hv-Ale26YW18r1Wrh7oIAwK  文件夹下所有内容太
+from modelscope.hub.snapshot_download import snapshot_download
+model_dir = snapshot_download('', cache_dir='autodl-tmp', revision='master', ignore_file_pattern='.bn')
+from datasets import load_dataset
+dataset = load_dataset("ayz2/ldm_pdes", download_mode="force_redownload")
+print(dataset)
+# dataset.save_to_disk('/path/to/save/dataset')
+for split_name, split_dataset in dataset.items():
+    chunk_size = 5000  # 每片的样本数量，可根据内存大小调整
+    for i in range(0, len(split_dataset), chunk_size):
+        subset = split_dataset.select(range(i, min(i + chunk_size, len(split_dataset))))
+        subset.save_to_disk(f"./{split_name}_chunk_{i // chunk_size}")
+
+from huggingface_hub import hf_hub_download
+from shancx import crDir
+# 下载权重文件
+repo_id = "ayz2/ldm_pdes"
+filename = "ae_cylinder.ckpt"
+crDir("/mnt/wtx_weather_forecast/scx/ldm_pdes_checkpoint")
+save_dir = "/mnt/wtx_weather_forecast/scx/ldm_pdes_checkpoint"
+save_path = hf_hub_download(repo_id=repo_id, filename=filename, cache_dir=save_dir)
+ 
+
+print(f"权重文件已下载到: {save_path}")
+
+
+from modelscope.hub.snapshot_download import snapshot_download
+crDir("/mnt/wtx_weather_forecast/scx/ldm_pdes_checkpoint")
+save_dir = "/mnt/wtx_weather_forecast/scx/ldm_pdes_checkpoint"
+model_dir = snapshot_download('ayz2/ldm_pdes', cache_dir=save_dir, revision='master', ignore_file_pattern='.bn')     下载 ae_cylinder.ckpt"
+
+
+from modelscope.hub.snapshot_download import snapshot_download
+from shancx import crDir
+save_dir = "/mnt/wtx_weather_forecast/scx/ldm_pdes_checkpoint"
+crDir(save_dir)
+model_dir = snapshot_download(
+    'ayz2/ldm_pdes',
+    cache_dir=save_dir,
+    revision='master',
+    ignore_file_pattern='.bn')
+print(f"模型已下载到: {model_dir}")
+
+
+
+
+
+from google.colab import drive
+import requests
+import os
+
+# 挂载 Google Drive
+drive.mount('/content/drive')
+ptname = "ae_cylinder.ckpt"
+# 文件下载的URL
+file_url = f"https://huggingface.co/datasets/ayz2/ldm_pdes/resolve/main/{ptname}"
+
+# Google Drive中的保存路径（修改为你想保存的位置）
+output_path = "/content/drive/My Drive/{ptname}"
+
+# 下载文件并保存到指定路径
+def download_file(url, output_path):
+    try:
+        with requests.get(url, stream=True) as response:
+            response.raise_for_status()
+            with open(output_path, "wb") as file:
+                for chunk in response.iter_content(chunk_size=8192):
+                    if chunk:  # 防止空块
+                        file.write(chunk)
+        print(f"文件已成功下载并保存到 Google Drive: {output_path}")
+    except Exception as e:
+        print(f"下载文件时出错: {e}")
+download_file(file_url, output_path)
+
+import zipfile
+def unzip_file(zip_path, extract_to):
+    with zipfile.ZipFile(zip_path, 'r') as zip_ref:
+        zip_ref.extractall(extract_to)
+unzip_file(output_path, os.getcwd())

shancx/Inst.py

@@ -0,0 +1,22 @@
+import sys
+import os
+def current_dir_sys_path():
+    sys.path.append(os.getcwd())  #current_dir_sys_path()
+
+import importlib
+def instantiate_config(config):
+    if not "target" in config:
+        if config == '__is_first_stage__':
+            return None
+        elif config == "__is_unconditional__":
+            return None
+        raise KeyError("Expected key `target` to instantiate.")
+    return get_obj_from_str(config["target"])(**config.get("params", dict()))
+def get_obj_from_str(string, reload=False):
+    module, cls = string.rsplit(".", 1)
+    if reload:
+        module_imp = importlib.import_module(module)
+        importlib.reload(module_imp)
+    return getattr(importlib.import_module(module, package=None), cls)
+
+ 

shancx/Lib.py

@@ -0,0 +1,31 @@
+import cv2
+import copy
+import numpy as np
+
+def QC(mat, dbzTH=10, areaTH=20):  # 主QC函数
+    mat1 = copy.copy(mat)
+    varMask = np.where(mat1 > dbzTH, 1, 0)
+    varMask = varMask.astype(np.uint8)
+
+    ret, img_thre = cv2.threshold(varMask, 0, 255, cv2.THRESH_BINARY)
+    num_labels, labels, stats, centroids = cv2.connectedComponentsWithStats(img_thre, connectivity=8)
+    mask = np.zeros(varMask.shape, dtype=np.uint8)
+
+    for j in range(1, num_labels):
+        if stats[j, cv2.CC_STAT_AREA] >= areaTH:
+            mask[labels == j] = 255
+    mat1[mask != 255] = 0
+    return mat1
+
+def QC_batch(mat, dbzTH=10, areaTH=20, phase="QC"):  # 批量QC函数
+    from shancx.NN import _loggers
+    logger = _loggers(phase=phase)
+    for i in range(len(mat)):
+        logger.info(f"QC {i}")
+        mat[i] = QC(mat[i], dbzTH, areaTH)
+    return mat
+
+"""
+pre = QC_simple(pre,areaTH=30)   二维灰度
+pre = QC_ref(pre,areaTH=30)      三维灰度
+"""

shancx/Mos/__init__.py

@@ -0,0 +1,37 @@
+ 
+
+
+import numpy as np
+def start_points(size, split_size, overlap=0.0): 
+    stride = int(split_size * (1 - overlap))  # 计算步长
+    points = [i * stride for i in range((size - split_size) // stride + 1)]   
+    if size > points[-1] + split_size:   
+        points.append(size - split_size)
+    return points
+
+def weigthOverlap(overlap,radarpre,x,y,b,weight_sum):
+     weight = np.ones((256, 256))
+     for i in range(overlap):
+         weight[:, i] = 0.5 * (1 - np.cos(np.pi * i / overlap))
+         weight[:, -i-1] = 0.5 * (1 - np.cos(np.pi * i / overlap))
+         weight[i, :] = np.minimum(weight[i, :], 0.5 * (1 - np.cos(np.pi * i / overlap)))
+         weight[-i-1, :] = np.minimum(weight[-i-1, :], 0.5 * (1 - np.cos(np.pi * i / overlap)))
+     patch = radarpre * weight
+     b[x:x+256, y:y+256] = np.where(np.isnan(b[x:x+256, y:y+256]), 
+                            patch, 
+                            b[x:x+256, y:y+256] + patch)
+     weight_sum[x:x+256, y:y+256] = np.where(np.isnan(weight_sum[x:x+256, y:y+256]), 
+                                                 weight, 
+                                                 weight_sum[x:x+256, y:y+256] + weight)    
+     return b,weight_sum
+
+"""
+weight_sum = np.zeros_like(b)
+b,weight_sum = weigthOverlap(overlap,radarpre,x,y,b,weight_sum)
+b = np.divide(b, weight_sum, where=weight_sum!=0)
+
+weight_sum = np.zeros_like(b)
+overlap = 32  
+x_point = start_points(sat_data[0].shape[0], 256, overlap/256)
+y_point = start_points(sat_data[0].shape[1], 256, overlap/256)
+"""

shancx/NN/__init__.py

@@ -4,124 +4,253 @@ import os
 def start():
     print("import successful")
 # constants
-__author__ = 'shancx'
- 
-__author_email__ = 'shancx@126.com'
-
-# @Time : 2023/09/27 下午8:52
-# @Author : shanchangxi
-# @File : util_log.py
-import time
-import logging  
-from logging import handlers
-
-def mkDir(path):
-    if "." in path:
-        os.makedirs(os.path.dirname(path),exist_ok=True)
-    else:
-        os.makedirs(path, exist_ok=True)
-
-loggers = logging.getLogger()
-loggers.setLevel(logging.INFO) 
-log_name =  './project_N.log'
-mkDir(log_name)
-logfile = log_name
-time_rotating_file_handler = handlers.TimedRotatingFileHandler(filename=logfile, when='D', encoding='utf-8')
-time_rotating_file_handler.setLevel(logging.INFO)   
-formatter = logging.Formatter("%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s")
-time_rotating_file_handler.setFormatter(formatter)
-loggers.addHandler(time_rotating_file_handler)
-
-
-"""      
-import tracback
-try:
+__author__ = 'shancx' 
+__author_email__ = 'shanhe12@163.com'
+__time__ = '20251028 21:16'
+import logging
+from logging.handlers import RotatingFileHandler
+import os
+from shancx import crDir 
+def _loggers(logger_name="loggers", root="./logs", phase="project", level=logging.INFO, screen=True, max_bytes=10*1024*1024, backup_count=5, overwrite=False,handlersflag=False):
+    '''set up logger with rotating file handler'''
+    l = logging.getLogger(logger_name)
+    if handlersflag:
+        if l.handlers:
+            return l
+    formatter = logging.Formatter(
+        "%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s", datefmt='%Y-%m-%d %H:%M:%S') 
+    log_file = os.path.join(root, '{}.log'.format(phase))
+    crDir(log_file)
+    # Use RotatingFileHandler with 'w' mode to overwrite log file if needed
+    mode = 'w' if overwrite else 'a'
+    fh = RotatingFileHandler(log_file, mode=mode, maxBytes=max_bytes, backupCount=backup_count)
+    fh.setFormatter(formatter)    
+    l.setLevel(level)
+    l.addHandler(fh)    
+    if screen:
+        sh = logging.StreamHandler()
+        sh.setFormatter(formatter)
+        l.addHandler(sh)    
+    return l
 
-except Exception as e:
-   
-     logger.info(traceback.format_exc())
+"""
+logger = _loggers(logger_name="test_logger", root=curpathplus, phase="test_log", overwrite=True, screen=True)
+# 测试日志输出
+for i in range(5):
+    logger.info(f"这是日志消息 {i+1}")
+    time.sleep(1)
 """
 
 
-"""
-from main import makeAll,options
-from multiprocessing import Pool
-import datetime
-from config import logger,output
-import time
-import pandas as pd
+import logging
+from logging.handlers import RotatingFileHandler
 import os
-from itertools import product
-import threading
-
-def excuteCommand(cmd):
-    print(cmd)
-    os.system(cmd)
+from shancx import crDir 
+import multiprocessing
+def _loggersPlus(logger_name="loggers", root="./logs", phase="project", level=logging.INFO, 
+             screen=True, max_bytes=10 * 1024 * 1024, backup_count=5, overwrite=False, 
+             handlersflag=False, enable_rotating=None):
+    l = logging.getLogger(logger_name)
+    if handlersflag:
+        if l.handlers:
+            return l    
+    formatter = logging.Formatter(
+        "%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s", 
+        datefmt='%Y-%m-%d %H:%M:%S'
+    )    
+    log_file = os.path.join(root, '{}.log'.format(phase))
+    crDir(log_file)
+    mode = 'w' if overwrite else 'a'    
+    if enable_rotating is None:
+        enable_rotating = (multiprocessing.current_process().name == 'MainProcess')
+    
+    if enable_rotating:
+        fh = RotatingFileHandler(
+            log_file, 
+            mode=mode, 
+            maxBytes=max_bytes, 
+            backupCount=backup_count
+        )
+    else:
+        fh = logging.FileHandler(log_file, mode=mode)
+    
+    fh.setFormatter(formatter)    
+    l.setLevel(level)
+    l.addHandler(fh)    
+    
+    if screen:
+        sh = logging.StreamHandler()
+        sh.setFormatter(formatter)
+        l.addHandler(sh)  
+    return l
 
-def gpuPro(makeListUTC, isPhase, isDebug, gpu, isOverwrite):
-    productList = product(makeListUTC, [isPhase], [isDebug], [gpu], [isOverwrite])
 
-    with Pool(4) as p:
-        p.map(makeAll, productList)
+import logging
+def setlogger(level=logging.INFO):
+ 
+    logging.basicConfig(
+        level=level,  # 动态接受级别参数
+        format="%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s",
+        handlers=[logging.StreamHandler()],
+        force=True  # 强制覆盖现有配置（Python 3.8+）
+    )
+    return logging.getLogger()
+'''
+# 使用示例
+if __name__ == "__main__":
+    logger = setlogger(level=logging.DEBUG)  # 设置为DEBUG级别
+    logger.debug("这条日志会显示")  # 默认情况下DEBUG不显示，但因为我们设置了级别，现在会显示
+    logger.info("这是一条INFO日志")
+'''
+import os
+import glob
+import logging
+from datetime import datetime
+from shancx import loggers as logger
+def cleanupLogs(log_dir='/mnt/wtx_weather_forecast/scx/SATH9SEAStest/logs', keep_count=10, 
+                 pattern='*.log', recursive=False, dry_run=False):
+    stats = {'total_dirs': 0, 'deleted_files': 0, 'errors': []}    
+    def _cleanup_dir(directory):
+        stats['total_dirs'] += 1
+        if not os.path.exists(directory):
+            logging.warning(f"目录不存在: {directory}")
+            return
+        file_paths = glob.glob(os.path.join(directory, pattern))
+        log_files = [(path, os.path.getmtime(path)) for path in file_paths]
+        log_files.sort(key=lambda x: x[1], reverse=True)        
+        if len(log_files) <= keep_count:
+            logging.info(f"目录 {directory} 中的文件数量 ({len(log_files)}) 不超过保留数量 ({keep_count})，无需清理")
+            return
+        files_to_delete = log_files[keep_count:]
+        for file_path, mtime in files_to_delete:
+            try:
+                if dry_run:
+                    logging.info(f"[试运行] 将删除: {file_path} (修改时间: {datetime.fromtimestamp(mtime).strftime('%Y-%m-%d %H:%M:%S')})")
+                else:
+                    os.remove(file_path)
+                    logging.info(f"已删除: {file_path} (修改时间: {datetime.fromtimestamp(mtime).strftime('%Y-%m-%d %H:%M:%S')})")
+                stats['deleted_files'] += 1
+            except Exception as e:
+                error_msg = f"删除失败 {file_path}: {str(e)}"
+                logging.error(error_msg)
+                stats['errors'].append(error_msg)
+    if recursive:
+        for root, _, _ in os.walk(log_dir):
+            _cleanup_dir(root)
+    else:
+        _cleanup_dir(log_dir)    
+    return stats
+"""
+if __name__ == "__main__": 
+    dir = "/mnt/wtx_weather_forecast/scx/SATH9SEAStest/logs/H9SEAS/"
+    stats = cleanupLogs(dir,3, '*.log', False, False)    
+    logging.info(f"清理完成: 处理目录数={stats['total_dirs']}, 删除文件数={stats['deleted_files']}, 错误数={len(stats['errors'])}")
+    if stats['errors']:
+        logging.error(f"错误详情: {stats['errors']}")
+"""  
 
-if __name__ == '__main__':
-    cfg = options()
-    isPhase = cfg.isPhase
-    isDebug = cfg.isDebug
-    sepSec = cfg.sepSec
-    gpu = cfg.gpu
-    pool = cfg.pool
-    isOverwrite = cfg.isOverwrite
-    timeList = pd.date_range(cfg.times[0], cfg.times[-1], freq=f"{sepSec}s")
-    logger.info(f"时间段check {timeList}")
-    gpuNum = 2
-    eachGPU = 4
+from itertools import product
+from concurrent.futures import ProcessPoolExecutor as PoolExecutor, as_completed 
+import sys
+from tqdm import tqdm 
+def validate_param_list(param_list):
+    if len(param_list) == 0:
+        raise ValueError("param_list cannot be empty.")    
+    for sublist in param_list:
+        if len(sublist) == 0:
+            raise ValueError("Sub-lists in param_list cannot be empty.")     
+def Mul_sub(task, param_list, num=6):
+    print(f"Pro num {num}")
+    validate_param_list(param_list)
+    if len(param_list) == 1:
+        product_list = [(x,) for x in param_list[0]]
+    else:
+        product_list = list(product(*param_list))
+    with PoolExecutor(max_workers=num) as executor:
+        try:           
+            futures = [executor.submit(task, item) for item in product_list]
+            for future in tqdm(as_completed(futures), total=len(futures), desc="Processing tasks", unit="task"):  
+                future.result() 
+        except KeyboardInterrupt:
+            sys.exit(1)    
+    print("All tasks completed")
 
-    makeListUTC = []
-    for UTC in timeList:
-        UTCStr = UTC.strftime("%Y%m%d%H%M")
-        outpath = f"{output}/{UTCStr[:4]}/{UTCStr[:8]}/MSP2_WTX_AIW_QPF_L88_CHN_{UTCStr}_00000-00300-00006.nc"
-        if not os.path.exists(outpath) or isOverwrite:
-            makeListUTC.append(UTC)
-    [print(element) for element in makeListUTC]
+from concurrent.futures import ThreadPoolExecutor
+from itertools import product
+def Mul_TH(task, param_list, max_workers=6):
+    print(f"Thread num: {max_workers}")
+    validate_param_list(param_list)
+    task_args = [
+        (arg,) if len(param_list) == 1 else arg
+        for arg in (
+            param_list[0] if len(param_list) == 1 
+            else product(*param_list)
+        )
+    ]    
+    with ThreadPoolExecutor(max_workers) as ex:
+        try:
+            list(ex.map(task, task_args))
+        except KeyboardInterrupt:
+            print("\n用户中断操作")
+            ex.shutdown(wait=False)
+            sys.exit(1) 
 
-    phaseCMD = "--isPhase" if isPhase else ""
-    debugCMD = "--isDebug" if isDebug else ""
-    OverwriteCMD = "--isOverwrite"  
-    gpuCMD = f"--gpu={gpu}"
-    # cmdList = list(map(lambda x:f"python main.py --times={x.strftime('%Y%m%d%H%M')} {phaseCMD} {debugCMD} {OverwriteCMD} {gpuCMD}",makeListUTC))
-    cmdList = list(map(lambda x:f"python main.py --times={x.strftime('%Y%m%d%H%M')} {phaseCMD} {debugCMD} {gpuCMD}",makeListUTC))
+import traceback
+import shutil, os
+def safe_del(path):
+    try:
+        shutil.rmtree(path) if os.path.isdir(path) else None
+        print(f"{path} deleted")
+    except Exception:
+        print(traceback.format_exc())
 
-    with Pool(pool) as p:
-        p.map(excuteCommand, cmdList)
+"""
+safe_del("./data")
 """
 
+import os, glob
+def clean_files(folder, keep=0):
+    if os.path.isdir(folder):
+        try:
+            files = [os.path.join(folder, f) for f in os.listdir(folder) 
+                    if os.path.isfile(os.path.join(folder, f))]
+            if keep > 0 and len(files) > keep:
+                files.sort(key=os.path.getmtime)
+                [os.remove(f) for f in files[:-keep]]
+            elif keep == 0:
+                [os.remove(f) for f in files]
+        except Exception as e:
+            print(traceback.format_exc())
 
-'''
-# @Time : 2023/09/27 下午8:52
-# @Author : shanchangxi
-# @File : util_log.py
-import time
-import logging  
-from logging import handlers
- 
-logger = logging.getLogger()
-logger.setLevel(logging.INFO) 
-log_name =  'project_tim_tor.log'
-logfile = log_name
-time_rotating_file_handler = handlers.TimedRotatingFileHandler(filename=logfile, when='D', encoding='utf-8')
-time_rotating_file_handler.setLevel(logging.INFO)   
-formatter = logging.Formatter("%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s")
-time_rotating_file_handler.setFormatter(formatter)
-logger.addHandler(time_rotating_file_handler)
-print_handler = logging.StreamHandler()   
-print_handler.setFormatter(formatter)   
-logger.addHandler(print_handler)
-
-'''
-
-'''
-###解决方法  pip install torch==2.4.0  torchvision    torchaudio三个同时安装  python 3.12  解决cuda启动不了的问题
+"""
+clean_files("./logs", keep=10)   
+clean_files("./temp") 
+"""        
 
-Res网络
-'''
+# import os
+# from datetime import datetime
+# from shancx.NN import _loggers
+# from shancx import lock_file
+# from shancx.wait import check_lock
+# from shancx import crDir
+# logger =_loggers()
+# def check_process_data(UTC, sat_cd,basepath ="/mnt/wtx_weather_forecast/scx/test/lock_files" ): 
+#     try:
+#         UTCStr = UTC.strftime("%Y%m%d%H%M")
+#         file = f"/mnt/wtx_weather_forecast/scx/test/lock_files/{sat_cd}/{UTCStr[:4]}/{UTCStr[:8]}/File_{UTCStr}.lock"
+#         crDir(file)
+#         if not lock_file(file):
+#             if check_lock(file):
+#                 logger.info("data is making or maked")
+#                 return True ,file       
+#         return False,file        
+#     except Exception as e:
+#         logger.error(f"Error in check_and_process_data: {str(e)}")
+#         return False,file  
+# """
+# flag1,file = check_process_data(UTC, "H9SEAS" )
+# if flag1:
+#     sys.exit() 
+# if os.path.exists(output_path): #配合使用
+#    sys.exit()    
+# """