shancx 1.8.92__py3-none-any.whl → 1.9.33.218__py3-none-any.whl

shancx/Time/timeCycle.py

@@ -0,0 +1,302 @@
+
+import glob
+import os
+import numpy as np
+import traceback
+from dateutil.relativedelta import relativedelta
+# 获取指定路径下所有文件
+paths = glob.glob("/root/data/ec_filter_npy_data/*")  
+
+# 遍历每个文件路径
+for p in paths:
+    try:
+        # 尝试加载.npy文件
+        data = np.load(p)
+    except Exception as e:
+        # 如果读取失败，打印错误信息并删除文件
+        print(f"Error loading {p}: {traceback.format_exc()}")
+        os.remove(p)
+        print(f"Deleted file: {p}")
+def GetMulData(conf):
+    sCST = conf[0]
+    eCST = conf[0]
+    sUTC = sCST+relativedelta(hours=-8)
+    sCSTstr = sCST.strftime("%Y%m%d%H%M%S") 
+    sUTCstr = sUTC.strftime("%Y%m%d%H%M%S") 
+    path = f"/root/data/{sUTCstr[:4]}/{sUTCstr:8}/CR_{sUTCstr[:12]}00.npy"
+    if os.path.exists(path):
+        print(f"outpath {path} existsing ")
+
+    else:
+        print(f"outpath {path} not existsing ")
+        return
+
+from shancx import Mul_sub
+import argparse
+import datetime
+import pandas as pd
+def options():
+    parser = argparse.ArgumentParser(description='examdatabasedata')
+    parser.add_argument('--times', type=str, default='202411100000,202411101000') 
+    config= parser.parse_args()
+    print(config)
+    config.times = config.times.split(",")
+    if len(config.times) == 1:
+        config.times = [config.times[0], config.times[0]]
+    config.times = [datetime.datetime.strptime(config.times[0], "%Y%m%d%H%M"),
+                    datetime.datetime.strptime(config.times[1], "%Y%m%d%H%M")]
+    return config
+if __name__ == '__main__':
+    cfg = options()
+    sCST = cfg.times[0]
+    eCST = cfg.times[-1]
+    timeList = pd.date_range(sCST, eCST, freq='6T')  #6T 分钟 
+    print(timeList)
+    Mul_sub(GetMulData,[timeList],31)
+
+"""
+for i in range(7):
+    UTC = sUTC +relativedelta(hours=-8,minutes=-diffT*6)
+"""
+
+"""
+import glob
+import os
+import numpy as np
+from shancx import crDir
+from shancx.NN import _loggers
+logger = _loggers()
+import netCDF4 as nc
+import numpy as np
+from shancx.Plot import plotRadar
+from shancx import crDir  
+import traceback
+from dateutil.relativedelta import relativedelta
+paths = glob.glob("/root/data/ec_filter_npy_data/*")  
+basePath = f"/mnt/wtx_weather_forecast/scx/sever7/test/RADA/MQPF1109_1"
+output_dirH9Npy = f"/mnt/wtx_weather_forecast/SAT/H9/sat_npy_CHN"
+def GetMulData(conf):  
+    sUTC = conf[0]
+    sUTCstr = sUTC.strftime("%Y%m%d%H%M") 
+    output_path =f"{output_dirH9Npy}/{sUTCstr[:4]}/{sUTCstr[:8]}/MSP3_PMSC_H9_GEO_FD_{sUTCstr[:12]}_00000-00000.npy"
+    inputPathstr = f"{basePath}/{sUTCstr[:4]}/{sUTCstr[:8]}/*{sUTCstr[:12]}*.nc"
+    inputPathL =  glob.glob(inputPathstr)
+    if len(inputPathL) ==0 :  
+        print(f"outpath {inputPathstr} is missing ")
+        return None 
+    inputPath = inputPathL[0]
+    with nc.Dataset(inputPath) as dataNC:  
+        CR = dataNC["CR"][:]
+        lat = dataNC["lat"][:]
+        lon = dataNC["lon"][:]        
+    crDir(output_path)
+    np.save(output_path,CR.data)
+    logger.info(f"{output_path} done ") 
+
+from shancx import Mul_sub    
+import argparse
+import datetime
+import pandas as pd
+def options():
+    parser = argparse.ArgumentParser(description='examdatabasedata')
+    parser.add_argument('--times', type=str, default='202507010000,202510010000') 
+    config= parser.parse_args()
+    print(config)
+    config.times = config.times.split(",")
+    if len(config.times) == 1:
+        config.times = [config.times[0], config.times[0]]
+    config.times = [datetime.datetime.strptime(config.times[0], "%Y%m%d%H%M"),
+                    datetime.datetime.strptime(config.times[1], "%Y%m%d%H%M")]
+    return config
+if __name__ == '__main__':
+    cfg = options()
+    sUTC = cfg.times[0]
+    eUTC = cfg.times[-1]
+    timeList = pd.date_range(sUTC, eUTC, freq='3h')  #6T 分钟 
+    print(timeList)
+    Mul_sub(GetMulData,[timeList],6)
+"""
+
+"""
+import glob
+import os
+import numpy as np
+import traceback
+from dateutil.relativedelta import relativedelta
+import os 
+from shancx.NN import Mul_TH
+import argparse
+import datetime
+import pandas as pd
+def GetMulData(conf):
+    sUTC = conf[0]
+    sUTCstr = sUTC.strftime("%Y%m%d%H%M") 
+    commandstr = f""
+    os.system(commandstr) 
+def options():
+    parser = argparse.ArgumentParser(description='examdatabasedata')
+    parser.add_argument('--times', type=str, default='202508010000,202508010500') 
+    config= parser.parse_args()
+    print(config)
+    config.times = config.times.split(",")
+    if len(config.times) == 1:
+        config.times = [config.times[0], config.times[0]]
+    config.times = [datetime.datetime.strptime(config.times[0], "%Y%m%d%H%M"),
+                    datetime.datetime.strptime(config.times[1], "%Y%m%d%H%M")]
+    return config
+if __name__ == '__main__':
+    cfg = options()
+    sUTC = cfg.times[0]
+    eUTC = cfg.times[-1]
+    timeList = pd.date_range(sUTC, eUTC, freq='10T')  #6T 分钟 
+    print(timeList)
+    Mul_TH(GetMulData,[timeList],3)
+cd /mnt/wtx_weather_forecast/scx/sever7/SATdata/mkH9 ;timeout 1200s /home/scx/miniconda3/envs/H9/bin/python mkH9.py --time {sUTCstr[:12]}
+
+"""
+
+"""
+from shancx import crDir
+import os
+from shancx import loggers as logger
+# Define the original and new filenames
+original_file = "CR_20241117050600.npy"
+new_file = "20241117050600.npy"
+rootpath ="/root/autodl-tmp"
+filepath = "data/radar" 
+def GetMulData(conf):
+    sCST = conf[0]
+    # eCST = conf[0]
+    sCSTstr = sCST.strftime("%Y%m%d%H%M%S")   
+    outpath = os.path.join(rootpath,filepath,f"CR_{sCSTstr}00.npy")
+    if os.path.exists(outpath):
+        logger.info(f"outpath {outpath} is existsing ")
+        print(f"outpath {outpath} existsing ")    
+    crDir(outpath)
+    array = np.load(f"./{original_file}")
+    np.save(outpath,array)
+    logger.info(f"outpath {outpath} done ")
+    print(f"outpath {outpath} done ")
+from shancx import Mul_sub
+import argparse
+import datetime
+import pandas as pd
+import numpy as np
+def options():
+    parser = argparse.ArgumentParser(description='examdatabasedata')
+    parser.add_argument('--times', type=str, default='202411101000,202411150000') 
+    config= parser.parse_args()
+    print(config)
+    config.times = config.times.split(",")
+    if len(config.times) == 1:
+        config.times = [config.times[0], config.times[0]]
+    config.times = [datetime.datetime.strptime(config.times[0], "%Y%m%d%H%M"),
+                    datetime.datetime.strptime(config.times[1], "%Y%m%d%H%M")]
+    return config
+if __name__ == '__main__':
+    cfg = options()
+    sCST = cfg.times[0]
+    eCST = cfg.times[-1]
+    timeList = pd.date_range(sCST, eCST, freq='6T')  #6T 分钟 
+    print(timeList)
+    Mul_sub(GetMulData,[timeList],48)
+
+------------------------------------
+
+import glob
+import os
+import numpy as np
+import traceback
+from shancx import Mul_sub
+from shancx import loggers as logger
+
+# 获取指定路径下所有文件
+paths = glob.glob("/root//autodl-tmp/data/radar/*")  #E:\
+
+# 遍历每个文件路径
+def getMul_sub(conf):
+    p = conf[0]
+    print(p)
+    try:
+        # 尝试加载.npy文件
+        data = np.load(p)
+        print(f"Loaded {p} with shape {data.shape}")
+    except Exception as e:
+        # 如果读取失败，打印错误信息并删除文件
+        print(f"Error loading {p}: {traceback.format_exc()}")
+        logger.error(f"Error loading {p}: {traceback.format_exc()}")
+        os.remove(p)
+        print(f"Deleted file: {p}")
+ 
+if __name__ == '__main__':
+    paths1 = [i for i in paths if '.npy' in i]
+    Mul_sub(getMul_sub,[paths1],20)
+
+np.tile(np.load(basedata), (8, 1, 1)).reshape((8, 4200, 6200))
+"""
+
+
+"""
+制作特定数据
+
+import glob
+import os
+import numpy as np
+import traceback
+from dateutil.relativedelta import relativedelta
+# 获取指定路径下所有文件
+import glob 
+import datetime
+from shancx import crDir
+import netCDF4 as nc
+# 遍历每个文件路径
+def GetMulData(conf):
+    sUTC = conf[0]
+    UTCStr = sUTC.strftime("%Y%m%d%H%M%S") 
+    path = f"/data2/mym/ifs_precipitation/{UTCStr[:4]}/{UTCStr[:8]}/ifs_precipitation_{UTCStr[:12]}.nc" 
+    if not  os.path.exists(path):
+        print(f"outpath {path} existsing ")
+    else:
+        with nc.Dataset(path) as dataNC:
+            # 获取 'time' 变量
+            latitude = dataNC.variables[list(dataNC.variables)[0]][:] 
+            longitude = dataNC.variables[list(dataNC.variables)[2]][:]  
+            ifs_precipitation = dataNC.variables[list(dataNC.variables)[1]][:]  
+            # print(list(dataNC.variables))
+    # crDir(path)
+    # np.save(path,d)
+    return {"min":np.min(d),"max":np.max(d)}
+from shancx import Mul_sub
+import argparse
+import datetime
+import pandas as pd
+def options():
+    parser = argparse.ArgumentParser(description='examdatabasedata')
+    parser.add_argument('--times', type=str, default='202411100000,202411101000') 
+    config= parser.parse_args()
+    print(config)
+    config.times = config.times.split(",")
+    if len(config.times) == 1:
+        config.times = [config.times[0], config.times[0]]
+    config.times = [datetime.datetime.strptime(config.times[0], "%Y%m%d%H%M"),
+                    datetime.datetime.strptime(config.times[1], "%Y%m%d%H%M")]
+    return config
+if __name__ == '__main__':
+ 
+    cfg = options()
+    sUTC = cfg.times[0]
+    eUTC = cfg.times[-1]
+    sUTC = datetime.datetime(2002,6,1,0,0) 
+    eUTC = datetime.datetime(2004,9,1,0,0) 
+    timeList = pd.date_range(sUTC, eUTC, freq='3h')  #6T 分钟 
+
+    summer_timeList = timeList[timeList.month.isin([6, 7, 8])]
+    print(timeList)
+    minmax = Mul_sub(GetMulData,[summer_timeList],10)
+    global_min = min(d['min'] for d in minmax)
+    global_max = max(d['max'] for d in minmax)  #117
+    print() 
+
+"""
+
+ 

shancx/Train/__init__.py

@@ -0,0 +1,74 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+import torch
+import torchvision.transforms as transforms
+import cv2
+import numpy as np
+from typing import List, Tuple
+from PIL import Image
+import os
+class ImageProcessor:
+    """Handles image preprocessing and transformations"""
+    
+    def __init__(self):
+        self.transform = transforms.Compose([
+            transforms.Resize(256),
+            transforms.CenterCrop(224),
+            transforms.ToTensor(),
+            transforms.Normalize(
+                mean=[0.485, 0.456, 0.406],
+                std=[0.229, 0.224, 0.225]
+            )
+        ])
+        
+    def preprocess_image(self, image_path: str) -> torch.Tensor:
+        """Load and preprocess image for model input"""
+        try:
+            if not os.path.exists(image_path):
+                raise FileNotFoundError(f"Image file not found at {image_path}")
+                
+            image = cv2.imread(image_path)
+            if image is None:
+                raise ValueError(f"Unable to read image at {image_path}")
+                
+            # Convert BGR to RGB
+            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+            # Convert to PIL Image for torchvision transforms
+            image = Image.fromarray(image)
+            return self.transform(image).unsqueeze(0)
+        except Exception as e:
+            print(f"Error processing image: {e}")
+            raise
+
+# if __name__ == "__main__":
+#     image_path = "./space_shuttle.jpg"
+#     image_processor = ImageProcessor()
+#     input_tensor = image_processor.preprocess_image(image_path)
+
+import torch
+import torch.nn as nn
+
+def multiGpu(model, gpu_ids):
+    # 检查是否有可用的 GPU
+    if not torch.cuda.is_available():
+        print("CUDA is not available. Using CPU.")
+        device = torch.device("cpu")
+        return model.to(device), device
+    device = torch.device(f"cuda:{gpu_ids[0]}")
+    if len(gpu_ids) > 1:
+        print(f"Using {len(gpu_ids)} GPUs: {gpu_ids}")
+        model = nn.DataParallel(model, device_ids=gpu_ids)
+    else:
+        print(f"Using GPU: {gpu_ids[0]}")
+    model = model.to(device)
+    return model, device
+
+    """
+    model = MyModel()
+    gpu_ids = [5, 6, 7]
+    model, device = setup_multi_gpu(model, gpu_ids)
+    print(f"Model is on device: {device}")
+    data = torch.randn(10, 3, 224, 224).to(device)
+    output = model(data)
+    """
+

shancx/Train/makelist.py

@@ -0,0 +1,187 @@
+
+def save_results(results, output_file, mode='w',title="input_path,tar_path"):  
+    with open(output_file, mode) as f:
+        if mode == 'w':  
+            f.write(f"{title}\n")
+        for result in results:
+            f.write(f"{result['input_path']},{result['tar_path']}\n")
+"""
+output_file = "rmse_results.txt"
+save_results([], output_file, mode='w')
+result = {
+          'input_path': input_path ,
+          'tar_path': tar_path 
+          }
+save_results([result], output_file, mode='a')  
+
+df = pd.read_csv(csv_file, sep=" ",header=None)   
+sample_list = df.values.tolist()
+self.paths = {
+    "input_path": [l[0] for l in self.sample_list],
+    "gt_path": [l[1] for l in self.sample_list]
+}
+"""    
+import os
+import numpy as np
+import pandas as pd
+import glob
+import datetime
+from shancx import Mul_sub_S,Mul_sub 
+from shancx import crDir
+def getcheckdata(conf):
+    satPath = conf[0]
+    radar_dir_path = conf[1]
+    sat_imin = conf[2]
+    try:
+        satdata = np.load(satPath)
+        radarpth = glob.glob(f"{radar_dir_path}/{sat_imin[:4]}/{sat_imin[:8]}/CR_{satPath.split('/')[-1][4:-4]}*.npy")[0]
+        radardata = np.load(radarpth)
+        if radardata.shape != satdata.shape :
+            return  
+        df = pd.DataFrame({'sat_path': [satPath], 'radar_path': [radarpth] })
+        return df
+    except Exception as e:
+        print(f"{satPath} can not load succeed: {e}")
+        return None
+def generateList(conf):
+    sat_dir_path, radar_dir_path, sat_imin= conf
+    satpath = glob.glob(f"{sat_dir_path}/{sat_imin[:4]}/{sat_imin[:8]}/SAT_{sat_imin}_*.npy")
+    satpath.sort()
+    if satpath:
+        datas = []
+        for path in satpath:
+            data = getcheckdata( (path,radar_dir_path,sat_imin))
+            datas.append(data)
+        datass = [i for i in datas if i is not None ]
+        if datass :
+            df = pd.concat(datass)
+            return df
+    else:
+            return None 
+import argparse
+import datetime
+import pandas as pd
+def options():
+    parser = argparse.ArgumentParser(description='examdatabasedata')
+    parser.add_argument('--times', type=str, default='202502010000,202506220000') 
+    parser.add_argument('--flag', type=str, default='val') 
+    config= parser.parse_args()
+    print(config)
+    config.times = config.times.split(",")
+    if len(config.times) == 1:
+        config.times = [config.times[0], config.times[0]]
+    config.times = [datetime.datetime.strptime(config.times[0], "%Y%m%d%H%M"),
+                    datetime.datetime.strptime(config.times[1], "%Y%m%d%H%M")]
+    return config
+"""
+if __name__ == '__main__':
+    cfg = options()
+    sUTC = cfg.times[0]
+    eUTC = cfg.times[-1]
+    flag = cfg.flag     
+    sat_dir_path ="./"
+    radar_dir_path = f"./"
+    timelist = pd.date_range(sUTC, eUTC, t=f"{flag}")
+    timeListfliter = timelist[timelist.month.isin([1,4,7,10])&(timelist.day<=15)]
+    savepath = f'/mnt/wtx_weather_forecast/SAT/GOES18train_N/0624_1' 
+    crDir(savepath)
+    # 调用方法    1.split_time   2. timelist  3. 路径
+    dataL = Mul_sub(generateList,[ [sat_dir_path]        
+                                , [radar_dir_path]
+                                , timelist
+                                 ] 
+                                 )            
+    dataLs = [i for i in dataL if i is not None]    
+    if  flag =="trn":
+        train_df = pd.concat(dataLs) 
+        crDir(savepath)
+        train_df.to_csv(f"{savepath}/df_train.csv", index=False, sep=',')
+        print(f"train_df {len(train_df)}")
+        print('complete!!!') 
+        print(savepath)
+    if  flag == "val":
+        valid_df = pd.concat(dataLs)  
+        crDir(savepath)
+        valid_df.to_csv(f"{savepath}/df_valid.csv", index=False, sep=',')
+        print(f"valid_df {len(valid_df)}")
+        print('complete!!!')
+        print(savepath)   
+        
+
+"""
+"""
+import os
+import numpy as np
+import pandas as pd
+import glob
+import datetime
+from shancx import Mul_sub_S,Mul_sub 
+from shancx import crDir
+def getcheckdata(conf):
+    inputPath = conf[0]
+    target_dir_path = conf[1]
+    imin = conf[2]
+    try:
+        inputdata = np.load(inputPath)
+        targetpth = glob.glob(f"{target_dir_path}/{imin[:4]}/{imin[:8]}/analy_{inputPath.split('/')[-1][3:-4]}*.npy")[0]
+        targetdata = np.load(targetpth)
+        if inputdata.shape != targetdata.shape :
+            return  
+        df = pd.DataFrame({'inputPath': [inputPath], 'targetpth': [targetpth] })
+        return df
+    except Exception as e:
+        print(f"{inputPath} can not load succeed: {e}")
+        return None
+def generateList(conf):
+    imin,input_dir_path, target_dir_path= conf
+    imin = imin.strftime("%Y%m%d%H%M")
+    targetpath = glob.glob(f"{input_dir_path}/{imin[:4]}/{imin[:8]}/EC_{imin}_*.npy")
+    targetpath.sort()
+    if targetpath:
+        datas = []
+        for path in targetpath:
+            data = getcheckdata( (path,target_dir_path,imin))
+            datas.append(data)
+        datass = [i for i in datas if i is not None ]
+        if datass :
+            df = pd.concat(datass)
+            return df
+    else:
+            return None 
+import argparse
+import datetime
+import pandas as pd
+def options():
+    parser = argparse.ArgumentParser(description='examdatabasedata')
+    parser.add_argument('--times', type=str, default='202505210030,202505210130') 
+    parser.add_argument('--flag', type=str, default='val') 
+    parser.add_argument('--freq', type=str, default="1h") 
+    config= parser.parse_args()
+    print(config)
+    config.times = config.times.split(",")
+    if len(config.times) == 1:
+        config.times = [config.times[0], config.times[0]]
+    config.times = [datetime.datetime.strptime(config.times[0], "%Y%m%d%H%M"),
+                    datetime.datetime.strptime(config.times[1], "%Y%m%d%H%M")]
+    return config
+if __name__ == '__main__':
+    cfg = options()
+    sUTC = cfg.times[0]
+    eUTC = cfg.times[-1]
+    freq = cfg.freq     
+    input_dir_path ="/mnt/wtx_weather_forecast/scx/smart_grid/train/clip_EC"
+    target_dir_path = f"/mnt/wtx_weather_forecast/scx/smart_grid/train/analy_1_256"
+    timelist = pd.date_range(sUTC, eUTC, freq=f"{freq}")
+    timeListfliter = timelist[timelist.month.isin([1,4,5,7,10])]
+    timeListfliter = list(timeListfliter)
+    savepath = f'/mnt/wtx_weather_forecast/SAT/downscale/0901' 
+    crDir(savepath)
+    # 调用方法    1.split_time   2. timelist  3. 路径
+ 
+    dataL = Mul_sub(generateList,[ timeListfliter,[input_dir_path] , [target_dir_path]] )          
+    dataLs = [i for i in dataL if i is not None]  
+    pd.concat(dataLs).to_csv('train.txt', sep=' ', index=False)
+    a = pd.read_csv("./train.txt",sep=" ").values.tolist()
+"""
+        
+ 

shancx/Train/multiGpu.py

@@ -0,0 +1,27 @@
+
+import torch
+import torch.nn as nn
+
+def multiGpu(model, gpu_ids):
+    # 检查是否有可用的 GPU
+    if not torch.cuda.is_available():
+        print("CUDA is not available. Using CPU.")
+        device = torch.device("cpu")
+        return model.to(device), device
+    device = torch.device(f"cuda:{gpu_ids[0]}")
+    if len(gpu_ids) > 1:
+        print(f"Using {len(gpu_ids)} GPUs: {gpu_ids}")
+        model = nn.DataParallel(model, device_ids=gpu_ids)
+    else:
+        print(f"Using GPU: {gpu_ids[0]}")
+    model = model.to(device)
+    return model, device
+
+    """
+    model = MyModel()
+    gpu_ids = [5, 6, 7]
+    model, device = setup_multi_gpu(model, gpu_ids)
+    print(f"Model is on device: {device}")
+    data = torch.randn(10, 3, 224, 224).to(device)
+    output = model(data)
+    """