shancx 1.9.33.109__py3-none-any.whl → 1.9.33.218__py3-none-any.whl

shancx/Train/makelist.py

@@ -1,181 +1,187 @@
 
-
+def save_results(results, output_file, mode='w',title="input_path,tar_path"):  
+    with open(output_file, mode) as f:
+        if mode == 'w':  
+            f.write(f"{title}\n")
+        for result in results:
+            f.write(f"{result['input_path']},{result['tar_path']}\n")
+"""
+output_file = "rmse_results.txt"
+save_results([], output_file, mode='w')
+result = {
+          'input_path': input_path ,
+          'tar_path': tar_path 
+          }
+save_results([result], output_file, mode='a')  
+
+df = pd.read_csv(csv_file, sep=" ",header=None)   
+sample_list = df.values.tolist()
+self.paths = {
+    "input_path": [l[0] for l in self.sample_list],
+    "gt_path": [l[1] for l in self.sample_list]
+}
+"""    
 import os
 import numpy as np
 import pandas as pd
 import glob
 import datetime
-from hjnwtx.mkNCHJN import mkDir
-from shancx import Mul_sub_S
-
-# 将 getcheckdata 移到模块顶层
+from shancx import Mul_sub_S,Mul_sub 
+from shancx import crDir
 def getcheckdata(conf):
-    iph = conf[0]
+    satPath = conf[0]
     radar_dir_path = conf[1]
-    maskTrainPathm = conf[2]
-    sat_imin = conf[3]
+    sat_imin = conf[2]
     try:
-        satdata = np.load(iph)
-        radarpth = glob.glob(f"{radar_dir_path}/{sat_imin[:4]}/{sat_imin[:8]}/CR_{iph.split('/')[-1][5:-4]}*.npy")[0]
+        satdata = np.load(satPath)
+        radarpth = glob.glob(f"{radar_dir_path}/{sat_imin[:4]}/{sat_imin[:8]}/CR_{satPath.split('/')[-1][4:-4]}*.npy")[0]
         radardata = np.load(radarpth)
-        maskpath = glob.glob(f"{maskTrainPathm}/{sat_imin[:4]}/{sat_imin[:8]}/mask_{iph.split('/')[-1][5:-4]}*.npy")[0]
-        maskdata = np.load(maskpath)
-        print(satdata.shape, radardata.shape, maskdata.shape)
-        if radardata.shape != (1, 256, 256) or satdata.shape != (9, 256, 256) or maskdata.shape != (1, 256, 256):
-            return None
-        df = pd.DataFrame({'sat_path': [iph], 'radar_path': [radarpth], 'mask_path': [maskpath]})
+        if radardata.shape != satdata.shape :
+            return  
+        df = pd.DataFrame({'sat_path': [satPath], 'radar_path': [radarpth] })
         return df
     except Exception as e:
-        print(f"{iph} can not load succeed: {e}")
+        print(f"{satPath} can not load succeed: {e}")
         return None
-
-def generateList(sat_dir_path, radar_dir_path, maskTrainPathm, savepath, split_time, timelist):
-    """
-    生成训练、测试和验证数据集的文件列表，并保存为CSV文件。
-
-    参数:
-    - sat_dir_path: 卫星数据路径
-    - radar_dir_path: 雷达数据路径
-    - maskTrainPathm: 掩码数据路径
-    - savepath: 保存CSV文件的路径
-    - split_time: 划分训练、测试和验证集的时间点列表
-    - timelist: 要处理的时间范围列表
-    """
-    dataframes = {'train': [], 'test': [], 'valid': []}
-
-    for sat_imin in timelist:
-        print(f"Processing time: {sat_imin}")
-        sat_imin_dt = datetime.datetime.strptime(sat_imin, '%Y%m%d%H%M')
-        if sat_imin_dt < split_time[0]:
-            split_name = 'train'
-        elif split_time[0] <= sat_imin_dt < split_time[1]:
-            split_name = 'test'
-        else:
-            split_name = 'valid'
-
-        satpath = glob.glob(f"{sat_dir_path}/{sat_imin[:4]}/{sat_imin[:8]}/FY4B_{sat_imin}*.npy")
-        satpath.sort()
-        if satpath:
-            data = Mul_sub_S(getcheckdata, [satpath,[radar_dir_path],[maskTrainPathm],[sat_imin]], 6)
-            data = [i for i in data if i is not None ]
-            if data :
-                df = pd.concat(data)
-                dataframes[split_name].append(df)
-        else:
-            continue
-
-    train_df = pd.concat(dataframes['train']) if dataframes['train'] else pd.DataFrame()
-    test_df = pd.concat(dataframes['test']) if dataframes['test'] else pd.DataFrame()
-    valid_df = pd.concat(dataframes['valid']) if dataframes['valid'] else pd.DataFrame()
-    mkDir(savepath)
-    train_df.to_csv(f"{savepath}/df_train.csv", index=False, sep=',')
-    test_df.to_csv(f"{savepath}/df_test.csv", index=False, sep=',')
-    valid_df.to_csv(f"{savepath}/df_valid.csv", index=False, sep=',')
-    print(f"train_df {len(train_df)} test_df {len(test_df)} valid_df {len(valid_df)}")
-    print('complete!!!')
-
-if __name__ == "__main__":
-    print()
-    # 定义路径和参数
-    # sat_dir_path = '/mnt/wtx_weather_forecast/scx/dataset/sample/sta2radar_N/AGRI_area_4KM'
-    # radar_dir_path = '/mnt/wtx_weather_forecast/scx/dataset/sample/sta2radar_N/radar_selectArea_4KM_256'
-    # maskTrainPathm = "/mnt/wtx_weather_forecast/scx/dataset/sample/sta2radar_N/mask_selectArea_4KM"
-    # savepath = '/mnt/wtx_weather_forecast/scx/dataset/sample/datalist_vmask_mask_530T'
-    # split_time = [datetime.datetime(2025, 2, 27),
-    #               datetime.datetime(2025, 2, 28),
-    #               datetime.datetime(2025, 3, 4)]
-
-    # # 定义时间范围
-    # start_time = datetime.datetime(2024,8,3)
-    # end_time = datetime.datetime(2025, 3, 5)
-    # timelist = pd.date_range(start=start_time, end=end_time, freq='30T').strftime('%Y%m%d%H%M').tolist()
-
-    # # 调用方法    1.split_time   2. timelist  3. 路径
-    # generate_data_list(sat_dir_path, radar_dir_path, maskTrainPathm, savepath, split_time, timelist)
-
-
+def generateList(conf):
+    sat_dir_path, radar_dir_path, sat_imin= conf
+    satpath = glob.glob(f"{sat_dir_path}/{sat_imin[:4]}/{sat_imin[:8]}/SAT_{sat_imin}_*.npy")
+    satpath.sort()
+    if satpath:
+        datas = []
+        for path in satpath:
+            data = getcheckdata( (path,radar_dir_path,sat_imin))
+            datas.append(data)
+        datass = [i for i in datas if i is not None ]
+        if datass :
+            df = pd.concat(datass)
+            return df
+    else:
+            return None 
+import argparse
+import datetime
+import pandas as pd
+def options():
+    parser = argparse.ArgumentParser(description='examdatabasedata')
+    parser.add_argument('--times', type=str, default='202502010000,202506220000') 
+    parser.add_argument('--flag', type=str, default='val') 
+    config= parser.parse_args()
+    print(config)
+    config.times = config.times.split(",")
+    if len(config.times) == 1:
+        config.times = [config.times[0], config.times[0]]
+    config.times = [datetime.datetime.strptime(config.times[0], "%Y%m%d%H%M"),
+                    datetime.datetime.strptime(config.times[1], "%Y%m%d%H%M")]
+    return config
 """
+if __name__ == '__main__':
+    cfg = options()
+    sUTC = cfg.times[0]
+    eUTC = cfg.times[-1]
+    flag = cfg.flag     
+    sat_dir_path ="./"
+    radar_dir_path = f"./"
+    timelist = pd.date_range(sUTC, eUTC, t=f"{flag}")
+    timeListfliter = timelist[timelist.month.isin([1,4,7,10])&(timelist.day<=15)]
+    savepath = f'/mnt/wtx_weather_forecast/SAT/GOES18train_N/0624_1' 
+    crDir(savepath)
+    # 调用方法    1.split_time   2. timelist  3. 路径
+    dataL = Mul_sub(generateList,[ [sat_dir_path]        
+                                , [radar_dir_path]
+                                , timelist
+                                 ] 
+                                 )            
+    dataLs = [i for i in dataL if i is not None]    
+    if  flag =="trn":
+        train_df = pd.concat(dataLs) 
+        crDir(savepath)
+        train_df.to_csv(f"{savepath}/df_train.csv", index=False, sep=',')
+        print(f"train_df {len(train_df)}")
+        print('complete!!!') 
+        print(savepath)
+    if  flag == "val":
+        valid_df = pd.concat(dataLs)  
+        crDir(savepath)
+        valid_df.to_csv(f"{savepath}/df_valid.csv", index=False, sep=',')
+        print(f"valid_df {len(valid_df)}")
+        print('complete!!!')
+        print(savepath)   
+        
 
-
+"""
+"""
 import os
 import numpy as np
 import pandas as pd
 import glob
 import datetime
-from hjnwtx.mkNCHJN import mkDir
-from shancx import Mul_sub_S
-from shancx.Plot import plotRadar,plotMat
-
-# 将 getcheckdata 移到模块顶层
+from shancx import Mul_sub_S,Mul_sub 
+from shancx import crDir
 def getcheckdata(conf):
-    iph = conf[0]
-    radar_dir_path = conf[1]
-    maskTrainPathm = conf[2]
-    sat_imin = conf[3]
+    inputPath = conf[0]
+    target_dir_path = conf[1]
+    imin = conf[2]
     try:
-        satdata = np.load(iph)
-        radarpth = glob.glob(f"{radar_dir_path}/{sat_imin[:4]}/{sat_imin[:8]}/CR_{iph.split('/')[-1][5:-4]}*.npy")[0]
-        radardata = np.load(radarpth)
-        maskpath = glob.glob(f"{maskTrainPathm}/{sat_imin[:4]}/{sat_imin[:8]}/mask_{iph.split('/')[-1][5:-4]}*.npy")[0]
-        maskdata = np.load(maskpath)
-        print(satdata.shape, radardata.shape, maskdata.shape)
-        if radardata.shape != (1, 256, 256) or satdata.shape != (9, 256, 256) or maskdata.shape != (1, 256, 256):
-            return None
-        if np.mean(radardata) > 20  or np.mean(satdata) > 285 :
-            plotMat(satdata[0],name=f"satdata_{sat_imin}")
-            plotRadar(satdata[0],name=f"radar_{sat_imin}")
-            return None
-        df = pd.DataFrame({'sat_path': [iph], 'radar_path': [radarpth], 'mask_path': [maskpath]})
+        inputdata = np.load(inputPath)
+        targetpth = glob.glob(f"{target_dir_path}/{imin[:4]}/{imin[:8]}/analy_{inputPath.split('/')[-1][3:-4]}*.npy")[0]
+        targetdata = np.load(targetpth)
+        if inputdata.shape != targetdata.shape :
+            return  
+        df = pd.DataFrame({'inputPath': [inputPath], 'targetpth': [targetpth] })
         return df
     except Exception as e:
-        print(f"{iph} can not load succeed: {e}")
+        print(f"{inputPath} can not load succeed: {e}")
         return None
-
-def generateList(sat_dir_path, radar_dir_path, maskTrainPathm, savepath, timelist,flag):
-    dataframes = {'train': [], 'test': [], 'valid': []}
-    for sat_imin in timelist:
-        split_name = flag
-        satpath = glob.glob(f"{sat_dir_path}/{sat_imin[:4]}/{sat_imin[:8]}/FY4B_{sat_imin}*.npy")
-        satpath.sort()
-        if satpath:
-            data = Mul_sub_S(getcheckdata, [satpath,[radar_dir_path],[maskTrainPathm],[sat_imin]], 6)
-            data = [i for i in data if i is not None ]
-            if data :
-                df = pd.concat(data)
-                dataframes[split_name].append(df)
-        else:
-            continue
-    if  flag =="train":
-        train_df = pd.concat(dataframes['train']) if dataframes['train'] else pd.DataFrame()
-        mkDir(savepath)
-        train_df.to_csv(f"{savepath}/df_train.csv", index=False, sep=',')
-        print(f"train_df {len(train_df)}")
-        print('complete!!!')
-    if  flag == "valid":
-        valid_df = pd.concat(dataframes['valid']) if dataframes['valid'] else pd.DataFrame()
-        mkDir(savepath)
-        valid_df.to_csv(f"{savepath}/df_valid.csv", index=False, sep=',')
-        print(f"valid_df {len(valid_df)}")
-        print('complete!!!')
-
-if __name__ == "__main__":
-    print()
-    # 定义路径和参数
-    sat_dir_path = '/mnt/wtx_weather_forecast/scx/dataset/sample/sta2radar_N/AGRI_area_4KMQC2'
-    radar_dir_path = '/mnt/wtx_weather_forecast/scx/dataset/sample/sta2radar_N/radar_selectArea_4KMQC2_256'
-    maskTrainPathm = "/mnt/wtx_weather_forecast/scx/dataset/sample/sta2radar_N/mask_selectArea_4KMQC2"
-    savepath = '/mnt/wtx_weather_forecast/scx/dataset/sample/datalist_vmask_filter'
-    split_time = [datetime.datetime(2025, 2, 27),
-                  datetime.datetime(2025, 2, 28),
-                  datetime.datetime(2025, 3, 4)]
-
-    # 定义时间范围
-    start_time = datetime.datetime(2025,4,9)
-    end_time =   datetime.datetime(2025,4,14)
-    flag = "train"
-    timelist = pd.date_range(start=start_time, end=end_time, freq='30T').strftime('%Y%m%d%H%M').tolist()
-
+def generateList(conf):
+    imin,input_dir_path, target_dir_path= conf
+    imin = imin.strftime("%Y%m%d%H%M")
+    targetpath = glob.glob(f"{input_dir_path}/{imin[:4]}/{imin[:8]}/EC_{imin}_*.npy")
+    targetpath.sort()
+    if targetpath:
+        datas = []
+        for path in targetpath:
+            data = getcheckdata( (path,target_dir_path,imin))
+            datas.append(data)
+        datass = [i for i in datas if i is not None ]
+        if datass :
+            df = pd.concat(datass)
+            return df
+    else:
+            return None 
+import argparse
+import datetime
+import pandas as pd
+def options():
+    parser = argparse.ArgumentParser(description='examdatabasedata')
+    parser.add_argument('--times', type=str, default='202505210030,202505210130') 
+    parser.add_argument('--flag', type=str, default='val') 
+    parser.add_argument('--freq', type=str, default="1h") 
+    config= parser.parse_args()
+    print(config)
+    config.times = config.times.split(",")
+    if len(config.times) == 1:
+        config.times = [config.times[0], config.times[0]]
+    config.times = [datetime.datetime.strptime(config.times[0], "%Y%m%d%H%M"),
+                    datetime.datetime.strptime(config.times[1], "%Y%m%d%H%M")]
+    return config
+if __name__ == '__main__':
+    cfg = options()
+    sUTC = cfg.times[0]
+    eUTC = cfg.times[-1]
+    freq = cfg.freq     
+    input_dir_path ="/mnt/wtx_weather_forecast/scx/smart_grid/train/clip_EC"
+    target_dir_path = f"/mnt/wtx_weather_forecast/scx/smart_grid/train/analy_1_256"
+    timelist = pd.date_range(sUTC, eUTC, freq=f"{freq}")
+    timeListfliter = timelist[timelist.month.isin([1,4,5,7,10])]
+    timeListfliter = list(timeListfliter)
+    savepath = f'/mnt/wtx_weather_forecast/SAT/downscale/0901' 
+    crDir(savepath)
     # 调用方法    1.split_time   2. timelist  3. 路径
-    generateList(sat_dir_path, radar_dir_path, maskTrainPathm, savepath, timelist,flag)
-
-"""
+ 
+    dataL = Mul_sub(generateList,[ timeListfliter,[input_dir_path] , [target_dir_path]] )          
+    dataLs = [i for i in dataL if i is not None]  
+    pd.concat(dataLs).to_csv('train.txt', sep=' ', index=False)
+    a = pd.read_csv("./train.txt",sep=" ").values.tolist()
+"""
+        
+