shancx 1.9.33.109__py3-none-any.whl → 1.9.33.218__py3-none-any.whl

shancx/QC.py

@@ -0,0 +1,223 @@
+import cv2
+import numpy as np
+from numba import jit
+def removeSmallPatches(binary_mask, min_pixels=50, min_area=40):
+    binary_mask = (binary_mask > 0).astype(np.uint8)
+    num_labels, labels, stats, centroids = cv2.connectedComponentsWithStats(
+        binary_mask, connectivity=8
+    )    
+    output_mask = np.zeros_like(binary_mask)    
+    for i in range(1, num_labels):
+        pixel_count = stats[i, cv2.CC_STAT_AREA]        
+        if pixel_count < min_pixels:
+            continue
+        component_mask = (labels == i).astype(np.uint8)
+        contours, _ = cv2.findContours(component_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)        
+        if contours:
+            contour = contours[0]
+            area = cv2.contourArea(contour)            
+            if area < min_area:
+                continue        
+        output_mask[labels == i] = 255    
+    return output_mask
+
+"""
+mask = removeSmallPatches(b, min_pixels=50, min_area=40)
+data = np.where(mask, data, 0)
+filtered_data = np.full([256,256],0)
+filtered_data[mask] = e[mask]
+"""
+
+import cv2
+import numpy as np
+from concurrent.futures import ThreadPoolExecutor
+def process_block_optimized(args):
+    block, coords, min_pixels, min_area = args
+    y, x, y_end, x_end = coords    
+    num_labels, labels, stats, _ = cv2.connectedComponentsWithStats(block, 8)
+    result = np.zeros_like(block)    
+    valid_labels = []
+    for i in range(1, num_labels):
+        if stats[i, cv2.CC_STAT_AREA] >= min_pixels:
+            valid_labels.append(i)   
+    for i in valid_labels:
+        component_mask = (labels == i).astype(np.uint8)
+        contours, _ = cv2.findContours(component_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)        
+        if contours and cv2.contourArea(contours[0]) >= min_area:
+            result[labels == i] = 255    
+    return result, coords
+def removeSmallPatches_fast(binary_mask, min_pixels=100, min_area=40, num_workers=3):
+    binary_mask = (binary_mask > 0).astype(np.uint8)
+    h, w = binary_mask.shape
+    output = np.zeros_like(binary_mask)
+    block_size = 2000    
+    blocks = []
+    for y in range(0, h, block_size):
+        for x in range(0, w, block_size):
+            y_end, x_end = min(y+block_size, h), min(x+block_size, w)
+            block = binary_mask[y:y_end, x:x_end]
+            blocks.append((block, (y, x, y_end, x_end), min_pixels, min_area))    
+    with ThreadPoolExecutor(num_workers) as executor:
+        for result, (y, x, y_end, x_end) in executor.map(process_block_optimized, blocks):
+            output[y:y_end, x:x_end] = result    
+    return output
+
+"""
+mask = removeSmallPatches(b, min_pixels=50, min_area=40)
+data = np.where(mask, data, 0)
+filtered_data = np.full([256,256],0)
+filtered_data[mask] = e[mask]
+"""
+
+import cv2
+import numpy as np
+from numba import jit, prange
+from concurrent.futures import ThreadPoolExecutor
+def removeSmallPatches_optimized(binary_mask, min_pixels=50, min_area=40):
+    binary_mask = (binary_mask > 0).astype(np.uint8)
+    num_labels, labels, stats, _ = cv2.connectedComponentsWithStats(binary_mask, connectivity=8)
+    output_mask = np.zeros_like(binary_mask)
+    valid_labels = [i for i in range(1, num_labels) if stats[i, cv2.CC_STAT_AREA] >= min_pixels]
+    for i in valid_labels:
+        contour_mask = (labels == i).astype(np.uint8)
+        contours, _ = cv2.findContours(contour_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+        
+        if contours and cv2.contourArea(contours[0]) >= min_area:
+            output_mask[labels == i] = 255
+    
+    return output_mask
+
+@jit(nopython=True, parallel=True, nogil=True)
+def numba_filter_components(labels, stats, min_pixels, min_area):
+    height, width = labels.shape
+    output = np.zeros((height, width), dtype=np.uint8)
+    for i in prange(1, stats.shape[0]):
+        if stats[i, 4] >= min_pixels:  # stats[i, 4] 对应 cv2.CC_STAT_AREA
+            for y in range(height):
+                for x in range(width):
+                    if labels[y, x] == i:
+                        output[y, x] = 255    
+    return output
+def removeSmallPatches_numba(binary_mask, min_pixels=50, min_area=40):
+    binary_mask = (binary_mask > 0).astype(np.uint8)
+    num_labels, labels, stats, _ = cv2.connectedComponentsWithStats(binary_mask, connectivity=8)
+    output_mask = numba_filter_components(labels, stats, min_pixels, min_area)
+    if min_area > 0:
+        num_labels2, labels2, stats2, _ = cv2.connectedComponentsWithStats(output_mask, connectivity=8)
+        final_output = np.zeros_like(output_mask)        
+        for i in range(1, num_labels2):
+            contour_mask = (labels2 == i).astype(np.uint8)
+            contours, _ = cv2.findContours(contour_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)            
+            if contours and cv2.contourArea(contours[0]) >= min_area:
+                final_output[labels2 == i] = 255        
+        return final_output    
+    return output_mask
+def process_block_optimized_v2(args):
+    block, coords, min_pixels, min_area = args
+    num_labels, labels, stats, _ = cv2.connectedComponentsWithStats(block, connectivity=8)
+    result = np.zeros_like(block)
+    valid_labels = []
+    for i in range(1, num_labels):
+        if stats[i, cv2.CC_STAT_AREA] >= min_pixels:
+            valid_labels.append(i)
+    for i in valid_labels:
+        component_indices = (labels == i)
+        if component_indices.any():
+            component_mask = component_indices.astype(np.uint8)
+            contours, _ = cv2.findContours(component_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)            
+            if contours and cv2.contourArea(contours[0]) >= min_area:
+                result[component_indices] = 255    
+    return result, coords
+def removeSmallPatches_fast_v2(binary_mask, min_pixels=100, min_area=40, num_workers=4):
+    binary_mask = (binary_mask > 0).astype(np.uint8)
+    h, w = binary_mask.shape
+    optimal_block_size = max(500, min(2000, (h * w) // (num_workers * 10000)))    
+    output = np.zeros_like(binary_mask)
+    blocks = []
+    for y in range(0, h, optimal_block_size):
+        for x in range(0, w, optimal_block_size):
+            y_end, x_end = min(y + optimal_block_size, h), min(x + optimal_block_size, w)
+            block = binary_mask[y:y_end, x:x_end].copy() 
+            blocks.append((block, (y, x, y_end, x_end), min_pixels, min_area))    
+    actual_workers = min(num_workers, len(blocks))    
+    with ThreadPoolExecutor(max_workers=actual_workers) as executor:
+        for result, (y, x, y_end, x_end) in executor.map(process_block_optimized_v2, blocks):
+            output[y:y_end, x:x_end] = result    
+    return output
+def get_optimal_function(binary_mask, min_pixels=50, min_area=40):
+    h, w = binary_mask.shape
+    total_pixels = h * w    
+    if total_pixels < 1000000:  # 小于100万像素
+        return removeSmallPatches_optimized    
+    if min_area <= 0:
+        return removeSmallPatches_numba    
+    return removeSmallPatches_fast_v2
+def auto_remove_small_patches(binary_mask, min_pixels=50, min_area=40):
+    optimal_func = get_optimal_function(binary_mask, min_pixels, min_area)
+    return optimal_func(binary_mask, min_pixels, min_area)
+""" 
+try:
+    result = auto_remove_small_patches(binary_mask, min_pixels=50, min_area=40)
+except Exception as e:
+    from original_module import removeSmallPatches
+    result = removeSmallPatches(binary_mask, min_pixels=50, min_area=40)
+ removeSmallPatches_numba optimum performance min_area <= 0
+ removeSmallPatches_fast_v2 area
+"""
+@jit(nopython=True, parallel=True, nogil=True)
+def QC_simple_numba(mat, dbzTH = 10.0, areaTH=20):
+    # Create a copy of the matrix
+    mat1 = np.copy(mat)
+    rows, cols = mat1.shape
+    
+    # Create binary mask based on threshold
+    mask = np.zeros((rows, cols), dtype=np.uint8)
+    for i in range(rows):
+        for j in range(cols):
+            if mat1[i, j] > dbzTH:
+                mask[i, j] = 1    
+    # Simple 8-connectivity region labeling (flood fill algorithm)
+    labels = np.zeros((rows, cols), dtype=np.int32)
+    current_label = 1
+    region_areas = []    
+    for i in range(rows):
+        for j in range(cols):
+            # If current pixel is foreground and not yet labeled
+            if mask[i, j] == 1 and labels[i, j] == 0:
+                # Start flood fill
+                stack = [(i, j)]
+                labels[i, j] = current_label
+                area = 0
+                
+                while stack:
+                    x, y = stack.pop()
+                    area += 1
+                    
+                    # Check 8 surrounding pixels
+                    for dx in [-1, 0, 1]:
+                        for dy in [-1, 0, 1]:
+                            nx, ny = x + dx, y + dy
+                            # Check bounds and conditions
+                            if (0 <= nx < rows and 0 <= ny < cols and 
+                                mask[nx, ny] == 1 and labels[nx, ny] == 0):
+                                labels[nx, ny] = current_label
+                                stack.append((nx, ny))
+                
+                region_areas.append(area)
+                current_label += 1
+    
+    # Apply area threshold filtering
+    for i in range(rows):
+        for j in range(cols):
+            if labels[i, j] > 0 and region_areas[labels[i, j] - 1] < areaTH:
+                mat1[i, j] = 0    
+    return mat1
+def QC_ref_numba(mat, dbzTH = 10, areaTH=20):
+    for i in range(len(mat)):      
+        mat[i] = QC_simple_numba(mat[i], dbzTH, areaTH)
+    return mat 
+"""
+CR = subset["CR"].data[0].copy()
+CR[CR < 6] = 0
+CR = QC_ref_numba(CR[None], areaTH=15)[0]
+"""

shancx/Read.py

@@ -12,7 +12,6 @@ def readGrib(file_path, target_param=None):
                                    'shortName': getattr(grb, 'shortName', 'N/A'),
                                    'level': getattr(grb, 'level', -999),
                                    'typeOfLevel': getattr(grb, 'typeOfLevel', 'N/A'),
-                                   'forecastTime': getattr(grb, 'forecastTime', -999),
                                    'validDate': getattr(grb, 'validDate', 'N/A'),
                                    'units': getattr(grb, 'units', 'N/A'),
                                    'shape': grb.values.shape
@@ -54,12 +53,20 @@ if __name__ == "__main__":
         """
         latArr = latMat[:,0]
         lonArr = lonMat[0]
-        """
-    
-import yaml
-def _load_config(config_path: str) :
-    """加载YAML配置文件"""
-    print(f"load config file Get configuration parameters: {config_path}")
-    with open(config_path, 'r') as f:
-        config = yaml.safe_load(f)
-    return config
+        """    
+
+import numpy as np
+from pathlib import Path
+def npsavez(output, data):
+    output = output.replace('.npy', '.npz')
+    output_path = Path(output).with_suffix('.npz')
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    np.savez_compressed(output_path, data=data)
+    print(f"{output_path} done")
+def nploadz(output_path):
+    return np.load(Path(output_path).with_suffix('.npz'))['data']
+
+"""
+savez(output, data)
+nploadz(output_path)
+"""

shancx/Resize.py

@@ -0,0 +1,79 @@
+import numpy as np
+import random
+# import albumentations as A
+# import cv2
+
+# def resize_array(array, size):
+#     # 定义变换管道
+#     transform = A.Compose([
+#         A.SmallestMaxSize(max_size=size, interpolation=cv2.INTER_AREA)
+#     ])
+#     transformed_array = transform(image=array)["image"]
+#     return transformed_array
+
+def crop_array(array, crop_side_len):
+    cropper = A.RandomCrop(height=crop_side_len, width=crop_side_len)
+    cropped_array = cropper(image=array)["image"]
+    return cropped_array
+
+def crop_cna_pair(min_side_len, low_res_data, high_res_data):
+    crop_side_len = min_side_len
+    top = random.randint(0, low_res_data.shape[-2] - crop_side_len)
+    left = random.randint(0, low_res_data.shape[-1] - crop_side_len)
+    super_factor = high_res_data.shape[-2] / low_res_data.shape[-2] # Assuming the ratio in height dimensio
+    cropped_low_res = low_res_data[top:top + crop_side_len, left:left + crop_side_len]
+    cropped_high_res = high_res_data[int(top * super_factor):int((top + crop_side_len) * super_factor),
+                                     int(left * super_factor):int((left + crop_side_len) * super_factor)]
+    return cropped_low_res, cropped_high_res
+def random_crop_pair(min_side_len, low_res_data, high_res_data):
+    crop_side_len = min_side_len
+    top = random.randint(0, low_res_data.shape[-2] - crop_side_len)
+    left = random.randint(0, low_res_data.shape[-1] - crop_side_len)
+    super_factor = high_res_data.shape[-2] / low_res_data.shape[-2] # Assuming the ratio in height dimension
+    cropped_low_res = low_res_data[top:top + crop_side_len, left:left + crop_side_len]
+    cropped_high_res = high_res_data[int(top * super_factor):int((top + crop_side_len) * super_factor),
+                                     int(left * super_factor):int((left + crop_side_len) * super_factor)]
+    return cropped_low_res, cropped_high_res
+
+import random
+
+def random_crop_triplet(min_side_len, low_res_data, high_res_data1, high_res_data2):
+    top = random.randint(0, low_res_data.shape[-2] - min_side_len)
+    left = random.randint(0, low_res_data.shape[-1] - min_side_len)
+    cropped_low_res = low_res_data[..., top:top + min_side_len, left:left + min_side_len]
+    factor1_h = high_res_data1.shape[-2] / low_res_data.shape[-2]
+    factor1_w = high_res_data1.shape[-1] / low_res_data.shape[-1]
+    cropped_high_res1 = high_res_data1[...,
+                                       int(top * factor1_h):int((top + min_side_len) * factor1_h),
+                                       int(left * factor1_w):int((left + min_side_len) * factor1_w)]
+    factor2_h = high_res_data2.shape[-2] / low_res_data.shape[-2]
+    factor2_w = high_res_data2.shape[-1] / low_res_data.shape[-1]
+    cropped_high_res2 = high_res_data2[...,
+                                       int(top * factor2_h):int((top + min_side_len) * factor2_h),
+                                       int(left * factor2_w):int((left + min_side_len) * factor2_w)]
+    return cropped_low_res, cropped_high_res1, cropped_high_res2
+
+def random_crop_single(cropsize, input_data):
+    # 确定裁剪的边长
+    crop_side_len = cropsize
+
+    # 随机选择左上角裁剪点
+    top = random.randint(0, input_data.shape[0] - crop_side_len)
+    left = random.randint(0, input_data.shape[1] - crop_side_len)
+
+    # 裁剪输入数据
+    cropped_data = input_data[top:top + crop_side_len, left:left + crop_side_len]
+
+    return cropped_data
+
+
+if __name__ == "__main__":
+    low_res_data = np.load("./SAT_202507010900_49.42_117.82_100.npy")
+    high_res_data = np.load("./CR_202507010900_49.42_117.82_100.npy")
+    high_res_data1 = np.load("./mask_202507010900_49.42_117.82_100.npy")
+    d1,d2,d3 =  random_crop_triplet(128, low_res_data, high_res_data[0], high_res_data1[0])
+    transformed_low_res_data = resize_array(low_res_data, 240)
+    transformed_high_res_data = resize_array(high_res_data, 960)
+    np.save("transformed_low_res_data.npy", transformed_low_res_data)
+    np.save("transformed_high_res_data.npy", transformed_high_res_data)
+

shancx/SN/__init__.py

@@ -67,4 +67,11 @@ def sendMESplus(message,base = None):
         res = requests.post(url,json=data,headers=headers)
     except Exception as e:
         print(e)
- 
+"""
+    import torch.nn as nn
+    # gpu_ids = [3, 4, 5]
+    device = torch.device(f"cuda:{gpu_ids[0]}" if torch.cuda.is_available() else "cpu")
+    model = get_model(model_name, in_channels=in_channels).to(device)
+    if len(gpu_ids) > 1:
+        model = torch.nn.DataParallel(model, device_ids=gpu_ids)  
+"""

shancx/Time/timeCycle.py

@@ -60,8 +60,102 @@ for i in range(7):
 """
 
 """
-检查文件夹
+import glob
+import os
+import numpy as np
+from shancx import crDir
+from shancx.NN import _loggers
+logger = _loggers()
+import netCDF4 as nc
+import numpy as np
+from shancx.Plot import plotRadar
+from shancx import crDir  
+import traceback
+from dateutil.relativedelta import relativedelta
+paths = glob.glob("/root/data/ec_filter_npy_data/*")  
+basePath = f"/mnt/wtx_weather_forecast/scx/sever7/test/RADA/MQPF1109_1"
+output_dirH9Npy = f"/mnt/wtx_weather_forecast/SAT/H9/sat_npy_CHN"
+def GetMulData(conf):  
+    sUTC = conf[0]
+    sUTCstr = sUTC.strftime("%Y%m%d%H%M") 
+    output_path =f"{output_dirH9Npy}/{sUTCstr[:4]}/{sUTCstr[:8]}/MSP3_PMSC_H9_GEO_FD_{sUTCstr[:12]}_00000-00000.npy"
+    inputPathstr = f"{basePath}/{sUTCstr[:4]}/{sUTCstr[:8]}/*{sUTCstr[:12]}*.nc"
+    inputPathL =  glob.glob(inputPathstr)
+    if len(inputPathL) ==0 :  
+        print(f"outpath {inputPathstr} is missing ")
+        return None 
+    inputPath = inputPathL[0]
+    with nc.Dataset(inputPath) as dataNC:  
+        CR = dataNC["CR"][:]
+        lat = dataNC["lat"][:]
+        lon = dataNC["lon"][:]        
+    crDir(output_path)
+    np.save(output_path,CR.data)
+    logger.info(f"{output_path} done ") 
+
+from shancx import Mul_sub    
+import argparse
+import datetime
+import pandas as pd
+def options():
+    parser = argparse.ArgumentParser(description='examdatabasedata')
+    parser.add_argument('--times', type=str, default='202507010000,202510010000') 
+    config= parser.parse_args()
+    print(config)
+    config.times = config.times.split(",")
+    if len(config.times) == 1:
+        config.times = [config.times[0], config.times[0]]
+    config.times = [datetime.datetime.strptime(config.times[0], "%Y%m%d%H%M"),
+                    datetime.datetime.strptime(config.times[1], "%Y%m%d%H%M")]
+    return config
+if __name__ == '__main__':
+    cfg = options()
+    sUTC = cfg.times[0]
+    eUTC = cfg.times[-1]
+    timeList = pd.date_range(sUTC, eUTC, freq='3h')  #6T 分钟 
+    print(timeList)
+    Mul_sub(GetMulData,[timeList],6)
+"""
+
+"""
+import glob
+import os
+import numpy as np
+import traceback
+from dateutil.relativedelta import relativedelta
+import os 
+from shancx.NN import Mul_TH
+import argparse
+import datetime
+import pandas as pd
+def GetMulData(conf):
+    sUTC = conf[0]
+    sUTCstr = sUTC.strftime("%Y%m%d%H%M") 
+    commandstr = f""
+    os.system(commandstr) 
+def options():
+    parser = argparse.ArgumentParser(description='examdatabasedata')
+    parser.add_argument('--times', type=str, default='202508010000,202508010500') 
+    config= parser.parse_args()
+    print(config)
+    config.times = config.times.split(",")
+    if len(config.times) == 1:
+        config.times = [config.times[0], config.times[0]]
+    config.times = [datetime.datetime.strptime(config.times[0], "%Y%m%d%H%M"),
+                    datetime.datetime.strptime(config.times[1], "%Y%m%d%H%M")]
+    return config
+if __name__ == '__main__':
+    cfg = options()
+    sUTC = cfg.times[0]
+    eUTC = cfg.times[-1]
+    timeList = pd.date_range(sUTC, eUTC, freq='10T')  #6T 分钟 
+    print(timeList)
+    Mul_TH(GetMulData,[timeList],3)
+cd /mnt/wtx_weather_forecast/scx/sever7/SATdata/mkH9 ;timeout 1200s /home/scx/miniconda3/envs/H9/bin/python mkH9.py --time {sUTCstr[:12]}
+
+"""
 
+"""
 from shancx import crDir
 import os
 from shancx import loggers as logger
@@ -201,28 +295,8 @@ if __name__ == '__main__':
     minmax = Mul_sub(GetMulData,[summer_timeList],10)
     global_min = min(d['min'] for d in minmax)
     global_max = max(d['max'] for d in minmax)  #117
-    print()
- 
+    print() 
 
 """
 
-"""
-BASE_TARGET_PATH = "/mnt/wtx_weather_forecast/SAT/H9/sat_nc"
-def get_mqpf_paths(UCTstr,sat_cd = "H9"):
-    year = UCTstr[:4]
-    date = UCTstr[:8]
-    mqpfPath_pattern = os.path.join(BASE_TARGET_PATH,year, date,f"MSP3_PMSC_{sat_cd}_GEO_FD_{UCTstr}_00000-00000.nc")
-    return mqpfPath_pattern  
-def map_data(conf):
-    UCT = conf[0]
-    # UCT = CST + relativedelta(hours=-8)
-    UCTstr = UCT.strftime("%Y%m%d%H%M")
-    mqpfPath_pattern = get_mqpf_paths(UCTstr)
-    mqpfPath_list = glob.glob(mqpfPath_pattern)
-    if len(mqpfPath_list) > 0:
-        with nc.Dataset(mqpfPath_list[0]) as dataNC:
-            mqpf = dataNC.variables["B10"][:]
-        return mqpf
-    else:
-        return  np.full((1, 6201, 5201), np.nan)
-"""
+