jacksung-dev 0.0.4.15__py3-none-any.whl

jacksung/ai/utils/fy.py

@@ -0,0 +1,333 @@
+import sys
+import threading
+
+sys.path.append('../')
+import shutil
+from osgeo import gdal, osr
+import numpy as np
+import matplotlib.pyplot as plt
+from random import randint
+import os
+from jacksung.utils.data_convert import nc2np, np2tif
+from jacksung.utils.image import crop_png, zoom_image, zoomAndDock
+from jacksung.utils.cache import Cache
+import rasterio
+from cartopy.mpl.ticker import LongitudeFormatter, LatitudeFormatter
+from matplotlib.colors import LinearSegmentedColormap
+import cartopy.feature as cfeature
+import cartopy.crs as ccrs
+from datetime import datetime, timedelta
+import netCDF4 as nc
+from jacksung.utils.data_convert import np2tif, Coordinate
+from tqdm import tqdm
+from jacksung.utils.multi_task import MultiTasks, type_process
+import cv2
+
+reference_cache = Cache(10)
+
+x_range = {'left': -60, 'top': 60, 'bottom': -60, 'right': 60, 'width': 2400, 'height': 2400}
+min_x_range = {'left': -60, 'top': 60, 'bottom': -60, 'right': 60, 'width': 480, 'height': 480}
+static_params = {4000: {'l': 2747, 'c': 2747, 'COFF': 1373.5, 'CFAC': 10233137, 'LOFF': 1373.5, 'LFAC': 10233137},
+                 2000: {'l': 5495, 'c': 5495, 'COFF': 2747.5, 'CFAC': 20466274, 'LOFF': 2747.5, 'LFAC': 20466274},
+                 1000: {'l': 10991, 'c': 10991, 'COFF': 5495.5, 'CFAC': 40932549, 'LOFF': 5495.5, 'LFAC': 40932549},
+                 500: {'l': 21983, 'c': 21983, 'COFF': 10991.5, 'CFAC': 81865099, 'LOFF': 10991.5, 'LFAC': 81865099},
+                 250: {'l': 43967, 'c': 43967, 'COFF': 21983.5, 'CFAC': 163730199, 'LOFF': 21983.5, 'LFAC': 163730199}}
+
+
+def getFY_coord(ld):
+    return Coordinate(left=ld + x_range['left'], top=x_range['top'], right=ld + x_range['right'],
+                      bottom=x_range['bottom'], h=x_range['height'], w=x_range['width'])
+    # return Coordinate(left=ld - 45, top=36, right=ld + 45, bottom=-36, h=1571, w=1963)
+
+
+def getFY_coord_min(ld):
+    return Coordinate(left=ld + min_x_range['left'], top=min_x_range['top'], right=ld + min_x_range['right'],
+                      bottom=min_x_range['bottom'], h=min_x_range['height'], w=min_x_range['width'])
+
+
+def getFY_coord_clip(area=((100, 140, 10), (20, 60, 10))):
+    return Coordinate(left=area[0][0], top=area[1][1], right=area[0][1], bottom=area[1][0], h=800, w=800)
+
+
+# FY4星下点行列号转经纬度
+def xy2coordinate(l, c, ld=105, res=4000):
+    ea = 6378.137
+    eb = 6356.7523
+    h = 42164
+    # 4000m分辨率
+    COFF = static_params[res]['COFF']
+    CFAC = static_params[res]['CFAC']
+    LOFF = static_params[res]['LOFF']
+    LFAC = static_params[res]['LFAC']
+
+    x = (np.pi * (c - COFF)) / (180 * (2 ** -16) * CFAC)
+    y = (np.pi * (l - LOFF)) / (180 * (2 ** -16) * LFAC)
+
+    sd_t1 = np.square(h * np.cos(x) * np.cos(y))
+    sd_t2 = np.square(np.cos(y)) + np.square(ea) / np.square(eb) * np.square(np.sin(y))
+    sd_t3 = np.square(h) - np.square(ea)
+    sd = np.sqrt(sd_t1 - sd_t2 * sd_t3)
+    sn = (h * np.cos(x) * np.cos(y) - sd) / (np.cos(y) ** 2 + ea ** 2 / eb ** 2 * np.sin(y) ** 2)
+
+    s1 = h - sn * np.cos(x) * np.cos(y)
+    s2 = sn * np.sin(x) * np.cos(y)
+    s3 = -sn * np.sin(y)
+    sxy = np.sqrt(s1 ** 2 + s2 ** 2)
+
+    lon = 180 / np.pi * np.arctan(s2 / s1) + ld
+    lat = 180 / np.pi * np.arctan(ea ** 2 / eb ** 2 * s3 / sxy)
+    return lon, lat
+
+
+def convert_file2idx(file_name):
+    file_name = file_name.replace('.npy', '')
+    h, m = file_name[:2], file_name[2:4]
+    return int(h) * 4 + int(m) // 15
+
+
+def get_reference(ld):
+    # 构造控制点列表 gcps_list
+    gcps_list = []
+    step = 50
+    last_corrd = []
+    lc_list = []
+    latlon_list = []
+    for l in range(0, 2748, step):
+        for c in range(0, 2748, step):
+            lon, lat = xy2coordinate(l, c, ld=ld)
+            if str(lon) == 'nan' or str(lat) == 'nan':
+                continue
+            skip_flag = False
+            for corrd in last_corrd:
+                if (corrd[0] - lon) ** 2 + (corrd[1] - lat) ** 2 <= 100:
+                    skip_flag = True
+                    break
+            if skip_flag:
+                continue
+            last_corrd.append([lon, lat])
+            gcps_list.append(gdal.GCP(lon, lat, 0, c, l))
+            lc_list.append((l, c))
+            latlon_list.append((lon, lat))
+    # 设置空间参考
+    # print('控制点数目：', len(gcps_list))
+    # print([(l, c) for l, c in lc_list])
+    # print([(lon, lat) for lon, lat in latlon_list])
+    spatial_reference = osr.SpatialReference()
+    spatial_reference.SetWellKnownGeogCS('WGS84')
+    return spatial_reference, gcps_list
+
+
+def get_agri_file_path(data_path, data_date):
+    e_date = data_date + timedelta(minutes=14, seconds=59)
+    # /mnt/data1/szj/FY/downloaded_file
+    # FY4B-_AGRI--_N_DISK_1330E_L1-_FDI-_MULT_NOM_20221007234500_20221007235959_4000M_V0001.HDF
+    parent_dir = rf'{data_path}/{data_date.year}/{data_date.month}/{data_date.day}'
+    start_date_str = data_date.strftime('%Y%m%d%H%M%S')
+    end_date_str = e_date.strftime('%Y%m%d%H%M%S')
+    file_path = rf'{parent_dir}/FY4B-_AGRI--_N_DISK_{get_ld(data_date)}0E_L1-_FDI-_MULT_NOM_{start_date_str}_{end_date_str}_4000M_V0001.HDF'
+    if os.path.exists(file_path):
+        return file_path
+    else:
+        return None
+
+
+def getNPfromHDFClip(ld, file_path, file_type='FDI', lock=None, area=((100, 140, 10), (20, 60, 10)), cache=None):
+    lon_d = int((ld - (area[0][0] + area[0][1]) / 2) * 20)
+    lat_d = int(((area[1][0] + area[1][1]) / 2) * 20)
+    np_data = getNPfromHDF(file_path, file_type, lock, cache=cache)
+    np_data = np_data[:, 800 - lat_d:1600 - lat_d, 800 - lon_d:1600 - lon_d]
+    return np_data
+
+
+def getNPfromHDF(hdf_path, file_type='FDI', lock=None, cache=None, return_coord=False, only_coord=False):
+    prase_data = prase_filename(os.path.basename(hdf_path))
+    ld = prase_data['position']
+    coord = Coordinate(left=ld - 60, top=60, right=ld + 60, bottom=-60, x_res=0.05, y_res=0.05)
+    if only_coord:
+        return coord
+    if cache is not None:
+        n_data = cache.get_key_in_cache(hdf_path + file_type)
+        if n_data is None:
+            n_data = _getNPfromHDF(hdf_path, file_type=file_type, lock=lock)
+            cache.add_key(hdf_path + file_type, 'None' if n_data is None else n_data)
+        if type(n_data) == str and n_data == 'None':
+            if return_coord:
+                return None, None
+            else:
+                return None
+        else:
+            if return_coord:
+                return n_data, coord
+            else:
+                return n_data
+    else:
+        return _getNPfromHDF(hdf_path, file_type=file_type, lock=lock), coord
+
+
+def _getNPfromHDF(hdf_path, file_type='FDI', lock=None):
+    file_name = hdf_path.split(os.sep)[-1]
+    file_info = prase_filename(file_name)
+    if lock:
+        lock.acquire()
+    try:
+        ds = nc.Dataset(hdf_path)
+    except:
+        print(f'open {hdf_path} failed')
+    if lock:
+        lock.release()
+    if file_type == 'FDI':
+        f = ds.groups['Data']
+        np_data = np.zeros((15, 2748, 2748), dtype=np.float32)
+        for i in range(1, 16):
+            s_i = '0' + str(i) if i < 10 else str(i)
+            data = np.array(f[f'NOMChannel{s_i}'][:]).astype(np.float32)
+            data[data > 10000] = np.nan
+            np_data[i - 1] = data
+        # np_data = np_data[6:15]
+        in_out_idx = [6, 15]
+    elif file_type == 'QPE':
+        np_data = np.array(ds['Precipitation'][:]).astype(np.float32)
+        np_data = np_data[np.newaxis, :]
+        in_out_idx = [0, 1]
+    else:
+        np_data = None
+        in_out_idx = None
+        raise Exception(rf'file_type {file_type} err')
+    ds.close()
+    r = reference_cache.get_key_in_cache(file_info['position'])
+    if r is None:
+        print(f'get reference of {file_info["position"]}')
+        r = reference_cache.add_key(file_info['position'], get_reference(ld=file_info['position']))
+
+    np_data = _getNPfromHDF_worker(np_data, file_info['start'], r=r, ld=file_info['position'], to_file=False,
+                                   in_out_idx=in_out_idx)
+    return np_data
+
+
+def _getNPfromHDF_worker(read_np_data, current_date, data_dir=None, ld=None, r=None, to_file=True, in_out_idx=(6, 15)):
+    tmp_dir = 'make_temp' + str(randint(10000000, 99999999))
+    file = current_date.strftime("%H%M") + '.npy'
+    os.makedirs(tmp_dir, exist_ok=True)
+    save_path = f'{current_date.year}{current_date.month}{current_date.day}{file.replace(".npy", "")}'
+    np2tif(read_np_data, tmp_dir, save_path, print_log=False)
+    in_idx, out_idx = in_out_idx
+    np_data = np.zeros((out_idx - in_idx, x_range['height'], x_range['width']), dtype=np.float16)
+    for i in range(in_idx, out_idx):
+        out_path = f'{tmp_dir}/{save_path}-{i}-ctrl.tif'
+        _registration(f'{tmp_dir}/{save_path}-{i}.tif', out_path, ld, *r)
+        img = cv2.imread(out_path, -1)
+        if np.isnan(img).any():
+            shutil.rmtree(tmp_dir)
+            return None
+        np_data[i - in_idx] = img.astype(np.float16)
+    # raise Exception('manual stop')
+    shutil.rmtree(tmp_dir)
+    if to_file:
+        os.makedirs(f'{data_dir}/dataset/{current_date.year}/{current_date.month}/{current_date.day}', exist_ok=True)
+        np.save(
+            f'{data_dir}/dataset/{current_date.year}/{current_date.month}/{current_date.day}/{convert_file2idx(file)}',
+            np_data)
+    else:
+        return np_data
+
+
+# 解析FY文件的文件名
+def prase_filename(filename):
+    m_list = filename.replace('.HDF', '').split('_')
+    # FY4B-_AGRI--_N_DISK_1050E_L1-_FDI-_MULT_NOM_20250606171500_20250606172959_4000M_V0001.HDF
+    return {'satellite': m_list[0], 'sensor': m_list[1], 'area': m_list[3], 'position': round(float(m_list[4][:3]), 2),
+            'file_level': m_list[5], 'data_name': m_list[6], 'start': datetime.strptime(m_list[9], '%Y%m%d%H%M%S'),
+            'end': datetime.strptime(m_list[10], '%Y%m%d%H%M%S'), 'resolution': m_list[11]}
+
+
+def _registration(input_path, out_path, ld, spatial_reference, gcps_list):
+    """
+    基于python GDAL配准
+    :param input_path: 需要配准的栅格文件
+    :param out_path: 输出配准后的栅格文件位置
+    :param top_left: 左上角坐标
+    :param bottom_right: 右下角坐标
+    :param ik: 行空白分辨率
+    :param jk: 列空白分辨率
+    :return:
+    """
+    # 打开栅格文件
+    dataset = gdal.Open(input_path, gdal.GA_Update)
+    # 添加控制点
+    dataset.SetGCPs(gcps_list, spatial_reference.ExportToWkt())
+    # tps校正 重采样:最邻近法
+    gdal.Warp(out_path, dataset,
+              format='GTiff',
+              outputBounds=[ld + x_range['left'], x_range['bottom'], ld + x_range['right'], x_range['top']],
+              resampleAlg=gdal.GRIORA_NearestNeighbour,
+              width=x_range['width'],
+              height=x_range['height'],
+              tps=True,
+              dstSRS='EPSG:4326')
+
+
+def _prase_nc_worker(root_path, target_dir, file, lock=None):
+    ps = prase_filename(file)
+    file_name = ps["start"].strftime("%H%M")
+    hdf_path = f'{root_path}/{target_dir}/{ps["start"].year}/{ps["start"].month}/{ps["start"].day}/{file}'
+    save_path = f'{root_path}/npy/{ps["start"].year}/{ps["start"].month}/{ps["start"].day}'
+    if not os.path.exists(hdf_path):
+        tqdm.write(f'## {ps["start"].strftime("%Y%m%d %H:%M")} None ##')
+        return False
+    if os.path.exists(f'{save_path}/{convert_file2idx(file_name)}.npy'):
+        tqdm.write(f'## {ps["start"].strftime("%Y%m%d %H:%M")} already exist ##')
+        return True
+    try:
+        n_data = getNPfromHDF(hdf_path, lock=lock)
+        os.makedirs(save_path, exist_ok=True)
+        if n_data is None:
+            tqdm.write(f'## {ps["start"].strftime("%Y%m%d %H:%M")} None ##')
+            return False
+            # raise Exception(f'nan in cliped {hdf_path}')
+        np.save(f'{save_path}/{convert_file2idx(file_name)}', n_data)
+    except Exception as e:
+        # raise e
+        tqdm.write(f'## {ps["start"].strftime("%Y%m%d %H:%M")} err ##')
+        return False
+    tqdm.write(f'{ps["start"].strftime("%Y%m%d %H:%M")} down')
+    return True
+
+
+# 把FY4的HDF文件转为npy文件
+def make_fynp(root_path, target_dir, time_set):
+    mt = MultiTasks(40)
+    for current_date in time_set:
+        for file in os.listdir(
+                f'{root_path}/{target_dir}/{current_date.year}/{current_date.month}/{current_date.day}'):
+            if file.endswith('.HDF'):
+                mt.add_task(file, _prase_nc_worker, [root_path, target_dir, file, mt.thread_mutex])
+                # _prase_nc_worker(root_path, target_dir, file, None)
+    err_list = []
+    for key, flag in mt.execute_task().items():
+        if not flag:
+            err_list.append(key + '\n')
+    with open(f'err.log', 'w') as f:
+        f.writelines(err_list)
+    print(f'all done, {len(err_list)} in err.log')
+
+
+def get_ld(data):
+    if data < datetime(2024, 3, 1):
+        return 133
+    else:
+        return 105
+
+
+# 根据日期获取星下点位置
+def get_filename_by_date(file_date):
+    ld = get_ld(file_date)
+    filename = rf'FY4B-_AGRI--_N_DISK_{ld}E_L1-_FDI-_MULT_NOM_{file_date.strftime("%Y%m%d%H%M%S")}_{(file_date + timedelta(minutes=14, seconds=59)).strftime("%Y%m%d%H%M%S")}_4000M_V0001.HDF'
+    return filename
+
+
+if __name__ == '__main__':
+    # getNPfromHDF(rf'FY4B-_AGRI--_N_DISK_1330E_L2-_QPE-_MULT_NOM_20220702121500_20220702122959_4000M_V0001.NC',
+    #              file_type='QPE')
+    np_data = getNPfromHDF(
+        rf'FY4B-_AGRI--_N_DISK_1330E_L1-_FDI-_MULT_NOM_20221230030000_20221230031459_4000M_V0001.HDF')
+    np2tif(np_data, save_path='.', out_name='FY4B', left=133 - 60, top=60, x_res=0.05, y_res=0.05, dtype=np.float32)

jacksung/ai/utils/goes.py

@@ -0,0 +1,161 @@
+import os
+from datetime import datetime, timedelta
+import netCDF4 as nc
+import numpy as np
+from einops import rearrange, repeat
+from jacksung.utils.data_convert import np2tif, get_transform_from_lonlat_matrices, Coordinate
+import xarray as xr
+from pyresample import create_area_def, kd_tree
+from pyresample.geometry import AreaDefinition
+from jacksung.utils.time import Stopwatch
+from jacksung.utils.cache import Cache
+
+
+def get_resample_infos(hdf_path, lock=None, cache=None):
+    if lock:
+        lock.acquire()
+    ds = nc.Dataset(hdf_path)
+    if lock:
+        lock.release()
+    ld = float(ds['nominal_satellite_subpoint_lon'][:])
+    ld = round(ld, 2)
+    if cache:
+        cache_result = cache.get_key_in_cache(ld)
+        if cache_result is not None:
+            return cache_result
+    # 原始GEOS投影
+    goes_proj_str = ds['goes_imager_projection']
+    h = float(goes_proj_str.perspective_point_height)
+    x = ds['x'][:] * h  # 投影x坐标 (radians)
+    y = ds['y'][:] * h  # 投影y坐标 (radians)
+    # 计算投影范围 (根据x, y的边界)
+    half_pixel_width = (x[1] - x[0]) / 2.0
+    half_pixel_height = (y[1] - y[0]) / 2.0
+    area_extent = (x[0] - half_pixel_width, y[-1] - half_pixel_height,
+                   x[-1] + half_pixel_width, y[0] + half_pixel_height)
+    goes_area = AreaDefinition(
+        area_id='goes_fixed_grid', proj_id='goes_geos', description='GOES Fixed Grid',
+        projection={
+            'proj': 'geos',
+            'lon_0': goes_proj_str.longitude_of_projection_origin,
+            'h': goes_proj_str.perspective_point_height,
+            'x_0': 0,
+            'y_0': 0,
+            'a': goes_proj_str.semi_major_axis,
+            'b': goes_proj_str.semi_minor_axis,
+            'sweep': goes_proj_str.sweep_angle_axis
+        },
+
+        width=len(x), height=len(y), area_extent=area_extent)
+    left = ld - 60
+    right = ld + 60
+    target_areas = []
+    if left < -180:
+        # 跨越180度经线，分两部分重采样
+        # 左半部分
+        target_area_left = create_area_def(
+            area_id='wgs84_left', projection='EPSG:4326', area_extent=[left + 360, -60, 180, 60],
+            resolution=(0.05, 0.05), units='degrees')
+        target_areas.append(target_area_left)
+        # 右半部分
+        target_area_right = create_area_def(
+            area_id='wgs84_right', projection='EPSG:4326', area_extent=[-180, -60, right, 60],
+            resolution=(0.05, 0.05), units='degrees')
+        target_areas.append(target_area_right)
+    else:
+        target_area = create_area_def(
+            area_id='wgs84', projection='EPSG:4326', area_extent=[left, -60, right, 60],
+            resolution=(0.05, 0.05), units='degrees')
+        target_areas.append(target_area)
+    resample_infos = []
+    for target_area in target_areas:
+        # 使用最近邻法重采样，对于分类数据；对于连续数据，可以使用 ‘bilinear’
+        resample_infos.append(
+            kd_tree.get_neighbour_info(goes_area, target_area, radius_of_influence=5000, neighbours=1))
+    if cache:
+        cache.add_key(ld, resample_infos)
+    return resample_infos
+
+
+def getSingleChannelNPfromHDF(hdf_path, lock=None, return_coord=False, only_coord=False, resample_infos=None):
+    if lock:
+        lock.acquire()
+    ds = nc.Dataset(hdf_path)
+    if lock:
+        lock.release()
+    ld = float(ds['nominal_satellite_subpoint_lon'][:])
+    np_data = np.array(ds['Rad'][:]).astype(np.float32)
+    ld = round(ld, 2)
+    left = ld - 60
+    right = ld + 60
+    coord = Coordinate(left=left, bottom=-60, right=right, top=60, x_res=0.05, y_res=0.05)
+    if only_coord:
+        return coord
+    np_datas = []
+    if resample_infos is None:
+        resample_infos = get_resample_infos(hdf_path, lock=lock)
+    for info in resample_infos:
+        valid_input_index, valid_output_index, index_array, distance_array = info
+        # 使用最近邻法重采样，对于分类数据；对于连续数据，可以使用 ‘bilinear’
+        results = kd_tree.get_sample_from_neighbour_info(
+            'nn', output_shape=(coord.h, int(len(info[1]) / 2400)), data=np_data, valid_input_index=valid_input_index,
+            valid_output_index=valid_output_index, index_array=index_array, fill_value=np.nan)
+        np_datas.append(results)
+    # 合并两部分数据
+    np_data = np.concatenate(np_datas, axis=1)
+    if return_coord:
+        return np_data, coord
+    else:
+        return np_data
+
+
+def get_filename_by_date_from_dir(dir_path, date, satellite='G18'):
+    file_lists = {}
+    for file in os.listdir(dir_path):
+        if not file.endswith('.nc'):
+            continue
+        splits = file.split('_')
+        year = int(splits[3][1:5])
+        doy = int(splits[3][5:8])
+        hour = int(splits[3][8:10])
+        minute = int(splits[3][10:12])
+        file_date = datetime(year=year, month=1, day=1) + timedelta(days=doy - 1, hours=hour, minutes=minute)
+        if date == file_date and splits[2] == satellite:
+            file_lists[int(splits[1].split('-')[3][3:])] = file
+    return file_lists
+
+
+def getNPfromDir(dir_path, date, satellite='G18', lock=None, return_coord=False, infos=None, cache=None):
+    np_data = None
+    coord = None
+    data_channel_count = 0
+    files = get_filename_by_date_from_dir(dir_path, date, satellite)
+    for channel, file in files.items():
+        if infos is None:
+            infos = get_resample_infos(os.path.join(dir_path, file), lock=lock, cache=cache)
+        channel_data, coord = getSingleChannelNPfromHDF(
+            os.path.join(dir_path, file), return_coord=True, resample_infos=infos)
+        if channel_data is None:
+            raise Exception(f"文件{file}，通道 {channel} 数据获取失败")
+        if np_data is None:
+            np_data = np.full([9] + list(channel_data.shape), np.nan)
+        np_data[channel - 8] = channel_data
+        data_channel_count += 1
+    if data_channel_count < 9:
+        raise Exception(
+            f"文件夹{dir_path}中，卫星 {satellite} 在时间 {date} 的数据通道不完整，仅获取到 {data_channel_count} 个通道")
+    if return_coord:
+        return np_data, coord
+    else:
+        return np_data
+
+
+if __name__ == '__main__':
+    np_data = getNPfromDir(rf'D:\python_Project\Huayu_Global\file_download\2022\12\30',
+                           datetime(year=2022, month=12, day=30, hour=3, minute=0), satellite='G18')
+    np2tif(np_data, save_path='test_goes', out_name='GOES18',
+           left=-137 - 60, top=60, x_res=0.05, y_res=0.05, dtype=np.float32)
+    np_data = getNPfromDir(rf'D:\python_Project\Huayu_Global\file_download\2022\12\30',
+                           datetime(year=2022, month=12, day=30, hour=3, minute=0), satellite='G16')
+    np2tif(np_data, save_path='test_goes', out_name='GOES16',
+           left=-75.2 - 60, top=60, x_res=0.05, y_res=0.05, dtype=np.float32)

jacksung/ai/utils/gsmap.py

@@ -0,0 +1,24 @@
+import netCDF4 as nc
+import numpy as np
+from einops import rearrange, repeat
+from jacksung.utils.data_convert import np2tif, get_transform_from_lonlat_matrices
+
+
+def getNPfromHDF(hdf_path, lock=None, save_file=True):
+    if lock:
+        lock.acquire()
+    ds = nc.Dataset(hdf_path)
+    if lock:
+        lock.release()
+    np_data = np.array(ds['hourlyPrecipRateGC'][:]).astype(np.float32)[0]
+    ds.close()
+    np_data[np_data < 0] = np.nan
+    np_data = np_data[::-1, :]
+    if save_file:
+        np2tif(np_data, save_path='np2tif_dir', out_name='gsmap', dtype='float32',
+               left=-180, top=90, x_res=0.1, y_res=0.1)
+    return np_data
+
+
+if __name__ == '__main__':
+    getNPfromHDF(rf'D:\python_Project\Huayu_Global\file_download\gsmap_now_rain.20220702.0300.nc')

jacksung/ai/utils/imerg.py

@@ -0,0 +1,159 @@
+from jacksung.utils.data_convert import nc2np, np2tif
+import numpy as np
+import netCDF4 as nc
+from einops import rearrange
+import os
+import shutil
+import requests
+import time
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.common.by import By
+import platform
+
+
+class Downloader:
+    def __init__(self, download_file_path, username,passwd,save_path=None):
+        self.download_file_path = download_file_path
+        if save_path is not None:
+            self.save_path = save_path
+        else:
+            if platform.system().lower() == 'windows':
+                self.save_path = 'D:\\imerg'
+            else:
+                self.save_path = '/mnt/data1/szj/imerg'
+        self.username=username
+        self.passwd=passwd
+
+    def make_driver(self, url, is_headless=False, tmp_path=None, download_dir=None):
+        options = webdriver.ChromeOptions()
+        if tmp_path:
+            options.add_argument("crash-dumps-dir=" + tmp_path)
+        options.add_argument("--no-sandbox")
+        # options.add_argument("--auto-open-devtools-for-tabs")
+        options.add_argument('--disable-dev-shm-usage')
+        options.add_argument("--disable-web-security")  # 禁用Web安全
+        options.add_argument("--allow-running-insecure-content")  # 允许不安全的内容
+        options.add_argument('--user-agent=Mozilla/5.0')
+        options.add_argument('--ignore-ssl-errors=yes')
+        options.add_argument('--allow-insecure-localhost')
+        options.add_argument('--ignore-certificate-errors')
+        options.add_argument("--lang=zh-CN")  # 将语言设置为简体中文，英文为en-US
+        options.add_experimental_option("detach", True)
+        if download_dir:
+            options.add_experimental_option("prefs", {
+                "download.default_directory": download_dir,
+                # "download.prompt_for_download": False,
+                # "download.directory_upgrade": True,
+                # "safebrowsing.enabled": True
+            })
+        options.set_capability('pageLoadStrategy', 'none')
+        options.set_capability("unhandledPromptBehavior", "accept")
+        if is_headless:
+            options.add_argument('--headless')  # 浏览器隐式启动
+        # driver_path = os.path.expanduser("~/chrome/chromedriver.exe")
+        print('driver is going to init!')
+        if platform.system().lower() == 'windows':
+            driver_path = None
+            # driver_path = os.path.expanduser("~/chrome/chromedriver.exe")
+        else:
+            driver_path = os.path.expanduser("~/chrome/chromedriver")
+        driver = webdriver.Chrome(service=Service(driver_path) if driver_path else None, options=options)
+        # driver.maximize_window()
+        driver.implicitly_wait(10)
+        driver.set_page_load_timeout(10)
+        print('driver is inited!')
+        print(f'请求地址：{url}')
+        driver.get(url)
+        return driver
+
+    # 进度条模块
+    def progressbar(self, url, path):
+        if not os.path.exists(path):  # 看是否有该文件夹，没有则创建文件夹
+            os.mkdir(path)
+        start = time.time()  # 下载开始时间
+        response = requests.get(url, stream=True)
+        # https://data.gesdisc.earthdata.nasa.gov/data/GPM_L3/GPM_3IMERGHH.07/2023/001/3B-HHR.MS.MRG.3IMERG.20230101-S000000-E002959.0000.V07B.HDF5
+        name = url. \
+            replace(
+            'https://data.gesdisc.earthdata.nasa.gov/data/GPM_L3/GPM_3IMERGHH.07/2023/001/3B-HHR.MS.MRG.3IMERG.',
+            '').replace('.0000.V07B.HDF5', '.HDF5')
+        size = 0  # 初始化已下载大小
+        chunk_size = 1024  # 每次下载的数据大小
+        content_size = int(response.headers['content-length'])  # 下载文件总大小
+        if response.status_code == 200:  # 判断是否响应成功
+            print('Start download,[File size]:{size:.2f} MB'.format(
+                size=content_size / chunk_size / 1024))  # 开始下载，显示下载文件大小
+            filepath = rf'{path}\{name}'  # 设置图片name，注：必须加上扩展名
+            with open(filepath, 'wb') as file:  # 显示进度条
+                for data in response.iter_content(chunk_size=chunk_size):
+                    file.write(data)
+                    size += len(data)
+                    print('\r' + '[下载进度]:%s%.2f%%' % (
+                        '>' * int(size * 50 / content_size), float(size / content_size * 100)), end=' ')
+        end = time.time()  # 下载结束时间
+        print('Download completed!,times: %.2f秒' % (end - start))  # 输出下载用时时间
+
+    def simulate(self, url, driver, path):
+        start = time.time()  # 下载开始时间
+        driver.get(url)
+        while not os.path.exists(path):
+            time.sleep(1)
+        # 3B-HHR-E.MS.MRG.3IMERG.20230101-S033000-E035959.0210.V07B.HDF5
+        names = path.split('/')[-1].split('.')
+        date = names[4].split('-')[0]
+        move_path = path.replace('.'.join(names[:4]), f'{date}{os.path.sep}{".".join(names[:4])}')
+        if not os.path.exists(os.path.dirname(move_path)):
+            os.makedirs(os.path.dirname(move_path))
+        shutil.move(path, move_path)
+        end = time.time()  # 下载结束时间
+        print('Download completed!,times: %.2f秒' % (end - start))  # 输出下载用时时间
+
+    def start_download(self):
+        download_file_path = self.download_file_path
+        print(f'开始下载:{download_file_path}')
+        f = open(download_file_path, 'r')
+        save_f = open('downloaded.txt', 'r')
+
+        downloaded_list = save_f.readlines()
+        downloaded_list = [u.replace('\n', '') for u in downloaded_list if u.count('.pdf') == 0]
+        save_f.close()
+        with open('downloaded.txt', 'a') as save_f_w:
+            driver = self.make_driver('https://urs.earthdata.nasa.gov', download_dir=self.save_path, is_headless=True)
+            time.sleep(10)
+            username = driver.find_element(By.ID, 'username')
+            passwd = driver.find_element(By.ID, 'password')
+            username.send_keys(self.username)
+            passwd.send_keys(self.passwd)
+            driver.find_element(By.NAME, 'commit').click()
+            time.sleep(5)
+            for line in f.readlines():
+                url = line.strip()
+                if line.replace('\n', '') not in downloaded_list:
+                    print(url)
+                    file_path = self.save_path + os.sep + url.split('/')[-1]
+                    try:
+                        self.simulate(url, driver, file_path)
+                        save_f_w.write(url + '\n')
+                    except Exception as e:
+                        print(f'{url} download failed')
+                else:
+                    print(f'{url} already downloaded')
+            driver.close()
+
+
+def getNPfromHDF(hdf_path, lock=None, save_file=True):
+    if lock:
+        lock.acquire()
+    ds = nc.Dataset(hdf_path)
+    if lock:
+        lock.release()
+    np_data = np.array(ds.groups['Grid']['precipitation'][:]).astype(np.float32)
+    np_data = np_data[np.newaxis, :]
+    ds.close()
+    np_data = rearrange(np_data[0][0], 'w h->h w')[::-1, :]
+    np_data[np_data < 0] = 0
+    if save_file:
+        np2tif(np_data, save_path='np2tif_dir', left=-180, top=90, x_res=0.1, y_res=0.1, out_name='IMERG',
+               dtype='float32')
+    return np_data