oafuncs 0.0.70__py2.py3-none-any.whl → 0.0.72__py2.py3-none-any.whl

oafuncs/oa_down/hycom_3hourly.py

@@ -4,7 +4,7 @@
 Author: Liu Kun && 16031215@qq.com
 Date: 2024-11-01 10:31:09
 LastEditors: Liu Kun && 16031215@qq.com
-LastEditTime: 2024-12-01 10:57:04
+LastEditTime: 2024-12-01 11:45:18
 FilePath: \\Python\\My_Funcs\\OAFuncs\\oafuncs\\oa_down\\hycom_3hourly.py
 Description:
 EditPlatform: vscode
@@ -602,6 +602,31 @@ def dlownload_file(target_url, store_path, file_name, check=False):
     request_times = 0
     filename = Path(store_path) / file_name
 
+    def calculate_wait_time(time_str):
+        import re
+
+        # 定义正则表达式，匹配YYYYMMDDHH格式的时间
+        time_pattern = r'\d{10}'
+
+        # 定义两个字符串
+        # str1 = 'HYCOM_water_u_2018010100_2018010112.nc'
+        # str2 = 'HYCOM_water_u_2018010100.nc'
+
+        # 使用正则表达式查找时间
+        times_in_str = re.findall(time_pattern, time_str)
+
+        # 计算每个字符串中的时间数量
+        num_times_str = len(times_in_str)
+
+        if num_times_str > 1:
+            delta_t = datetime.datetime.strptime(times_in_str[1], '%Y%m%d%H') - datetime.datetime.strptime(times_in_str[0], '%Y%m%d%H')
+            delta_t = delta_t.total_seconds() / 3600
+            delta_t = delta_t / 3 + 1
+
+        return int(delta_t*15)
+    
+    max_timeout = calculate_wait_time(file_name)
+
     if check:
         if check_existing_file(filename):
             return
@@ -620,7 +645,7 @@ def dlownload_file(target_url, store_path, file_name, check=False):
         # 尝试下载文件
         try:
             headers = {'User-Agent': get_ua()}
-            response = s.get(target_url, headers=headers, timeout=random.randint(5, 15))
+            response = s.get(target_url, headers=headers, timeout=random.randint(5, max_timeout))
             response.raise_for_status()  # 如果请求返回的不是200，将抛出HTTPError异常
 
             # 保存文件
@@ -730,6 +755,8 @@ def prepare_url_to_download(var, lon_min=0, lon_max=359.92, lat_min=-80, lat_max
             var = var[0]
             submit_url = get_submit_url_var(var, depth, level_num, lon_min, lon_max, lat_min, lat_max, dataset_name, version_name, download_time, download_time_end)
             file_name = f"HYCOM_{variable_info[var]['var_name']}_{download_time}.nc"
+            if download_time_end is not None:
+                file_name = f"HYCOM_{variable_info[var]['var_name']}_{download_time}_{download_time_end}.nc"
             dlownload_file(submit_url, store_path, file_name, check)
         else:
             varlist = [_ for _ in var]
@@ -752,6 +779,8 @@ def prepare_url_to_download(var, lon_min=0, lon_max=359.92, lat_min=-80, lat_max
                     submit_url = submit_url.replace(old_str, new_str)
                     # file_name = f'HYCOM_{'-'.join([variable_info[v]["var_name"] for v in current_group])}_{download_time}.nc'
                     file_name = f'HYCOM_{key}_{download_time}.nc'
+                    if download_time_end is not None:
+                        file_name = f'HYCOM_{key}_{download_time}_{download_time_end}.nc'
                 dlownload_file(submit_url, store_path, file_name, check)
 
 
@@ -779,7 +808,7 @@ def download_task(var, time_str, time_str_end, lon_min, lon_max, lat_min, lat_ma
     prepare_url_to_download(var, lon_min, lon_max, lat_min, lat_max, time_str, time_str_end, depth, level, store_path, dataset_name, version_name, check)
 
 
-def download_hourly_func(var, time_s, time_e=None, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90,  depth=None, level=None, store_path=None, dataset_name=None, version_name=None, num_workers=None, check=False, ftimes=1):
+def download_hourly_func(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90,  depth=None, level=None, store_path=None, dataset_name=None, version_name=None, num_workers=None, check=False, ftimes=1):
     '''
     Description:
     Download the data of single time or a series of time
@@ -802,15 +831,7 @@ def download_hourly_func(var, time_s, time_e=None, lon_min=0, lon_max=359.92, la
     Returns:
     None
     '''
-    ymdh_time_s = str(time_s)
-    if len(ymdh_time_s) == 8:
-        ymdh_time_s += '00'
-    if time_e is None:
-        ymdh_time_e = ymdh_time_s[:]
-    else:
-        ymdh_time_e = str(time_e)
-        if len(ymdh_time_e) == 8:
-            ymdh_time_e += '21'
+    ymdh_time_s, ymdh_time_e = str(time_s), str(time_e)
     if ymdh_time_s == ymdh_time_e:
         prepare_url_to_download(var, lon_min, lon_max, lat_min, lat_max, ymdh_time_s, None, depth, level, store_path, dataset_name, version_name)
     elif int(ymdh_time_s) < int(ymdh_time_e):
@@ -826,10 +847,6 @@ def download_hourly_func(var, time_s, time_e=None, lon_min=0, lon_max=359.92, la
                         progress.update(task, advance=1, description=f'[cyan]Downloading... {i+1}/{len(time_list)}')
                 else:
                     # 并行方式
-                    if num_workers > 10:
-                        print('The number of workers is too large!')
-                        print('In order to avoid the server being blocked, the number of workers is set to 10')
-                        num_workers = 10
                     with ThreadPoolExecutor(max_workers=num_workers) as executor:
                         futures = [executor.submit(download_task, var, time_str, None, lon_min, lon_max, lat_min, lat_max, depth, level, store_path, dataset_name, version_name, check) for time_str in time_list]
                         for i, future in enumerate(futures):
@@ -846,16 +863,12 @@ def download_hourly_func(var, time_s, time_e=None, lon_min=0, lon_max=359.92, la
                         progress.update(task, advance=1, description=f'[cyan]Downloading... {i+1}/{total_num}')
                 else:
                     # 并行方式
-                    if num_workers > 10:
-                        print('The number of workers is too large!')
-                        print('In order to avoid the server being blocked, the number of workers is set to 10')
-                        num_workers = 10
                     with ThreadPoolExecutor(max_workers=num_workers) as executor:
                         futures = [executor.submit(download_task, var, new_time_list[i], time_list[int(min(len(time_list), int(i*ftimes+ftimes-1)))], lon_min, lon_max, lat_min, lat_max, depth, level, store_path, dataset_name, version_name, check) for i in range(total_num)]
                         for i, future in enumerate(futures):
                             future.add_done_callback(lambda _: progress.update(task, advance=1, description=f'[cyan]Downloading... {i+1}/{total_num}'))
     else:
-        print('Please ensure the time_s is less than the time_e')
+        print('Please ensure the time_s is no more than time_e')
 
 
 def download(var, time_s, time_e=None, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90,  depth=None, level=None, store_path=None, dataset_name=None, version_name=None, num_workers=None, check=False, ftimes=1):
@@ -923,6 +936,19 @@ def download(var, time_s, time_e=None, lon_min=0, lon_max=359.92, lat_min=-80, l
     else:
         os.makedirs(str(store_path), exist_ok=True)
 
+    if num_workers is not None:
+        num_workers = max(min(num_workers, 10), 1)
+
+    time_s = str(time_s)
+    if len(time_s) == 8:
+        time_s += '00'
+    if time_e is None:
+        time_e = time_s[:]
+    else:
+        time_e = str(time_e)
+        if len(time_e) == 8:
+            time_e += '21'
+
     download_hourly_func(var, time_s, time_e, lon_min, lon_max, lat_min, lat_max, depth, level, store_path, dataset_name, version_name, num_workers, check, ftimes)
 
 

{oafuncs-0.0.70.dist-info → oafuncs-0.0.72.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: oafuncs
-Version: 0.0.70
+Version: 0.0.72
 Summary: My short description for my project.
 Home-page: https://github.com/Industry-Pays/OAFuncs
 Author: Kun Liu

{oafuncs-0.0.70.dist-info → oafuncs-0.0.72.dist-info}/RECORD

@@ -7,7 +7,7 @@ oafuncs/oa_help.py,sha256=ppNktmtNzs15R20MD1bM7yImlTQ_ngMwvoIglePOKXA,1000
 oafuncs/oa_nc.py,sha256=ALAYfqDy5lbUNJsTU29j6ZWkM4wgqQU3p2Fxn5pkvsQ,12102
 oafuncs/oa_python.py,sha256=XPTP3o7zTFzfJR_YhsKfQksa3bSYwXsne9YxlJplCEA,3994
 oafuncs/oa_down/__init__.py,sha256=a6rgxHQi8spvlI-TaVEqnrDNhYsKm5_IQf7ckAZ8U4w,603
-oafuncs/oa_down/hycom_3hourly.py,sha256=ATKA-p-gLTUR4MInzN5yql1sCKi8C9nvQ_yXqjMwzQI,56655
+oafuncs/oa_down/hycom_3hourly.py,sha256=aVyZk8hs5tPH0DYvhKQIFwor2RkJHZGIXnwGKr2o_iY,57394
 oafuncs/oa_down/literature.py,sha256=dT3-7-beEzQ9mTP8LNV9Gf3q5Z1Pqqjc6FOS010HZeQ,17833
 oafuncs/oa_down/refs_pdf.py,sha256=lgGKO2gQ0hRjuaBYOLeBgksJB_KXYpFcXMDwMQeYVkI,18719
 oafuncs/oa_sign/__init__.py,sha256=QKqTFrJDFK40C5uvk48GlRRbGFzO40rgkYwu6dYxatM,563
@@ -16,8 +16,8 @@ oafuncs/oa_sign/ocean.py,sha256=xrW-rWD7xBWsB5PuCyEwQ1Q_RDKq2KCLz-LOONHgldU,5932
 oafuncs/oa_sign/scientific.py,sha256=a4JxOBgm9vzNZKpJ_GQIQf7cokkraV5nh23HGbmTYKw,5064
 oafuncs/oa_tool/__init__.py,sha256=IKOlqpWlb4cMDCtq2VKR_RTxQHDNqR_vfqqsOsp_lKQ,466
 oafuncs/oa_tool/email.py,sha256=4lJxV_KUzhxgLYfVwYTqp0qxRugD7fvsZkXDe5WkUKo,3052
-oafuncs-0.0.70.dist-info/LICENSE.txt,sha256=rMtLpVg8sKiSlwClfR9w_Dd_5WubTQgoOzE2PDFxzs4,1074
-oafuncs-0.0.70.dist-info/METADATA,sha256=6u5-PblWLD9YmMabXD7zKarROPKOxaoSTSHmwGXKqQ4,22531
-oafuncs-0.0.70.dist-info/WHEEL,sha256=pxeNX5JdtCe58PUSYP9upmc7jdRPgvT0Gm9kb1SHlVw,109
-oafuncs-0.0.70.dist-info/top_level.txt,sha256=bgC35QkXbN4EmPHEveg_xGIZ5i9NNPYWqtJqaKqTPsQ,8
-oafuncs-0.0.70.dist-info/RECORD,,
+oafuncs-0.0.72.dist-info/LICENSE.txt,sha256=rMtLpVg8sKiSlwClfR9w_Dd_5WubTQgoOzE2PDFxzs4,1074
+oafuncs-0.0.72.dist-info/METADATA,sha256=yMP6PaG4mCkJt6Jcm9o5M7kgcEON-tBkwVGK9N9Rkqw,22531
+oafuncs-0.0.72.dist-info/WHEEL,sha256=pxeNX5JdtCe58PUSYP9upmc7jdRPgvT0Gm9kb1SHlVw,109
+oafuncs-0.0.72.dist-info/top_level.txt,sha256=bgC35QkXbN4EmPHEveg_xGIZ5i9NNPYWqtJqaKqTPsQ,8
+oafuncs-0.0.72.dist-info/RECORD,,