oafuncs 0.0.98.4__py3-none-any.whl → 0.0.98.6__py3-none-any.whl

oafuncs/oa_down/{hycom_3hourly_20250416.py → hycom_3hourly_proxy.py}

@@ -2,9 +2,9 @@
 # coding=utf-8
 """
 Author: Liu Kun && 16031215@qq.com
-Date: 2025-04-16 11:36:15
+Date: 2025-04-17 15:16:02
 LastEditors: Liu Kun && 16031215@qq.com
-LastEditTime: 2025-04-16 11:36:16
+LastEditTime: 2025-04-17 15:16:04
 FilePath: \\Python\\My_Funcs\\OAFuncs\\oafuncs\\oa_down\\hycom_3hourly copy.py
 Description:
 EditPlatform: vscode
@@ -15,7 +15,9 @@ Python Version: 3.12
 
 
 
+import asyncio
 import datetime
+import logging
 import os
 import random
 import re
@@ -25,11 +27,11 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
 from pathlib import Path
 from threading import Lock
 
+import httpx
 import matplotlib.pyplot as plt
 import netCDF4 as nc
 import numpy as np
 import pandas as pd
-import requests
 import xarray as xr
 from rich import print
 from rich.progress import Progress
@@ -40,6 +42,9 @@ from oafuncs.oa_file import file_size
 from oafuncs.oa_nc import check as check_nc
 from oafuncs.oa_nc import modify as modify_nc
 
+logging.getLogger("httpx").setLevel(logging.WARNING)  # 关闭 httpx 的 INFO 日志，只显示 WARNING 及以上
+
+
 warnings.filterwarnings("ignore", category=RuntimeWarning, message="Engine '.*' loading failed:.*")
 
 __all__ = ["draw_time_range", "download"]
@@ -416,13 +421,13 @@ def _check_time_in_dataset_and_version(time_input, time_end=None):
                     trange_list.append(f"{time_s}-{time_e}")
                     have_data = True
 
-    # 输出结果
-    if match_time is None:
-        print(f"[bold red]{time_input_str} is in the following dataset and version:")
     if have_data:
         if match_time is None:
+            print(f"[bold red]Time {time_input_str} included in:")
+            dv_num = 1
             for d, v, trange in zip(d_list, v_list, trange_list):
-                print(f"[bold blue]{d} {v} {trange}")
+                print(f"{dv_num} -> [bold blue]{d} - {v} : {trange}")
+                dv_num += 1
         if is_single_time:
             return True
         else:
@@ -434,7 +439,7 @@ def _check_time_in_dataset_and_version(time_input, time_end=None):
                 print(f"[bold red]{time_start} to {time_end} is in different datasets or versions, so you can't download them together")
                 return False
     else:
-        print(f"[bold red]{time_input_str} is not in any dataset and version")
+        print(f"[bold red]Time {time_input_str} has no data")
         return False
 
 
@@ -509,7 +514,8 @@ def _direct_choose_dataset_and_version(time_input, time_end=None):
 
     if dataset_name_out is not None and version_name_out is not None:
         if match_time is None:
-            print(f"[bold purple]dataset: {dataset_name_out}, version: {version_name_out} is chosen")
+            # print(f"[bold purple]dataset: {dataset_name_out}, version: {version_name_out} is chosen")
+            print(f"[bold purple]Chosen dataset: {dataset_name_out} - {version_name_out}")
 
     # 如果没有找到匹配的数据集和版本，会返回 None
     return dataset_name_out, version_name_out
@@ -664,117 +670,145 @@ def _correct_time(nc_file):
     modify_nc(nc_file, "time", None, time_difference)
 
 
+class _HycomDownloader:
+    def __init__(self, tasks, delay_range=(3, 6), timeout_factor=120, max_var_count=5, max_retries=3):
+        """
+        :param tasks: List of (url, save_path)
+        """
+        self.tasks = tasks
+        self.delay_range = delay_range
+        self.timeout_factor = timeout_factor
+        self.max_var_count = max_var_count
+        self.max_retries = max_retries
+        self.count = {"success": 0, "fail": 0}
+
+    def user_agent(self):
+        return get_ua()
+
+    async def _download_one(self, url, save_path):
+        file_name = os.path.basename(save_path)
+        headers = {"User-Agent": self.user_agent()}
+        var_count = min(max(url.count("var="), 1), self.max_var_count)
+        timeout_max = self.timeout_factor * var_count
+        timeout = random.randint(timeout_max // 2, timeout_max)
+
+        retry = 0
+        while retry <= self.max_retries:
+            if proxy_txt_path:
+                from .read_proxy import get_valid_proxy
+
+                proxy_one = get_valid_proxy(proxy_txt_path)
+                if proxy_one:
+                    if proxy_one.startswith("http://"):
+                        proxy_one = proxy_one[7:]
+                    elif proxy_one.startswith("https://"):
+                        proxy_one = proxy_one[8:]
+            else:
+                proxy_one = None
+            if proxy_one:
+                proxy = f"http://{proxy_one}"
+                mounts = {
+                    "http://": httpx.AsyncHTTPTransport(proxy=proxy),
+                    "https://": httpx.AsyncHTTPTransport(proxy=proxy),
+                }
+            else:
+                proxy = None
+                mounts = None
+            try:
+                await asyncio.sleep(random.uniform(*self.delay_range))
+                start = datetime.datetime.now()
+
+                async with httpx.AsyncClient(
+                    timeout=httpx.Timeout(timeout),
+                    limits=httpx.Limits(max_connections=2, max_keepalive_connections=2),
+                    transport=httpx.AsyncHTTPTransport(retries=2),
+                    # proxy=proxy,
+                    mounts=mounts,
+                ) as client:
+                    print(f"[bold #f0f6d0]Requesting {file_name} (Attempt {retry + 1}) ...")
+                    response = await client.get(url, headers=headers, follow_redirects=True)
+                    response.raise_for_status()
+                    if not response.content:
+                        raise ValueError("Empty response received")
+
+                    print(f"[bold #96cbd7]Downloading {file_name} ...")
+                    with open(save_path, "wb") as f:
+                        async for chunk in response.aiter_bytes(32 * 1024):
+                            f.write(chunk)
+
+                    elapsed = datetime.datetime.now() - start
+                    print(f"[#3dfc40]File [bold #dfff73]{file_name} [#3dfc40]downloaded, Time: [#39cbdd]{elapsed}")
+                    self.count["success"] += 1
+                    count_dict["success"] += 1
+                    return
+
+            except Exception as e:
+                print(f"[bold red]Failed ({type(e).__name__}): {e}")
+                if retry < self.max_retries:
+                    backoff = 2**retry
+                    print(f"[yellow]Retrying in {backoff:.1f}s ...")
+                    await asyncio.sleep(backoff)
+                    retry += 1
+                else:
+                    print(f"[red]Giving up on {file_name}")
+                    self.count["fail"] += 1
+                    count_dict["fail"] += 1
+                    return
+
+    async def run(self):
+        print(f"📥 Starting download of {len(self.tasks)} files ...")
+        for url, save_path in self.tasks:
+            await self._download_one(url, save_path)
+
+        print("\n✅ All tasks completed.")
+        print(f"✔️  Success: {self.count['success']} | ❌ Fail: {self.count['fail']}")
+
+
 def _download_file(target_url, store_path, file_name, cover=False):
-    fname = Path(store_path) / file_name
+    save_path = Path(store_path) / file_name
     file_name_split = file_name.split("_")
     file_name_split = file_name_split[:-1]
     same_file = "_".join(file_name_split) + "*nc"
 
     if match_time is not None:
-        if check_nc(fname, print_messages=False):
-            if not _check_ftime(fname, if_print=True):
+        if check_nc(save_path, print_messages=False):
+            if not _check_ftime(save_path, if_print=True):
                 if match_time:
-                    _correct_time(fname)
+                    _correct_time(save_path)
                     count_dict["skip"] += 1
                 else:
-                    _clear_existing_file(fname)
+                    _clear_existing_file(save_path)
                     count_dict["no_data"] += 1
             else:
                 count_dict["skip"] += 1
                 print(f"[bold green]{file_name} is correct")
         return
 
-    if not cover and os.path.exists(fname):
-        print(f"[bold #FFA54F]{fname} exists, skipping ...")
+    if not cover and os.path.exists(save_path):
+        print(f"[bold #FFA54F]{save_path} exists, skipping ...")
         count_dict["skip"] += 1
         return
 
     if same_file not in fsize_dict.keys():
-        check_nc(fname, delete_if_invalid=True, print_messages=False)
+        check_nc(save_path, delete_if_invalid=True, print_messages=False)
 
-    get_mean_size = _get_mean_size_move(same_file, fname)
+    get_mean_size = _get_mean_size_move(same_file, save_path)
 
-    if _check_existing_file(fname, get_mean_size):
+    if _check_existing_file(save_path, get_mean_size):
         count_dict["skip"] += 1
         return
 
-    _clear_existing_file(fname)
+    _clear_existing_file(save_path)
 
     if not use_idm:
-        print(f"[bold #f0f6d0]Requesting {file_name} ...")
-        s = requests.Session()
-        download_success = False
-        request_times = 0
-
-        def calculate_wait_time(time_str, target_url):
-            time_pattern = r"\d{10}"
-            times_in_str = re.findall(time_pattern, time_str)
-            num_times_str = len(times_in_str)
-
-            if num_times_str > 1:
-                delta_t = datetime.datetime.strptime(times_in_str[1], "%Y%m%d%H") - datetime.datetime.strptime(times_in_str[0], "%Y%m%d%H")
-                delta_t = delta_t.total_seconds() / 3600
-                delta_t = delta_t / 3 + 1
-            else:
-                delta_t = 1
-            num_var = int(target_url.count("var="))
-            if num_var <= 0:
-                num_var = 1
-            return int(delta_t * 5 * 60 * num_var)
-
-        max_timeout = calculate_wait_time(file_name, target_url)
-        print(f"[bold #912dbc]Max timeout: {max_timeout} seconds")
-
-        download_time_s = datetime.datetime.now()
-        order_list = ["1st", "2nd", "3rd", "4th", "5th", "6th", "7th", "8th", "9th", "10th"]
-        while not download_success:
-            if request_times >= 10:
-                print(f"[bold #ffe5c0]Download failed after {request_times} times\nYou can skip it and try again later")
-                count_dict["fail"] += 1
-                break
-            if request_times > 0:
-                print(f"[bold #ffe5c0]Retrying the {order_list[request_times - 1]} time...")
-            try:
-                referer_center = target_url.split("?")[0].split("ncss/")[-1]
-                headers = {
-                    "User-Agent": get_ua(),  # 后面几项可以不加，依旧能下载
-                    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
-                    "Accept-Encoding": "gzip, deflate, br, zstd",
-                    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
-                    "Referer": rf"https://ncss.hycom.org/thredds/ncss/grid/{referer_center}/dataset.html",
-                }
-                response = s.get(target_url, headers=headers, stream=True, timeout=random.randint(5, max_timeout))
-                response.raise_for_status()
-                with open(fname, "wb") as f:
-                    print(f"[bold #96cbd7]Downloading {file_name} ...")
-                    for chunk in response.iter_content(chunk_size=1024):
-                        if chunk:
-                            f.write(chunk)
-
-                f.close()
-
-                if os.path.exists(fname):
-                    download_success = True
-                    download_time_e = datetime.datetime.now()
-                    download_delta = download_time_e - download_time_s
-                    print(f"[#3dfc40]File [bold #dfff73]{fname} [#3dfc40]has been downloaded successfully, Time: [#39cbdd]{download_delta}")
-                    count_dict["success"] += 1
-
-            except requests.exceptions.HTTPError as errh:
-                print(f"Http Error: {errh}")
-            except requests.exceptions.ConnectionError as errc:
-                print(f"Error Connecting: {errc}")
-            except requests.exceptions.Timeout as errt:
-                print(f"Timeout Error: {errt}")
-            except requests.exceptions.RequestException as err:
-                print(f"OOps: Something Else: {err}")
-
-            time.sleep(3)
-            request_times += 1
+        python_downloader = _HycomDownloader([(target_url, save_path)])
+        asyncio.run(python_downloader.run())
+        time.sleep(3 + random.uniform(0, 10))
     else:
         idm_downloader(target_url, store_path, file_name, given_idm_engine)
-        idm_download_list.append(fname)
-        print(f"[bold #3dfc40]File [bold #dfff73]{fname} [#3dfc40]has been submit to IDM for downloading")
+        idm_download_list.append(save_path)
+        # print(f"[bold #3dfc40]File [bold #dfff73]{save_path} [#3dfc40]has been submit to IDM for downloading")
+        time.sleep(3 + random.uniform(0, 10))
 
 
 def _check_hour_is_valid(ymdh_str):
@@ -962,6 +996,7 @@ def download(
     idm_path=None,
     validate_time=None,
     interval_hours=3,
+    proxy_txt=None,
 ):
     """
     Download data for a single time or a series of times.
@@ -1095,6 +1130,9 @@ def download(
     global mark_len
     mark_len = 100
 
+    global proxy_txt_path
+    proxy_txt_path = proxy_txt
+
     if validate_time is not None:
         workers = 1
         print("*" * mark_len)
@@ -1167,20 +1205,21 @@ if __name__ == "__main__":
 
     options = {
         "variables": var_list,
-        "start_time": "2025010300",
-        "end_time": "2025010309",
-        "output_dir": r"I:\Data\HYCOM\3hourly_test",
+        "start_time": "2018010100",
+        "end_time": "2019063000",
+        "output_dir": r"G:\Data\HYCOM\china_sea\hourly_24",
         "lon_min": 105,
-        "lon_max": 130,
-        "lat_min": 15,
+        "lon_max": 135,
+        "lat_min": 10,
         "lat_max": 45,
         "workers": 1,
         "overwrite": False,
         "depth": None,
         "level": None,
-        "validate_time": True,
-        "idm_path": r"D:\Programs\Internet Download Manager\IDMan.exe",
-        "interval_hours": 3,
+        "validate_time": None,
+        # "idm_path": r"D:\Programs\Internet Download Manager\IDMan.exe",
+        "interval_hours": 24,
+        "proxy_txt": None,
     }
 
     if single_var:

oafuncs/oa_down/read_proxy.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+# coding=utf-8
+"""
+Author: Liu Kun && 16031215@qq.com
+Date: 2025-04-17 13:59:14
+LastEditors: Liu Kun && 16031215@qq.com
+LastEditTime: 2025-04-17 14:00:38
+FilePath: \\Python\\My_Funcs\\OAFuncs\\oafuncs\\oa_down\\read_proxy.py
+Description:
+EditPlatform: vscode
+ComputerInfo: XPS 15 9510
+SystemInfo: Windows 11
+Python Version: 3.12
+"""
+
+import threading
+from queue import Queue
+import random
+import requests
+import os
+
+
+# 从文件中读取代理列表
+def read_proxies_from_file(filename):
+    try:
+        with open(filename, "r") as file:
+            proxies = [line.strip() for line in file if line.strip()]
+        return proxies
+    except FileNotFoundError:
+        print(f"未找到文件: {filename}，请检查文件是否存在。")
+        return []
+
+
+# 测试单个代理的可用性
+def test_single_proxy(proxy, test_url, working_proxies_queue):
+    try:
+        response = requests.get(test_url, proxies={"http": proxy, "https": proxy}, timeout=5)
+        if response.status_code == 200:
+            # print(f"代理 {proxy} 可用，返回 IP: {response.json()['origin']}")
+            working_proxies_queue.put(proxy)
+        else:
+            # print(f"代理 {proxy} 不可用，状态码: {response.status_code}")
+            pass
+    except Exception as e: # noqa: F841
+        # print(f"代理 {proxy} 不可用，错误: {e}")
+        pass
+
+
+# 测试代理的可用性（多线程）
+def test_proxies(proxies, test_url):
+    working_proxies_queue = Queue()
+    threads = []
+
+    # 为每个代理创建一个线程
+    for proxy in proxies:
+        thread = threading.Thread(target=test_single_proxy, args=(proxy, test_url, working_proxies_queue))
+        threads.append(thread)
+        thread.start()
+
+    # 等待所有线程完成
+    for thread in threads:
+        thread.join()
+
+    # 从队列中取出所有可用代理
+    working_proxies = []
+    while not working_proxies_queue.empty():
+        working_proxies.append(working_proxies_queue.get())
+
+    return working_proxies
+
+
+
+
+# 主函数
+def read_test(input_filename=r"E:\Code\Python\Tools\Yccol\output\http.txt"):
+    # 测试 URL
+    test_url = "http://httpbin.org/ip"
+
+    # 读取代理列表
+    proxies = read_proxies_from_file(input_filename)
+    if not proxies:
+        print(f"文件 '{input_filename}' 中没有找到有效的代理。")
+        return
+
+    # print(f"从文件 '{input_filename}' 中读取到 {len(proxies)} 个代理，开始测试...")
+
+    # 测试代理
+    working_proxies = test_proxies(proxies, test_url)
+    
+    return working_proxies
+
+def get_valid_proxy(input_filename=r"E:\Code\Python\Tools\Yccol\output\http.txt"):
+    working_proxies = read_test(input_filename)
+    if not working_proxies:
+        print("没有找到可用的代理。")
+        return None
+    choose_proxy = random.choice(working_proxies)
+    print(f"Randomly selected available proxy: {choose_proxy}")
+
+    # proxies = {"http": choose_proxy, "https": choose_proxy}
+    # return proxies
+    
+    return choose_proxy
+
+
+if __name__ == "__main__":
+    pwd = os.path.dirname(os.path.abspath(__file__))
+    read_test()

{oafuncs-0.0.98.4.dist-info → oafuncs-0.0.98.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: oafuncs
-Version: 0.0.98.4
+Version: 0.0.98.6
 Summary: Oceanic and Atmospheric Functions
 Home-page: https://github.com/Industry-Pays/OAFuncs
 Author: Kun Liu

{oafuncs-0.0.98.4.dist-info → oafuncs-0.0.98.6.dist-info}/RECORD

@@ -21,11 +21,11 @@ oafuncs/_script/plot_dataset.py,sha256=zkSEnO_-biyagorwWXPoihts_cwuvripzEt-l9bHJ
 oafuncs/_script/replace_file_content.py,sha256=eCFZjnZcwyRvy6b4mmIfBna-kylSZTyJRfgXd6DdCjk,5982
 oafuncs/oa_down/User_Agent-list.txt,sha256=pHaMlElMvZ8TG4vf4BqkZYKqe0JIGkr4kCN0lM1Y9FQ,514295
 oafuncs/oa_down/__init__.py,sha256=kRX5eTUCbAiz3zTaQM1501paOYS_3fizDN4Pa0mtNUA,585
-oafuncs/oa_down/hycom_3hourly.py,sha256=heG4L_eyPsc8pgUASCXYYJEdPGAY7rtRWQsiy-qrSxE,54202
-oafuncs/oa_down/hycom_3hourly_20250407.py,sha256=DQd_NmQgmSqu7jsrfpDB7k23mkUEy9kyWs-dLUg7GDw,64472
-oafuncs/oa_down/hycom_3hourly_20250416.py,sha256=X_fcV_xeJtYD-PB3GRdFWNHMPOVUYgh17MgWOtrdIbc,53493
+oafuncs/oa_down/hycom_3hourly.py,sha256=zU7eNFyg5oa-5MVrCeCVyZfoYHukY7MA0-DmvCj5yIk,54723
+oafuncs/oa_down/hycom_3hourly_proxy.py,sha256=1eaoJGI_m-7w4ZZ3n7NGxkZaeFdsm0d3U-hyw8RFNbc,54563
 oafuncs/oa_down/idm.py,sha256=4z5IvgfTyIKEI1kOtqXZwN7Jnfjwp6qDBOIoVyOLp0I,1823
 oafuncs/oa_down/literature.py,sha256=2bF9gSKQbzcci9LcKE81j8JEjIJwON7jbwQB3gDDA3E,11331
+oafuncs/oa_down/read_proxy.py,sha256=f5YidVA2ISRFNm-pgRsb_Omj0dySevrhcVoH6Y125UU,3237
 oafuncs/oa_down/test_ua.py,sha256=l8MCD6yU2W75zRPTDKUZTJhCWNF9lfk-MiSFqAqKH1M,1398
 oafuncs/oa_down/user_agent.py,sha256=TsPcAxFmMTYAEHRFjurI1bQBJfDhcA70MdHoUPwQmks,785
 oafuncs/oa_model/__init__.py,sha256=__ImltHkP1bSsIpsmKpDE8QwwA-2Z8K7mZUHGGcRdro,484
@@ -37,8 +37,8 @@ oafuncs/oa_sign/__init__.py,sha256=QKqTFrJDFK40C5uvk48GlRRbGFzO40rgkYwu6dYxatM,5
 oafuncs/oa_sign/meteorological.py,sha256=8091SHo2L8kl4dCFmmSH5NGVHDku5i5lSiLEG5DLnOQ,6489
 oafuncs/oa_sign/ocean.py,sha256=xrW-rWD7xBWsB5PuCyEwQ1Q_RDKq2KCLz-LOONHgldU,5932
 oafuncs/oa_sign/scientific.py,sha256=a4JxOBgm9vzNZKpJ_GQIQf7cokkraV5nh23HGbmTYKw,5064
-oafuncs-0.0.98.4.dist-info/licenses/LICENSE.txt,sha256=rMtLpVg8sKiSlwClfR9w_Dd_5WubTQgoOzE2PDFxzs4,1074
-oafuncs-0.0.98.4.dist-info/METADATA,sha256=VWSW69LEhmMu9FHH_e0WcNT-YXme3mdxlNvMedU48tY,4242
-oafuncs-0.0.98.4.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-oafuncs-0.0.98.4.dist-info/top_level.txt,sha256=bgC35QkXbN4EmPHEveg_xGIZ5i9NNPYWqtJqaKqTPsQ,8
-oafuncs-0.0.98.4.dist-info/RECORD,,
+oafuncs-0.0.98.6.dist-info/licenses/LICENSE.txt,sha256=rMtLpVg8sKiSlwClfR9w_Dd_5WubTQgoOzE2PDFxzs4,1074
+oafuncs-0.0.98.6.dist-info/METADATA,sha256=g6aPVBYo1PEpyW-4-0o1HgXZiNJZKGg1p3YEP2skSAY,4242
+oafuncs-0.0.98.6.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+oafuncs-0.0.98.6.dist-info/top_level.txt,sha256=bgC35QkXbN4EmPHEveg_xGIZ5i9NNPYWqtJqaKqTPsQ,8
+oafuncs-0.0.98.6.dist-info/RECORD,,