oafuncs 0.0.97.11__tar.gz → 0.0.97.13__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {oafuncs-0.0.97.11/oafuncs.egg-info → oafuncs-0.0.97.13}/PKG-INFO +1 -1
- oafuncs-0.0.97.13/oafuncs/_script/cprogressbar.py +208 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/oa_data.py +22 -1
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/oa_down/hycom_3hourly.py +52 -21
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/oa_down/idm.py +11 -4
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/oa_draw.py +8 -2
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/oa_nc.py +15 -9
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13/oafuncs.egg-info}/PKG-INFO +1 -1
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs.egg-info/SOURCES.txt +1 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/setup.py +1 -1
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/LICENSE.txt +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/MANIFEST.in +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/README.md +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/__init__.py +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/_data/OAFuncs.png +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/_data/hycom_3hourly.png +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/_script/auto_optimized_parallel_executor.py +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/_script/netcdf_merge.py +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/_script/parallel_example_usage.py +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/_script/plot_dataset.py +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/_script/replace_file_concent.py +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/oa_cmap.py +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/oa_date.py +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/oa_down/User_Agent-list.txt +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/oa_down/__init__.py +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/oa_down/literature.py +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/oa_down/test_ua.py +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/oa_down/user_agent.py +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/oa_file.py +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/oa_help.py +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/oa_model/__init__.py +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/oa_model/roms/__init__.py +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/oa_model/roms/test.py +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/oa_model/wrf/__init__.py +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/oa_model/wrf/little_r.py +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/oa_python.py +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/oa_sign/__init__.py +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/oa_sign/meteorological.py +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/oa_sign/ocean.py +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/oa_sign/scientific.py +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/oa_tool/__init__.py +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/oa_tool/email.py +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs/oa_tool/parallel.py +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs.egg-info/dependency_links.txt +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs.egg-info/requires.txt +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/oafuncs.egg-info/top_level.txt +0 -0
- {oafuncs-0.0.97.11 → oafuncs-0.0.97.13}/setup.cfg +0 -0
@@ -0,0 +1,208 @@
|
|
1
|
+
# progressbar.py
|
2
|
+
import re
|
3
|
+
import shutil
|
4
|
+
import sys
|
5
|
+
import time
|
6
|
+
import warnings
|
7
|
+
from typing import Any, Iterable, List, Optional, Union
|
8
|
+
|
9
|
+
import numpy as np
|
10
|
+
|
11
|
+
try:
|
12
|
+
import matplotlib
|
13
|
+
from matplotlib.colors import LinearSegmentedColormap, to_hex, to_rgb
|
14
|
+
except ImportError:
|
15
|
+
raise ImportError("This module requires matplotlib. Install with: pip install matplotlib")
|
16
|
+
|
17
|
+
|
18
|
+
class ColorProgressBar:
|
19
|
+
def __init__(self, iterable: Iterable, prefix: str = "", color: Any = "cyan", cmap: Union[str, List[str]] = None, update_interval: float = 0.1, bar_length: int = None):
|
20
|
+
self.iterable = iterable
|
21
|
+
self.prefix = prefix
|
22
|
+
self.base_color = color
|
23
|
+
self.cmap = cmap
|
24
|
+
self.update_interval = update_interval
|
25
|
+
self.bar_length = bar_length
|
26
|
+
|
27
|
+
self._start_time = None
|
28
|
+
self._last_update = 0
|
29
|
+
self._count = len(iterable) if hasattr(iterable, "__len__") else None
|
30
|
+
self._file = sys.stdout
|
31
|
+
self._gradient_colors = self._generate_gradient() if cmap and self._count else None
|
32
|
+
|
33
|
+
def _generate_gradient(self) -> Optional[List[str]]:
|
34
|
+
"""生成渐变色列表(修复内置colormap支持)"""
|
35
|
+
try:
|
36
|
+
if isinstance(self.cmap, list):
|
37
|
+
cmap = LinearSegmentedColormap.from_list("custom_cmap", self.cmap)
|
38
|
+
elif hasattr(self.cmap, "__call__") and hasattr(self.cmap, "N"):
|
39
|
+
# 直接处理已经是colormap对象的情况
|
40
|
+
cmap = self.cmap
|
41
|
+
else:
|
42
|
+
# 兼容不同版本的matplotlib
|
43
|
+
try:
|
44
|
+
# 新版本matplotlib (>=3.6)
|
45
|
+
cmap = matplotlib.colormaps[self.cmap]
|
46
|
+
except (AttributeError, KeyError):
|
47
|
+
# 旧版本matplotlib
|
48
|
+
cmap = matplotlib.cm.get_cmap(self.cmap)
|
49
|
+
|
50
|
+
return [to_hex(cmap(i)) for i in np.linspace(0, 1, self._count)]
|
51
|
+
except Exception as e:
|
52
|
+
warnings.warn(f"Colormap generation failed: {str(e)}. cmap type: {type(self.cmap)}")
|
53
|
+
return None
|
54
|
+
|
55
|
+
def _hex_to_ansi(self, hex_color: str) -> str:
|
56
|
+
"""将颜色转换为ANSI真彩色代码"""
|
57
|
+
try:
|
58
|
+
rgb = [int(x * 255) for x in to_rgb(hex_color)]
|
59
|
+
return f"\033[38;2;{rgb[0]};{rgb[1]};{rgb[2]}m"
|
60
|
+
except ValueError as e:
|
61
|
+
warnings.warn(f"Invalid color value: {e}, falling back to cyan")
|
62
|
+
return "\033[96m"
|
63
|
+
|
64
|
+
def _resolve_color(self, index: int) -> str:
|
65
|
+
"""解析当前应使用的颜色"""
|
66
|
+
if self._gradient_colors and 0 <= index < len(self._gradient_colors):
|
67
|
+
try:
|
68
|
+
return self._hex_to_ansi(self._gradient_colors[index])
|
69
|
+
except (IndexError, ValueError):
|
70
|
+
pass
|
71
|
+
|
72
|
+
return self._process_color_value(self.base_color)
|
73
|
+
|
74
|
+
def _process_color_value(self, color: Any) -> str:
|
75
|
+
"""处理颜色输入格式"""
|
76
|
+
preset_map = {
|
77
|
+
"red": "\033[91m",
|
78
|
+
"green": "\033[92m",
|
79
|
+
"yellow": "\033[93m",
|
80
|
+
"cyan": "\033[96m",
|
81
|
+
}
|
82
|
+
|
83
|
+
if color in preset_map:
|
84
|
+
return preset_map[color]
|
85
|
+
|
86
|
+
try:
|
87
|
+
hex_color = to_hex(color)
|
88
|
+
return self._hex_to_ansi(hex_color)
|
89
|
+
except (ValueError, TypeError) as e:
|
90
|
+
warnings.warn(f"Color parsing failed: {e}, using cyan")
|
91
|
+
return preset_map["cyan"]
|
92
|
+
|
93
|
+
def _strip_ansi(self, text: str) -> str:
|
94
|
+
"""移除所有ANSI转义序列"""
|
95
|
+
ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
|
96
|
+
return ansi_escape.sub("", text)
|
97
|
+
|
98
|
+
def _format_bar(self, progress: float, width: int) -> str:
|
99
|
+
"""格式化进度条显示"""
|
100
|
+
filled = "▊"
|
101
|
+
empty = " "
|
102
|
+
# 为其他信息保留更多空间
|
103
|
+
max_width = max(10, width - 60) # 至少保留10个字符的进度条
|
104
|
+
filled_length = int(round(max_width * progress))
|
105
|
+
return filled * filled_length + empty * (max_width - filled_length)
|
106
|
+
|
107
|
+
def _calculate_speed(self, index: int, elapsed: float) -> tuple:
|
108
|
+
"""计算速率和剩余时间"""
|
109
|
+
# 防止除零错误
|
110
|
+
if index == 0 or elapsed < 1e-6:
|
111
|
+
return 0.0, 0.0
|
112
|
+
|
113
|
+
rate = index / max(elapsed, 1e-6) # 确保分母不为零
|
114
|
+
remaining = (self._count - index) / rate if self._count else 0
|
115
|
+
return rate, remaining
|
116
|
+
|
117
|
+
def __iter__(self):
|
118
|
+
self._start_time = time.time()
|
119
|
+
self._last_update = self._start_time
|
120
|
+
reset_code = "\033[0m"
|
121
|
+
|
122
|
+
# 判断是否在终端环境中
|
123
|
+
is_terminal = hasattr(self._file, "isatty") and self._file.isatty()
|
124
|
+
|
125
|
+
try:
|
126
|
+
term_width = self.bar_length or (shutil.get_terminal_size().columns if is_terminal else 80)
|
127
|
+
except (AttributeError, OSError):
|
128
|
+
term_width = 80 # 默认终端宽度
|
129
|
+
|
130
|
+
for i, item in enumerate(self.iterable):
|
131
|
+
now = time.time()
|
132
|
+
elapsed = now - self._start_time
|
133
|
+
yield item
|
134
|
+
|
135
|
+
# 非终端环境或更新间隔未到时跳过更新
|
136
|
+
if not is_terminal or ((now - self._last_update) < self.update_interval and i + 1 != self._count):
|
137
|
+
continue
|
138
|
+
|
139
|
+
progress = (i + 1) / self._count if self._count else 0
|
140
|
+
current_color = self._resolve_color(i) if self._gradient_colors else self._resolve_color(0)
|
141
|
+
|
142
|
+
# 确保进度条至少有一个字符的宽度
|
143
|
+
effective_width = max(15, term_width - 40) # 保留更多空间给信息显示
|
144
|
+
bar = self._format_bar(progress, effective_width)
|
145
|
+
|
146
|
+
rate, remaining = self._calculate_speed(i + 1, elapsed)
|
147
|
+
|
148
|
+
count_info = f"{i + 1}/{self._count}" if self._count else str(i + 1)
|
149
|
+
percent = f"{progress:.1%}" if self._count else ""
|
150
|
+
rate_info = f"{rate:.1f}it/s" if rate else ""
|
151
|
+
time_info = f"ETA: {remaining:.1f}s" if self._count and remaining > 0 else f"Elapsed: {elapsed:.1f}s"
|
152
|
+
|
153
|
+
# 构建新的进度条行
|
154
|
+
line = f"{self.prefix}{current_color}[{bar}]{reset_code} {count_info} {percent} [{time_info} | {rate_info}]"
|
155
|
+
|
156
|
+
# 清除之前的行并强制光标回到行首
|
157
|
+
self._file.write("\r")
|
158
|
+
|
159
|
+
# 确保不超出终端宽度
|
160
|
+
if len(self._strip_ansi(line)) > term_width:
|
161
|
+
line = line[: term_width - 3] + "..."
|
162
|
+
|
163
|
+
# 输出并强制刷新
|
164
|
+
self._file.write(line)
|
165
|
+
self._file.flush()
|
166
|
+
self._last_update = now
|
167
|
+
|
168
|
+
# 完成后添加换行符
|
169
|
+
if is_terminal:
|
170
|
+
self._file.write("\n")
|
171
|
+
self._file.flush()
|
172
|
+
|
173
|
+
@classmethod
|
174
|
+
def gradient_color(cls, colors: List[str], n: int) -> List[str]:
|
175
|
+
"""生成渐变色列表"""
|
176
|
+
cmap = LinearSegmentedColormap.from_list("gradient", colors)
|
177
|
+
return [to_hex(cmap(i)) for i in np.linspace(0, 1, n)]
|
178
|
+
|
179
|
+
|
180
|
+
# 验证示例
|
181
|
+
if __name__ == "__main__":
|
182
|
+
# 使用内置colormap示例
|
183
|
+
import oafuncs
|
184
|
+
|
185
|
+
cmap = oafuncs.oa_cmap.get("diverging_1")
|
186
|
+
for _ in ColorProgressBar(range(100), cmap=cmap, prefix="Diverging: "):
|
187
|
+
time.sleep(0.1)
|
188
|
+
|
189
|
+
for _ in ColorProgressBar(range(100), cmap="viridis", prefix="Viridis: "):
|
190
|
+
time.sleep(0.1)
|
191
|
+
|
192
|
+
# 使用自定义渐变色
|
193
|
+
for _ in ColorProgressBar(range(50), cmap=["#FF0000", "#0000FF"], prefix="Custom: "):
|
194
|
+
time.sleep(0.1)
|
195
|
+
|
196
|
+
# 测试无法获取长度的迭代器
|
197
|
+
def infinite_generator():
|
198
|
+
i = 0
|
199
|
+
while True:
|
200
|
+
yield i
|
201
|
+
i += 1
|
202
|
+
|
203
|
+
# 限制为20个元素,但进度条不知道总长度
|
204
|
+
gen = infinite_generator()
|
205
|
+
for i, _ in enumerate(ColorProgressBar(gen, prefix="Unknown length: ")):
|
206
|
+
if i >= 20:
|
207
|
+
break
|
208
|
+
time.sleep(0.1)
|
@@ -22,8 +22,9 @@ import salem
|
|
22
22
|
import xarray as xr
|
23
23
|
from scipy.interpolate import griddata
|
24
24
|
from scipy.interpolate import interp1d
|
25
|
+
from typing import Iterable
|
25
26
|
|
26
|
-
__all__ = ["interp_along_dim", "interp_2d", "ensure_list", "mask_shapefile"]
|
27
|
+
__all__ = ["interp_along_dim", "interp_2d", "ensure_list", "mask_shapefile", "pbar"]
|
27
28
|
|
28
29
|
|
29
30
|
def ensure_list(input_data):
|
@@ -254,6 +255,26 @@ def mask_shapefile(data: np.ndarray, lons: np.ndarray, lats: np.ndarray, shapefi
|
|
254
255
|
return None
|
255
256
|
|
256
257
|
|
258
|
+
def pbar(iterable: Iterable, prefix: str = "", color: str = "cyan", cmap: str = None, **kwargs) -> Iterable:
|
259
|
+
"""
|
260
|
+
快速创建进度条的封装函数
|
261
|
+
:param iterable: 可迭代对象
|
262
|
+
:param prefix: 进度条前缀
|
263
|
+
:param color: 基础颜色
|
264
|
+
:param cmap: 渐变色名称
|
265
|
+
:param kwargs: 其他ColorProgressBar支持的参数
|
266
|
+
|
267
|
+
example:
|
268
|
+
from oafuncs.oa_data import pbar
|
269
|
+
from time import sleep
|
270
|
+
for i in pbar(range(100), prefix="Processing", color="green", cmap="viridis"):
|
271
|
+
sleep(0.1)
|
272
|
+
"""
|
273
|
+
from ._script.cprogressbar import ColorProgressBar # 从progressbar.py导入类
|
274
|
+
|
275
|
+
return ColorProgressBar(iterable=iterable, prefix=prefix, color=color, cmap=cmap, **kwargs)
|
276
|
+
|
277
|
+
|
257
278
|
if __name__ == "__main__":
|
258
279
|
pass
|
259
280
|
""" import time
|
@@ -13,9 +13,6 @@ SystemInfo: Windows 11
|
|
13
13
|
Python Version: 3.12
|
14
14
|
"""
|
15
15
|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
16
|
import datetime
|
20
17
|
import os
|
21
18
|
import random
|
@@ -711,13 +708,13 @@ def _download_file(target_url, store_path, file_name, check=False):
|
|
711
708
|
file_name_split = file_name_split[:-1]
|
712
709
|
# same_file = f"{file_name_split[0]}_{file_name_split[1]}*nc"
|
713
710
|
same_file = "_".join(file_name_split) + "*nc"
|
714
|
-
|
711
|
+
|
715
712
|
if match_time is not None:
|
716
713
|
if check_nc(fname):
|
717
714
|
if not _check_ftime(fname, if_print=True):
|
718
715
|
if match_time:
|
719
716
|
_correct_time(fname)
|
720
|
-
count_dict[
|
717
|
+
count_dict["skip"] += 1
|
721
718
|
else:
|
722
719
|
_clear_existing_file(fname)
|
723
720
|
# print(f"[bold #ffe5c0]File time error, {fname}")
|
@@ -900,7 +897,7 @@ def _get_submit_url_var(var, depth, level_num, lon_min, lon_max, lat_min, lat_ma
|
|
900
897
|
|
901
898
|
|
902
899
|
def _prepare_url_to_download(var, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, download_time="2024083100", download_time_end=None, depth=None, level_num=None, store_path=None, dataset_name=None, version_name=None, check=False):
|
903
|
-
print("[bold #ecdbfe]-" *
|
900
|
+
print("[bold #ecdbfe]-" * mark_len)
|
904
901
|
download_time = str(download_time)
|
905
902
|
if download_time_end is not None:
|
906
903
|
download_time_end = str(download_time_end)
|
@@ -997,7 +994,7 @@ def _done_callback(future, progress, task, total, counter_lock):
|
|
997
994
|
global parallel_counter
|
998
995
|
with counter_lock:
|
999
996
|
parallel_counter += 1
|
1000
|
-
progress.update(task, advance=1, description=f"[cyan]
|
997
|
+
progress.update(task, advance=1, description=f"[cyan]{bar_desc} {parallel_counter}/{total}")
|
1001
998
|
|
1002
999
|
|
1003
1000
|
def _download_hourly_func(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, depth=None, level=None, store_path=None, dataset_name=None, version_name=None, num_workers=None, check=False, ftimes=1, interval_hour=3):
|
@@ -1034,19 +1031,19 @@ def _download_hourly_func(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_mi
|
|
1034
1031
|
print("Downloading a series of files...")
|
1035
1032
|
time_list = get_time_list(ymdh_time_s, ymdh_time_e, interval_hour, "hour")
|
1036
1033
|
with Progress() as progress:
|
1037
|
-
task = progress.add_task("[cyan]
|
1034
|
+
task = progress.add_task(f"[cyan]{bar_desc}", total=len(time_list))
|
1038
1035
|
if ftimes == 1:
|
1039
1036
|
if num_workers is None or num_workers <= 1:
|
1040
1037
|
# 串行方式
|
1041
1038
|
for i, time_str in enumerate(time_list):
|
1042
1039
|
_prepare_url_to_download(var, lon_min, lon_max, lat_min, lat_max, time_str, None, depth, level, store_path, dataset_name, version_name, check)
|
1043
|
-
progress.update(task, advance=1, description=f"[cyan]
|
1040
|
+
progress.update(task, advance=1, description=f"[cyan]{bar_desc} {i + 1}/{len(time_list)}")
|
1044
1041
|
else:
|
1045
1042
|
# 并行方式
|
1046
1043
|
with ThreadPoolExecutor(max_workers=num_workers) as executor:
|
1047
1044
|
futures = [executor.submit(_download_task, var, time_str, None, lon_min, lon_max, lat_min, lat_max, depth, level, store_path, dataset_name, version_name, check) for time_str in time_list]
|
1048
1045
|
""" for i, future in enumerate(futures):
|
1049
|
-
future.add_done_callback(lambda _: progress.update(task, advance=1, description=f"[cyan]
|
1046
|
+
future.add_done_callback(lambda _: progress.update(task, advance=1, description=f"[cyan]{bar_desc} {i+1}/{len(time_list)}")) """
|
1050
1047
|
for feature in as_completed(futures):
|
1051
1048
|
_done_callback(feature, progress, task, len(time_list), counter_lock)
|
1052
1049
|
else:
|
@@ -1059,13 +1056,13 @@ def _download_hourly_func(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_mi
|
|
1059
1056
|
time_str_end_index = int(min(len(time_list) - 1, int(i * ftimes + ftimes - 1)))
|
1060
1057
|
time_str_end = time_list[time_str_end_index]
|
1061
1058
|
_prepare_url_to_download(var, lon_min, lon_max, lat_min, lat_max, time_str, time_str_end, depth, level, store_path, dataset_name, version_name, check)
|
1062
|
-
progress.update(task, advance=1, description=f"[cyan]
|
1059
|
+
progress.update(task, advance=1, description=f"[cyan]{bar_desc} {i + 1}/{total_num}")
|
1063
1060
|
else:
|
1064
1061
|
# 并行方式
|
1065
1062
|
with ThreadPoolExecutor(max_workers=num_workers) as executor:
|
1066
1063
|
futures = [executor.submit(_download_task, var, new_time_list[i], time_list[int(min(len(time_list) - 1, int(i * ftimes + ftimes - 1)))], lon_min, lon_max, lat_min, lat_max, depth, level, store_path, dataset_name, version_name, check) for i in range(total_num)]
|
1067
1064
|
""" for i, future in enumerate(futures):
|
1068
|
-
future.add_done_callback(lambda _: progress.update(task, advance=1, description=f"[cyan]
|
1065
|
+
future.add_done_callback(lambda _: progress.update(task, advance=1, description=f"[cyan]{bar_desc} {i+1}/{total_num}")) """
|
1069
1066
|
for feature in as_completed(futures):
|
1070
1067
|
_done_callback(feature, progress, task, len(time_list), counter_lock)
|
1071
1068
|
else:
|
@@ -1100,6 +1097,9 @@ def download(var, time_s, time_e=None, lon_min=0, lon_max=359.92, lat_min=-80, l
|
|
1100
1097
|
Returns:
|
1101
1098
|
None
|
1102
1099
|
"""
|
1100
|
+
from oafuncs.oa_data import pbar
|
1101
|
+
from oafuncs.oa_cmap import get as get_cmap
|
1102
|
+
|
1103
1103
|
_get_initial_data()
|
1104
1104
|
|
1105
1105
|
# 打印信息并处理数据集和版本名称
|
@@ -1166,45 +1166,76 @@ def download(var, time_s, time_e=None, lon_min=0, lon_max=359.92, lat_min=-80, l
|
|
1166
1166
|
|
1167
1167
|
global fsize_dict_lock
|
1168
1168
|
fsize_dict_lock = Lock()
|
1169
|
-
|
1169
|
+
|
1170
1170
|
if fill_time is not None:
|
1171
1171
|
num_workers = 1
|
1172
1172
|
|
1173
|
-
global use_idm, given_idm_engine, idm_download_list
|
1173
|
+
global use_idm, given_idm_engine, idm_download_list, bar_desc
|
1174
1174
|
if idm_engine is not None:
|
1175
1175
|
use_idm = True
|
1176
1176
|
num_workers = 1
|
1177
1177
|
given_idm_engine = idm_engine
|
1178
1178
|
idm_download_list = []
|
1179
|
+
bar_desc = "Submitting to IDM ..."
|
1179
1180
|
else:
|
1180
1181
|
use_idm = False
|
1182
|
+
bar_desc = "Downloading ..."
|
1181
1183
|
|
1182
1184
|
global match_time
|
1183
1185
|
match_time = fill_time
|
1184
1186
|
|
1187
|
+
global mark_len
|
1188
|
+
mark_len = 100
|
1189
|
+
|
1185
1190
|
_download_hourly_func(var, time_s, time_e, lon_min, lon_max, lat_min, lat_max, depth, level, store_path, dataset_name, version_name, num_workers, check, ftimes, int(interval_hour))
|
1186
1191
|
|
1187
1192
|
if idm_engine is not None:
|
1193
|
+
print("[bold #ecdbfe]*" * mark_len)
|
1194
|
+
str_info = "All files have been submitted to IDM for downloading"
|
1195
|
+
str_info = str_info.center(mark_len, "*")
|
1196
|
+
print(f"[bold #3dfc40]{str_info}")
|
1197
|
+
print("[bold #ecdbfe]*" * mark_len)
|
1188
1198
|
if idm_download_list:
|
1189
|
-
|
1199
|
+
""" file_download_time = 60 # 预设下载时间为1分钟
|
1200
|
+
for f in pbar(idm_download_list,cmap='bwr',prefix='HYCOM: '):
|
1201
|
+
file_download_start_time = time.time()
|
1190
1202
|
wait_success = 0
|
1191
1203
|
success = False
|
1192
1204
|
while not success:
|
1193
|
-
if check_nc(f):
|
1205
|
+
if check_nc(f,print_switch=False):
|
1194
1206
|
count_dict["success"] += 1
|
1195
1207
|
success = True
|
1208
|
+
# print(f"[bold #3dfc40]File [bold #dfff73]{f} [#3dfc40]has been downloaded successfully")
|
1209
|
+
file_download_end_time = time.time()
|
1210
|
+
file_download_time = file_download_end_time - file_download_start_time
|
1211
|
+
file_download_time = int(file_download_time)
|
1212
|
+
# print(f"[bold #3dfc40]Time: {file_download_time} seconds")
|
1213
|
+
file_download_time = max(60, file_download_time) # 预设下载时间为1分钟起步
|
1196
1214
|
else:
|
1197
1215
|
wait_success += 1
|
1198
|
-
|
1199
|
-
|
1216
|
+
# print(f"[bold #ffe5c0]Waiting {file_download_time} seconds to check the file {f}...")
|
1217
|
+
time.sleep(file_download_time)
|
1218
|
+
if wait_success >= 10:
|
1200
1219
|
success = True
|
1201
1220
|
# print(f'{f} download failed')
|
1221
|
+
print(f"[bold #ffe5c0]Waiting for more than 10 times, skipping the file {f}...")
|
1202
1222
|
count_dict["fail"] += 1
|
1223
|
+
# print("[bold #ecdbfe]-" * mark_len) """
|
1224
|
+
remain_list = idm_download_list.copy()
|
1225
|
+
for f_count in pbar(range(len(idm_download_list)), cmap=get_cmap('diverging_1'), prefix="HYCOM: "):
|
1226
|
+
success = False
|
1227
|
+
while not success:
|
1228
|
+
for f in remain_list:
|
1229
|
+
if check_nc(f, print_switch=False):
|
1230
|
+
count_dict["success"] += 1
|
1231
|
+
success = True
|
1232
|
+
remain_list.remove(f)
|
1233
|
+
break
|
1203
1234
|
|
1204
1235
|
count_dict["total"] = count_dict["success"] + count_dict["fail"] + count_dict["skip"] + count_dict["no_data"]
|
1205
|
-
print("[bold #ecdbfe]
|
1236
|
+
print("[bold #ecdbfe]=" * mark_len)
|
1206
1237
|
print(f"[bold #ff80ab]Total: {count_dict['total']}\nSuccess: {count_dict['success']}\nFail: {count_dict['fail']}\nSkip: {count_dict['skip']}\nNo data: {count_dict['no_data']}")
|
1207
|
-
print("[bold #ecdbfe]
|
1238
|
+
print("[bold #ecdbfe]=" * mark_len)
|
1208
1239
|
if count_dict["fail"] > 0:
|
1209
1240
|
print("[bold #be5528]Please try again to download the failed data later")
|
1210
1241
|
if count_dict["no_data"] > 0:
|
@@ -1214,7 +1245,7 @@ def download(var, time_s, time_e=None, lon_min=0, lon_max=359.92, lat_min=-80, l
|
|
1214
1245
|
print(f"[bold #f90000]These are {count_dict['no_data']} data that do not exist in any dataset and version")
|
1215
1246
|
for no_data in count_dict["no_data_list"]:
|
1216
1247
|
print(f"[bold #d81b60]{no_data}")
|
1217
|
-
print("[bold #ecdbfe]
|
1248
|
+
print("[bold #ecdbfe]=" * mark_len)
|
1218
1249
|
|
1219
1250
|
|
1220
1251
|
def how_to_use():
|
@@ -2,9 +2,9 @@
|
|
2
2
|
# coding=utf-8
|
3
3
|
"""
|
4
4
|
Author: Liu Kun && 16031215@qq.com
|
5
|
-
Date: 2025-
|
5
|
+
Date: 2025-03-27 16:51:26
|
6
6
|
LastEditors: Liu Kun && 16031215@qq.com
|
7
|
-
LastEditTime: 2025-01
|
7
|
+
LastEditTime: 2025-04-01 22:31:39
|
8
8
|
FilePath: \\Python\\My_Funcs\\OAFuncs\\oafuncs\\oa_down\\idm.py
|
9
9
|
Description:
|
10
10
|
EditPlatform: vscode
|
@@ -13,6 +13,7 @@ SystemInfo: Windows 11
|
|
13
13
|
Python Version: 3.12
|
14
14
|
"""
|
15
15
|
|
16
|
+
|
16
17
|
import datetime
|
17
18
|
import os
|
18
19
|
from subprocess import call
|
@@ -46,5 +47,11 @@ def downloader(task_url, folder_path, file_name, idm_engine=r"D:\Programs\Intern
|
|
46
47
|
# 开始任务队列
|
47
48
|
call([idm_engine, "/s"])
|
48
49
|
# print(f"IDM下载器:{file_name}下载任务已添加至队列...")
|
49
|
-
print("[purple]-" *
|
50
|
-
print(
|
50
|
+
# print("[purple]-" * 150 + f"\n{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n" + "[purple]-" * 150)
|
51
|
+
print("[purple]*" * 100)
|
52
|
+
time_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
53
|
+
time_str = time_str.center(100, " ")
|
54
|
+
print(f"[bold purple]{time_str}")
|
55
|
+
print(f"[green]IDM Downloader: {file_name} download task has been added to the queue ...[/green]")
|
56
|
+
print("[purple]*" * 100)
|
57
|
+
print("\n")
|
@@ -165,13 +165,19 @@ def add_lonlat_unit(lon=None, lat=None, decimal=2):
|
|
165
165
|
|
166
166
|
|
167
167
|
# ** 添加网格线
|
168
|
-
def add_gridlines(ax, projection=ccrs.PlateCarree(), color="k", alpha=0.5, linestyle="--", linewidth=0.5):
|
168
|
+
def add_gridlines(ax, xline=None, yline=None, projection=ccrs.PlateCarree(), color="k", alpha=0.5, linestyle="--", linewidth=0.5):
|
169
|
+
from matplotlib import ticker as mticker
|
169
170
|
# add gridlines
|
170
171
|
gl = ax.gridlines(crs=projection, draw_labels=True, linewidth=linewidth, color=color, alpha=alpha, linestyle=linestyle)
|
171
172
|
gl.right_labels = False
|
172
173
|
gl.top_labels = False
|
173
174
|
gl.xformatter = LongitudeFormatter(zero_direction_label=False)
|
174
175
|
gl.yformatter = LatitudeFormatter()
|
176
|
+
|
177
|
+
if xline is not None:
|
178
|
+
gl.xlocator = mticker.FixedLocator(np.array(xline))
|
179
|
+
if yline is not None:
|
180
|
+
gl.ylocator = mticker.FixedLocator(np.array(yline))
|
175
181
|
|
176
182
|
return ax, gl
|
177
183
|
|
@@ -186,7 +192,7 @@ def add_cartopy(ax, lon=None, lat=None, projection=ccrs.PlateCarree(), gridlines
|
|
186
192
|
|
187
193
|
# add gridlines
|
188
194
|
if gridlines:
|
189
|
-
ax, gl = add_gridlines(ax, projection)
|
195
|
+
ax, gl = add_gridlines(ax, projection=projection)
|
190
196
|
|
191
197
|
# set longitude and latitude format
|
192
198
|
lon_formatter = LongitudeFormatter(zero_direction_label=False)
|
@@ -328,7 +328,7 @@ def rename(ncfile_path, old_name, new_name):
|
|
328
328
|
print(f"An error occurred: {e}")
|
329
329
|
|
330
330
|
|
331
|
-
def check(ncfile: str, delete_switch: bool = False) -> bool:
|
331
|
+
def check(ncfile: str, delete_switch: bool = False, print_switch: bool = True) -> bool:
|
332
332
|
"""
|
333
333
|
Check if a NetCDF file is corrupted with enhanced error handling.
|
334
334
|
|
@@ -337,9 +337,10 @@ def check(ncfile: str, delete_switch: bool = False) -> bool:
|
|
337
337
|
is_valid = False
|
338
338
|
|
339
339
|
if not os.path.exists(ncfile):
|
340
|
-
|
341
|
-
|
342
|
-
|
340
|
+
if print_switch:
|
341
|
+
print(f"[#ffeac5]Local file missing: [#009d88]{ncfile}")
|
342
|
+
# 提示:提示文件缺失也许是正常的,这只是检查文件是否存在于本地
|
343
|
+
print("[#d6d9fd]Note: File missing may be normal, this is just to check if the file exists locally.")
|
343
344
|
return False
|
344
345
|
|
345
346
|
try:
|
@@ -352,7 +353,8 @@ def check(ncfile: str, delete_switch: bool = False) -> bool:
|
|
352
353
|
# 二次验证确保变量可访问
|
353
354
|
with nc.Dataset(ncfile, "r") as ds_verify:
|
354
355
|
if not ds_verify.variables:
|
355
|
-
|
356
|
+
if print_switch:
|
357
|
+
print(f"[red]Empty variables: {ncfile}[/red]")
|
356
358
|
else:
|
357
359
|
# 尝试访问元数据
|
358
360
|
_ = ds_verify.__dict__
|
@@ -363,19 +365,23 @@ def check(ncfile: str, delete_switch: bool = False) -> bool:
|
|
363
365
|
is_valid = True
|
364
366
|
|
365
367
|
except Exception as e: # 捕获所有异常类型
|
366
|
-
|
368
|
+
if print_switch:
|
369
|
+
print(f"[red]HDF5 validation failed for {ncfile}: {str(e)}[/red]")
|
367
370
|
error_type = type(e).__name__
|
368
371
|
if "HDF5" in error_type or "h5" in error_type.lower():
|
369
|
-
|
372
|
+
if print_switch:
|
373
|
+
print(f"[red]Critical HDF5 structure error detected in {ncfile}[/red]")
|
370
374
|
|
371
375
|
# 安全删除流程
|
372
376
|
if not is_valid:
|
373
377
|
if delete_switch:
|
374
378
|
try:
|
375
379
|
os.remove(ncfile)
|
376
|
-
|
380
|
+
if print_switch:
|
381
|
+
print(f"[red]Removed corrupted file: {ncfile}[/red]")
|
377
382
|
except Exception as del_error:
|
378
|
-
|
383
|
+
if print_switch:
|
384
|
+
print(f"[red]Failed to delete corrupted file: {ncfile} - {str(del_error)}[/red]")
|
379
385
|
return False
|
380
386
|
|
381
387
|
return True
|
@@ -19,6 +19,7 @@ oafuncs.egg-info/top_level.txt
|
|
19
19
|
oafuncs/_data/OAFuncs.png
|
20
20
|
oafuncs/_data/hycom_3hourly.png
|
21
21
|
oafuncs/_script/auto_optimized_parallel_executor.py
|
22
|
+
oafuncs/_script/cprogressbar.py
|
22
23
|
oafuncs/_script/netcdf_merge.py
|
23
24
|
oafuncs/_script/parallel_example_usage.py
|
24
25
|
oafuncs/_script/plot_dataset.py
|
@@ -18,7 +18,7 @@ URL = "https://github.com/Industry-Pays/OAFuncs"
|
|
18
18
|
EMAIL = "liukun0312@stu.ouc.edu.cn"
|
19
19
|
AUTHOR = "Kun Liu"
|
20
20
|
REQUIRES_PYTHON = ">=3.9.0" # 2025/03/13
|
21
|
-
VERSION = "0.0.97.
|
21
|
+
VERSION = "0.0.97.13" # 下次用98.0,98.1已经被用过了
|
22
22
|
|
23
23
|
# What packages are required for this module to be executed?
|
24
24
|
REQUIRED = [
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|