oafuncs 0.0.97.4__py3-none-any.whl → 0.0.97.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oafuncs/__init__.py +2 -13
- oafuncs/_script/auto_optimized_parallel_executor.py +459 -0
- oafuncs/_script/parallel_example_usage.py +83 -0
- oafuncs/_script/replace_file_concent.py +151 -0
- oafuncs/oa_date.py +90 -0
- oafuncs/oa_down/hycom_3hourly.py +2 -2
- oafuncs/oa_down/hycom_3hourly_20250129.py +2 -2
- oafuncs/oa_down/literature.py +1 -1
- oafuncs/oa_draw.py +32 -7
- oafuncs/oa_file.py +80 -10
- oafuncs/oa_nc.py +32 -27
- oafuncs/oa_tool/__init__.py +1 -13
- oafuncs/oa_tool/parallel.py +479 -6
- {oafuncs-0.0.97.4.dist-info → oafuncs-0.0.97.6.dist-info}/METADATA +4 -2
- {oafuncs-0.0.97.4.dist-info → oafuncs-0.0.97.6.dist-info}/RECORD +18 -15
- {oafuncs-0.0.97.4.dist-info → oafuncs-0.0.97.6.dist-info}/WHEEL +1 -1
- oafuncs/oa_tool/time.py +0 -22
- {oafuncs-0.0.97.4.dist-info → oafuncs-0.0.97.6.dist-info/licenses}/LICENSE.txt +0 -0
- {oafuncs-0.0.97.4.dist-info → oafuncs-0.0.97.6.dist-info}/top_level.txt +0 -0
oafuncs/oa_tool/parallel.py
CHANGED
@@ -1,19 +1,40 @@
|
|
1
|
+
#!/usr/bin/env python
|
2
|
+
# coding=utf-8
|
3
|
+
"""
|
4
|
+
Author: Liu Kun && 16031215@qq.com
|
5
|
+
Date: 2025-01-11 19:47:08
|
6
|
+
LastEditors: Liu Kun && 16031215@qq.com
|
7
|
+
LastEditTime: 2025-03-18 19:21:36
|
8
|
+
FilePath: \\Python\\My_Funcs\\OAFuncs\\oafuncs\\oa_tool\\parallel.py
|
9
|
+
Description:
|
10
|
+
EditPlatform: vscode
|
11
|
+
ComputerInfo: XPS 15 9510
|
12
|
+
SystemInfo: Windows 11
|
13
|
+
Python Version: 3.12
|
14
|
+
"""
|
15
|
+
|
16
|
+
import contextlib
|
1
17
|
import logging
|
2
18
|
import multiprocessing as mp
|
19
|
+
import os
|
20
|
+
import platform
|
21
|
+
import time
|
3
22
|
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed
|
4
23
|
|
24
|
+
import psutil
|
25
|
+
|
5
26
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
|
6
27
|
|
7
|
-
__all__ = ["ParallelExecutor"]
|
28
|
+
__all__ = ["Simple_ParallelExecutor", "ParallelExecutor"]
|
8
29
|
|
9
30
|
|
10
|
-
class
|
31
|
+
class Simple_ParallelExecutor:
|
11
32
|
"""
|
12
33
|
A class for parallel execution of tasks using threads or processes.
|
13
34
|
|
14
35
|
If mode is "process", the tasks are executed in separate processes.
|
15
36
|
If mode is "thread", the tasks are executed in separate threads.
|
16
|
-
|
37
|
+
|
17
38
|
Parameters:
|
18
39
|
mode (str): The execution mode. Supported values are "process" and "thread".
|
19
40
|
process ~ Must use top function to run, can't use in jupyter notebook
|
@@ -66,24 +87,476 @@ class ParallelExecutor:
|
|
66
87
|
logging.info("Parallel execution completed.")
|
67
88
|
return results
|
68
89
|
|
90
|
+
|
69
91
|
def _compute_square(x):
|
70
92
|
return x * x
|
71
93
|
|
72
|
-
|
94
|
+
|
95
|
+
def _example():
|
73
96
|
def _compute_sum(a, b):
|
74
97
|
return a + b
|
75
98
|
|
76
|
-
executor1 =
|
99
|
+
executor1 = Simple_ParallelExecutor(mode="process", max_workers=4)
|
77
100
|
params1 = [(i,) for i in range(10)]
|
78
101
|
results1 = executor1.run(_compute_square, params1)
|
79
102
|
print("Results (compute_square):", results1)
|
80
103
|
|
81
|
-
executor2 =
|
104
|
+
executor2 = Simple_ParallelExecutor(mode="thread", max_workers=2)
|
82
105
|
params2 = [(1, 2), (3, 4), (5, 6)]
|
83
106
|
results2 = executor2.run(_compute_sum, params2)
|
84
107
|
print("Results (compute_sum):", results2)
|
85
108
|
|
86
109
|
|
110
|
+
class ParallelExecutor:
|
111
|
+
"""
|
112
|
+
自动优化的并行执行器,根据平台和任务特性自动选择最佳执行模式和工作线程/进程数量。
|
113
|
+
|
114
|
+
特性:
|
115
|
+
- 自动检测平台并选择最佳执行模式
|
116
|
+
- 动态调整工作线程/进程数量
|
117
|
+
- 针对Linux和Windows的特定优化
|
118
|
+
- 任务批处理功能以提高小任务的效率
|
119
|
+
- 自动故障转移机制
|
120
|
+
"""
|
121
|
+
|
122
|
+
def __init__(self):
|
123
|
+
# 检测平台
|
124
|
+
self.platform = self._detect_platform()
|
125
|
+
# 自动选择最佳执行模式和工作线程/进程数量
|
126
|
+
self.mode, self.max_workers = self._determine_optimal_settings()
|
127
|
+
# 初始化执行器
|
128
|
+
self._executor = None
|
129
|
+
self.executor_class = ProcessPoolExecutor if self.mode == "process" else ThreadPoolExecutor
|
130
|
+
# 进程池重用策略
|
131
|
+
self.reuse_pool = self.mode == "process" and self.platform != "windows"
|
132
|
+
|
133
|
+
# 特定于平台的优化参数
|
134
|
+
self.mp_context = None
|
135
|
+
self.chunk_size = self._get_default_chunk_size()
|
136
|
+
self.timeout_per_task = 3600 # 默认任务超时时间(秒)
|
137
|
+
self.worker_init_func = None
|
138
|
+
|
139
|
+
# 针对Linux的特定优化
|
140
|
+
if self.platform == "linux":
|
141
|
+
self._setup_linux_optimizations()
|
142
|
+
# 针对Windows的特定优化
|
143
|
+
elif self.platform == "windows":
|
144
|
+
self._setup_windows_optimizations()
|
145
|
+
|
146
|
+
logging.info(f"Initialized {self.__class__.__name__} with mode={self.mode}, max_workers={self.max_workers} on {self.platform} platform")
|
147
|
+
|
148
|
+
def _detect_platform(self):
|
149
|
+
"""检测当前运行的平台"""
|
150
|
+
system = platform.system().lower()
|
151
|
+
if system == "linux":
|
152
|
+
return "linux"
|
153
|
+
elif system == "windows":
|
154
|
+
return "windows"
|
155
|
+
elif system == "darwin":
|
156
|
+
return "macos"
|
157
|
+
else:
|
158
|
+
return "unknown"
|
159
|
+
|
160
|
+
def _determine_optimal_settings(self):
|
161
|
+
"""确定最佳执行模式和工作线程/进程数量"""
|
162
|
+
mode = "process" # 默认使用进程模式
|
163
|
+
|
164
|
+
# Linux平台优化
|
165
|
+
if self.platform == "linux":
|
166
|
+
# 在Linux上,根据之前的问题,我们优先使用进程模式
|
167
|
+
mode = "process"
|
168
|
+
|
169
|
+
# 检查是否在容器中运行(如Docker)
|
170
|
+
in_container = self._is_in_container()
|
171
|
+
|
172
|
+
# 获取物理和逻辑CPU核心数
|
173
|
+
physical_cores = psutil.cpu_count(logical=False) or 1
|
174
|
+
logical_cores = psutil.cpu_count(logical=True) or 1
|
175
|
+
|
176
|
+
# 获取系统内存信息
|
177
|
+
mem = psutil.virtual_memory()
|
178
|
+
# total_mem_gb = mem.total / (1024**3)
|
179
|
+
available_mem_gb = mem.available / (1024**3)
|
180
|
+
|
181
|
+
# 每个进程估计内存使用(根据应用程序特性调整)
|
182
|
+
est_mem_per_process_gb = 0.5
|
183
|
+
|
184
|
+
# 根据可用内存限制工作进程数
|
185
|
+
mem_limited_workers = max(1, int(available_mem_gb / est_mem_per_process_gb))
|
186
|
+
|
187
|
+
# 在容器环境中更保守一些
|
188
|
+
if in_container:
|
189
|
+
max_workers = min(physical_cores, mem_limited_workers, 4)
|
190
|
+
else:
|
191
|
+
max_workers = min(logical_cores, mem_limited_workers)
|
192
|
+
|
193
|
+
# Windows平台优化
|
194
|
+
elif self.platform == "windows":
|
195
|
+
# Windows上进程创建较快,线程和进程都可以考虑
|
196
|
+
# 但进程间通信开销大,所以对于小型任务,线程可能更高效
|
197
|
+
mode = "process" # 默认也使用进程模式,因为通常更可靠
|
198
|
+
|
199
|
+
# Windows通常使用超线程,所以我们可以使用逻辑核心数
|
200
|
+
logical_cores = psutil.cpu_count(logical=True) or 1
|
201
|
+
|
202
|
+
# Windows建议使用更少的进程以减少开销
|
203
|
+
if logical_cores > 4:
|
204
|
+
max_workers = logical_cores - 1
|
205
|
+
else:
|
206
|
+
max_workers = max(1, logical_cores)
|
207
|
+
|
208
|
+
# macOS平台优化
|
209
|
+
elif self.platform == "macos":
|
210
|
+
mode = "process"
|
211
|
+
logical_cores = psutil.cpu_count(logical=True) or 1
|
212
|
+
max_workers = max(1, logical_cores - 1)
|
213
|
+
|
214
|
+
# 未知平台的保守设置
|
215
|
+
else:
|
216
|
+
mode = "process"
|
217
|
+
max_workers = max(1, (psutil.cpu_count(logical=True) or 2) - 1)
|
218
|
+
|
219
|
+
return mode, max_workers
|
220
|
+
|
221
|
+
def _is_in_container(self):
|
222
|
+
"""检测是否在容器环境中运行"""
|
223
|
+
# 检查常见的容器环境指标
|
224
|
+
if os.path.exists("/.dockerenv"):
|
225
|
+
return True
|
226
|
+
|
227
|
+
try:
|
228
|
+
with open("/proc/1/cgroup", "rt") as f:
|
229
|
+
return any(("docker" in line or "kubepods" in line) for line in f)
|
230
|
+
except Exception:
|
231
|
+
pass
|
232
|
+
|
233
|
+
return False
|
234
|
+
|
235
|
+
def _setup_linux_optimizations(self):
|
236
|
+
"""设置Linux特定的优化参数"""
|
237
|
+
try:
|
238
|
+
# 在Linux上,选择最适合的多进程上下文
|
239
|
+
# fork: 最快但可能会导致多线程程序出现问题
|
240
|
+
# spawn: 更安全但更慢
|
241
|
+
# forkserver: 中间解决方案
|
242
|
+
|
243
|
+
# 根据应用程序特性选择合适的上下文
|
244
|
+
self.mp_context = mp.get_context("fork")
|
245
|
+
|
246
|
+
# 设置进程初始化函数来设置CPU亲和性
|
247
|
+
self.worker_init_func = self._linux_worker_init
|
248
|
+
|
249
|
+
except Exception as e:
|
250
|
+
logging.warning(f"Failed to set Linux optimizations: {e}")
|
251
|
+
self.mp_context = None
|
252
|
+
|
253
|
+
def _setup_windows_optimizations(self):
|
254
|
+
"""设置Windows特定的优化参数"""
|
255
|
+
# Windows优化参数
|
256
|
+
# 进程创建和启动开销在Windows上较高,因此增加每批的任务数
|
257
|
+
self.chunk_size = 10
|
258
|
+
# Windows通常不需要特殊的工作进程初始化
|
259
|
+
self.worker_init_func = None
|
260
|
+
|
261
|
+
def _linux_worker_init(self):
|
262
|
+
"""Linux工作进程初始化函数"""
|
263
|
+
try:
|
264
|
+
# 获取当前进程
|
265
|
+
p = psutil.Process()
|
266
|
+
|
267
|
+
# 设置进程优先级为稍低于正常,以避免争抢重要系统资源
|
268
|
+
p.nice(10)
|
269
|
+
|
270
|
+
# 尝试设置CPU亲和性以提高缓存局部性
|
271
|
+
# 这里我们不设置特定的CPU核心,让系统调度,因为手动设置可能导致不平衡
|
272
|
+
|
273
|
+
# 设置进程I/O优先级
|
274
|
+
# 需要root权限,所以只是尝试一下
|
275
|
+
try:
|
276
|
+
os.system(f"ionice -c 2 -n 4 -p {os.getpid()} > /dev/null 2>&1")
|
277
|
+
except Exception:
|
278
|
+
pass
|
279
|
+
|
280
|
+
except Exception as e:
|
281
|
+
logging.debug(f"Worker initialization warning (non-critical): {e}")
|
282
|
+
pass # 失败不中断程序运行
|
283
|
+
|
284
|
+
def _get_default_chunk_size(self):
|
285
|
+
"""获取默认任务分块大小"""
|
286
|
+
if self.platform == "linux":
|
287
|
+
# Linux下进程创建较快,可以使用较小的块大小
|
288
|
+
return 5
|
289
|
+
elif self.platform == "windows":
|
290
|
+
# Windows下进程创建较慢,使用较大的块大小
|
291
|
+
return 10
|
292
|
+
else:
|
293
|
+
return 5
|
294
|
+
|
295
|
+
@property
|
296
|
+
def executor(self):
|
297
|
+
"""懒加载并重用执行器"""
|
298
|
+
if self._executor is None and self.reuse_pool:
|
299
|
+
kwargs = {}
|
300
|
+
if self.mode == "process" and self.mp_context:
|
301
|
+
kwargs["mp_context"] = self.mp_context
|
302
|
+
|
303
|
+
if self.worker_init_func and self.mode == "process":
|
304
|
+
kwargs["initializer"] = self.worker_init_func
|
305
|
+
|
306
|
+
self._executor = self.executor_class(max_workers=self.max_workers, **kwargs)
|
307
|
+
return self._executor
|
308
|
+
|
309
|
+
@contextlib.contextmanager
|
310
|
+
def get_executor(self):
|
311
|
+
"""获取执行器的上下文管理器"""
|
312
|
+
if self.reuse_pool and self._executor:
|
313
|
+
yield self._executor
|
314
|
+
else:
|
315
|
+
kwargs = {}
|
316
|
+
if self.mode == "process" and self.mp_context:
|
317
|
+
kwargs["mp_context"] = self.mp_context
|
318
|
+
|
319
|
+
if self.worker_init_func and self.mode == "process":
|
320
|
+
kwargs["initializer"] = self.worker_init_func
|
321
|
+
|
322
|
+
with self.executor_class(max_workers=self.max_workers, **kwargs) as executor:
|
323
|
+
yield executor
|
324
|
+
|
325
|
+
def run(self, func, param_list, chunk_size=None, fallback_on_failure=True):
|
326
|
+
"""
|
327
|
+
并行执行函数
|
328
|
+
|
329
|
+
Args:
|
330
|
+
func (callable): 要执行的函数
|
331
|
+
param_list (list): 参数元组列表
|
332
|
+
chunk_size (int, optional): 任务分块大小,None表示使用默认值
|
333
|
+
fallback_on_failure (bool): 如果主执行模式失败,是否尝试其他模式
|
334
|
+
|
335
|
+
Returns:
|
336
|
+
list: 函数执行结果
|
337
|
+
"""
|
338
|
+
if not callable(func):
|
339
|
+
raise ValueError("func must be callable.")
|
340
|
+
if not isinstance(param_list, list):
|
341
|
+
raise ValueError("param_list must be a list.")
|
342
|
+
|
343
|
+
# 空列表直接返回
|
344
|
+
if not param_list:
|
345
|
+
return []
|
346
|
+
|
347
|
+
# 使用默认分块大小或自定义大小
|
348
|
+
effective_chunk_size = chunk_size or self.chunk_size
|
349
|
+
|
350
|
+
# 任务分块处理
|
351
|
+
if effective_chunk_size and len(param_list) > effective_chunk_size * 2:
|
352
|
+
return self._run_chunked(func, param_list, effective_chunk_size)
|
353
|
+
|
354
|
+
try:
|
355
|
+
return self._execute(func, param_list)
|
356
|
+
except Exception as e:
|
357
|
+
if fallback_on_failure:
|
358
|
+
logging.warning(f"Execution failed with {self.mode} mode: {e}. Trying fallback...")
|
359
|
+
# 如果当前模式失败,尝试其他模式
|
360
|
+
old_mode = self.mode
|
361
|
+
self.mode = "thread" if old_mode == "process" else "process"
|
362
|
+
self.executor_class = ProcessPoolExecutor if self.mode == "process" else ThreadPoolExecutor
|
363
|
+
self._executor = None # 重置执行器
|
364
|
+
|
365
|
+
try:
|
366
|
+
results = self._execute(func, param_list)
|
367
|
+
logging.info(f"Fallback to {self.mode} mode succeeded.")
|
368
|
+
return results
|
369
|
+
except Exception as e2:
|
370
|
+
logging.error(f"Fallback also failed: {e2}")
|
371
|
+
# 恢复原始模式
|
372
|
+
self.mode = old_mode
|
373
|
+
self.executor_class = ProcessPoolExecutor if self.mode == "process" else ThreadPoolExecutor
|
374
|
+
self._executor = None
|
375
|
+
raise
|
376
|
+
else:
|
377
|
+
raise
|
378
|
+
|
379
|
+
def _execute(self, func, param_list):
|
380
|
+
"""内部执行方法"""
|
381
|
+
results = [None] * len(param_list)
|
382
|
+
logging.info("Starting parallel execution in %s mode with %d workers.", self.mode, self.max_workers)
|
383
|
+
|
384
|
+
start_time = time.time()
|
385
|
+
|
386
|
+
with self.get_executor() as executor:
|
387
|
+
future_to_index = {executor.submit(func, *params): idx for idx, params in enumerate(param_list)}
|
388
|
+
|
389
|
+
for future in as_completed(future_to_index):
|
390
|
+
idx = future_to_index[future]
|
391
|
+
try:
|
392
|
+
# 添加超时保护
|
393
|
+
results[idx] = future.result(timeout=self.timeout_per_task)
|
394
|
+
except Exception as e:
|
395
|
+
logging.error("Task %d failed with error: %s", idx, e)
|
396
|
+
results[idx] = e
|
397
|
+
|
398
|
+
elapsed = time.time() - start_time
|
399
|
+
logging.info("Parallel execution completed in %.2f seconds.", elapsed)
|
400
|
+
return results
|
401
|
+
|
402
|
+
def _run_chunked(self, func, param_list, chunk_size):
|
403
|
+
"""处理大量小任务的批处理执行"""
|
404
|
+
|
405
|
+
def process_chunk(chunk):
|
406
|
+
return [func(*params) for params in chunk]
|
407
|
+
|
408
|
+
# 将参数列表分成多个块
|
409
|
+
chunks = [param_list[i : i + chunk_size] for i in range(0, len(param_list), chunk_size)]
|
410
|
+
|
411
|
+
logging.info(f"Processing {len(param_list)} tasks in {len(chunks)} chunks of size ~{chunk_size}")
|
412
|
+
|
413
|
+
chunk_results = self._execute(process_chunk, [(chunk,) for chunk in chunks])
|
414
|
+
|
415
|
+
# 将块结果展平成单个结果列表
|
416
|
+
return [result for sublist in chunk_results if isinstance(sublist, list) for result in sublist]
|
417
|
+
|
418
|
+
def map(self, func, *iterables, timeout=None, chunk_size=None):
|
419
|
+
"""
|
420
|
+
类似于内置map函数的并行版本
|
421
|
+
|
422
|
+
Args:
|
423
|
+
func: 要应用于每个元素的函数
|
424
|
+
*iterables: 一个或多个可迭代对象
|
425
|
+
timeout: 每个任务的超时时间
|
426
|
+
chunk_size: 任务分块大小
|
427
|
+
|
428
|
+
Returns:
|
429
|
+
生成器,产生的结果与输入顺序相同
|
430
|
+
"""
|
431
|
+
# 将zip后的可迭代对象转换为参数元组列表
|
432
|
+
param_list = [(args,) for args in zip(*iterables)]
|
433
|
+
|
434
|
+
# 临时存储超时设置
|
435
|
+
original_timeout = self.timeout_per_task
|
436
|
+
if timeout:
|
437
|
+
self.timeout_per_task = timeout
|
438
|
+
|
439
|
+
try:
|
440
|
+
results = self.run(lambda x: func(x), param_list, chunk_size=chunk_size)
|
441
|
+
for r in results:
|
442
|
+
yield r
|
443
|
+
finally:
|
444
|
+
# 恢复原超时设置
|
445
|
+
self.timeout_per_task = original_timeout
|
446
|
+
|
447
|
+
def __del__(self):
|
448
|
+
"""确保资源被正确释放"""
|
449
|
+
self.shutdown()
|
450
|
+
|
451
|
+
def shutdown(self):
|
452
|
+
"""显式关闭执行器"""
|
453
|
+
if self._executor:
|
454
|
+
try:
|
455
|
+
self._executor.shutdown(wait=True)
|
456
|
+
except Exception:
|
457
|
+
pass
|
458
|
+
self._executor = None
|
459
|
+
|
460
|
+
def imap(self, func, *iterables, timeout=None, chunk_size=None):
|
461
|
+
"""
|
462
|
+
类似concurrent.futures.Executor.map的接口,但返回迭代器
|
463
|
+
"""
|
464
|
+
return self.map(func, *iterables, timeout=timeout, chunk_size=chunk_size)
|
465
|
+
|
466
|
+
def imap_unordered(self, func, *iterables, timeout=None, chunk_size=None):
|
467
|
+
"""
|
468
|
+
类似multiprocessing.Pool.imap_unordered的接口,结果可能乱序返回
|
469
|
+
"""
|
470
|
+
# 将zip后的可迭代对象转换为参数元组列表
|
471
|
+
param_list = [(args,) for args in zip(*iterables)]
|
472
|
+
|
473
|
+
# 空列表直接返回
|
474
|
+
if not param_list:
|
475
|
+
return
|
476
|
+
|
477
|
+
# 临时存储超时设置
|
478
|
+
original_timeout = self.timeout_per_task
|
479
|
+
if timeout:
|
480
|
+
self.timeout_per_task = timeout
|
481
|
+
|
482
|
+
try:
|
483
|
+
# 使用默认分块大小或自定义大小
|
484
|
+
effective_chunk_size = chunk_size or self.chunk_size
|
485
|
+
|
486
|
+
# 任务分块处理
|
487
|
+
if effective_chunk_size and len(param_list) > effective_chunk_size * 2:
|
488
|
+
chunks = [param_list[i : i + effective_chunk_size] for i in range(0, len(param_list), effective_chunk_size)]
|
489
|
+
|
490
|
+
with self.get_executor() as executor:
|
491
|
+
futures = [executor.submit(self._process_chunk_for_imap, func, chunk) for chunk in chunks]
|
492
|
+
|
493
|
+
for future in as_completed(futures):
|
494
|
+
try:
|
495
|
+
chunk_results = future.result(timeout=self.timeout_per_task)
|
496
|
+
for result in chunk_results:
|
497
|
+
yield result
|
498
|
+
except Exception as e:
|
499
|
+
logging.error(f"Chunk processing failed: {e}")
|
500
|
+
else:
|
501
|
+
with self.get_executor() as executor:
|
502
|
+
futures = [executor.submit(func, *params) for params in param_list]
|
503
|
+
|
504
|
+
for future in as_completed(futures):
|
505
|
+
try:
|
506
|
+
yield future.result(timeout=self.timeout_per_task)
|
507
|
+
except Exception as e:
|
508
|
+
logging.error(f"Task failed: {e}")
|
509
|
+
yield e
|
510
|
+
finally:
|
511
|
+
# 恢复原超时设置
|
512
|
+
self.timeout_per_task = original_timeout
|
513
|
+
|
514
|
+
def _process_chunk_for_imap(self, func, chunk):
|
515
|
+
"""处理imap_unordered的数据块"""
|
516
|
+
return [func(*params) for params in chunk]
|
517
|
+
|
518
|
+
def starmap(self, func, iterable, timeout=None, chunk_size=None):
|
519
|
+
"""
|
520
|
+
类似于内置starmap函数的并行版本
|
521
|
+
|
522
|
+
Args:
|
523
|
+
func: 要应用于每个元素的函数
|
524
|
+
iterable: 可迭代对象,每个元素是函数参数的元组
|
525
|
+
timeout: 每个任务的超时时间
|
526
|
+
chunk_size: 任务分块大小
|
527
|
+
|
528
|
+
Returns:
|
529
|
+
生成器,产生结果
|
530
|
+
"""
|
531
|
+
|
532
|
+
# 将每个元素转换为单参数函数调用
|
533
|
+
def wrapper(args):
|
534
|
+
return func(*args)
|
535
|
+
|
536
|
+
# 使用map实现
|
537
|
+
return self.map(wrapper, iterable, timeout=timeout, chunk_size=chunk_size)
|
538
|
+
|
539
|
+
def gather(self, funcs_and_args):
|
540
|
+
"""
|
541
|
+
并行执行多个不同的函数,类似于asyncio.gather
|
542
|
+
|
543
|
+
Args:
|
544
|
+
funcs_and_args: 列表,每个元素是(func, args)元组,
|
545
|
+
其中args是要传递给func的参数元组
|
546
|
+
|
547
|
+
Returns:
|
548
|
+
list: 函数执行结果,顺序与输入相同
|
549
|
+
"""
|
550
|
+
if not isinstance(funcs_and_args, list):
|
551
|
+
raise ValueError("funcs_and_args must be a list of (func, args) tuples")
|
552
|
+
|
553
|
+
def wrapper(func_and_args):
|
554
|
+
func, args = func_and_args
|
555
|
+
return func(*args)
|
556
|
+
|
557
|
+
return self.run(wrapper, [(item,) for item in funcs_and_args])
|
558
|
+
|
559
|
+
|
87
560
|
if __name__ == "__main__":
|
88
561
|
_example()
|
89
562
|
# 也可以不要装饰器,直接运行没啥问题,就是避免在ipynb中使用,最好使用ipynb,或者把这个函数放到一个独立的py文件中运行
|
@@ -1,6 +1,6 @@
|
|
1
|
-
Metadata-Version: 2.
|
1
|
+
Metadata-Version: 2.4
|
2
2
|
Name: oafuncs
|
3
|
-
Version: 0.0.97.
|
3
|
+
Version: 0.0.97.6
|
4
4
|
Summary: Oceanic and Atmospheric Functions
|
5
5
|
Home-page: https://github.com/Industry-Pays/OAFuncs
|
6
6
|
Author: Kun Liu
|
@@ -32,6 +32,7 @@ Requires-Dist: geopandas
|
|
32
32
|
Requires-Dist: Cartopy
|
33
33
|
Requires-Dist: rasterio
|
34
34
|
Requires-Dist: salem
|
35
|
+
Requires-Dist: psutil
|
35
36
|
Dynamic: author
|
36
37
|
Dynamic: author-email
|
37
38
|
Dynamic: classifier
|
@@ -39,6 +40,7 @@ Dynamic: description
|
|
39
40
|
Dynamic: description-content-type
|
40
41
|
Dynamic: home-page
|
41
42
|
Dynamic: license
|
43
|
+
Dynamic: license-file
|
42
44
|
Dynamic: requires-dist
|
43
45
|
Dynamic: requires-python
|
44
46
|
Dynamic: summary
|
@@ -1,20 +1,24 @@
|
|
1
|
-
oafuncs/__init__.py,sha256=
|
1
|
+
oafuncs/__init__.py,sha256=T_-VtnWWllV3Q91twT5Yt2sUapeA051QbPNnBxmg9nw,1456
|
2
2
|
oafuncs/oa_cmap.py,sha256=OfJ2DOBOGX5X1GihE6CPcTV8bAFdDqJSXV7vbqUgV9s,7552
|
3
3
|
oafuncs/oa_data.py,sha256=32elvoBHibTunDXAK9ptQ-e-xrXxKM4GrPiMqWNv_lE,12169
|
4
|
-
oafuncs/
|
5
|
-
oafuncs/
|
4
|
+
oafuncs/oa_date.py,sha256=--3uXYq3_n9HBJbY1io1H1PE-FyiVxbQCyFUcudxQks,3210
|
5
|
+
oafuncs/oa_draw.py,sha256=d23R6OEco7EbvcrL5YsnrSupnKKUn55hLw1B_d-kxOg,12308
|
6
|
+
oafuncs/oa_file.py,sha256=nYTsMataaC790lSqxjuMB4uVqUJz_tMm_kRve4CHql4,17235
|
6
7
|
oafuncs/oa_help.py,sha256=loyzTbjU_0VpSIBvAEUA_tqxG8MVsO0xFE_2hgQ3zMw,4188
|
7
|
-
oafuncs/oa_nc.py,sha256=
|
8
|
+
oafuncs/oa_nc.py,sha256=eTioHFT8eeg3T7S5wFUiFm1zU1w8Fo11Un8fjBaPhuY,20860
|
8
9
|
oafuncs/oa_python.py,sha256=Q-6UGGw_dJff7Ef8i87fsLPoGeHV5jBzfb-7HP4THR0,4018
|
9
10
|
oafuncs/_data/OAFuncs.png,sha256=y1_x-mUP3gFjcy6m8FqfvQO_HgjzPhQKfXjnSHjslZE,3436152
|
10
11
|
oafuncs/_data/hycom_3hourly.png,sha256=azt_uPcXtl_8CSKRLLPCIf5pPrcxMiOzvoFQnwb0zUo,12411415
|
12
|
+
oafuncs/_script/auto_optimized_parallel_executor.py,sha256=4QaEk9AM-IneHm8KKSQ6MjSLNSaAWM4AQ-8OWXYdsaI,17300
|
13
|
+
oafuncs/_script/parallel_example_usage.py,sha256=uLvE7iwkMn9Cyq6-wk5_RpbQk7PXM9d16-26lTknW9s,2646
|
11
14
|
oafuncs/_script/plot_dataset.py,sha256=4PEMXI7NUgLMKwo-33y_AUrmUYS7BzmInCDyFalaZSk,13136
|
15
|
+
oafuncs/_script/replace_file_concent.py,sha256=eCFZjnZcwyRvy6b4mmIfBna-kylSZTyJRfgXd6DdCjk,5982
|
12
16
|
oafuncs/oa_down/User_Agent-list.txt,sha256=pazxSip8_lphEBOPHG902zmIBUg8sBKXgmqp_g6j_E4,661062
|
13
17
|
oafuncs/oa_down/__init__.py,sha256=kRX5eTUCbAiz3zTaQM1501paOYS_3fizDN4Pa0mtNUA,585
|
14
|
-
oafuncs/oa_down/hycom_3hourly.py,sha256=
|
15
|
-
oafuncs/oa_down/hycom_3hourly_20250129.py,sha256=
|
18
|
+
oafuncs/oa_down/hycom_3hourly.py,sha256=Lw7BgK-6hHp93S2D1nf_k-7oKwDxTV8yYP7ZosfZRh0,65309
|
19
|
+
oafuncs/oa_down/hycom_3hourly_20250129.py,sha256=wVOSR-n-7OZYNsLujV0XeMrNwZFuF-g6d7cSrGIReBs,65555
|
16
20
|
oafuncs/oa_down/idm.py,sha256=XfYCNnQWADxOhhJd-T8sNYN0nGiRrAs7zbQcsB5-UmI,1668
|
17
|
-
oafuncs/oa_down/literature.py,sha256=
|
21
|
+
oafuncs/oa_down/literature.py,sha256=D1ZpHULQ4OJ2WXG2TWgBovuucPrfSeyC0rr3P2vSKjo,11332
|
18
22
|
oafuncs/oa_down/test_ua.py,sha256=0IQq3NjqfNr7KkyjS_U-a4mYu-r-E7gzawwo4IfEa6Y,10851
|
19
23
|
oafuncs/oa_down/user_agent.py,sha256=TsPcAxFmMTYAEHRFjurI1bQBJfDhcA70MdHoUPwQmks,785
|
20
24
|
oafuncs/oa_model/__init__.py,sha256=__ImltHkP1bSsIpsmKpDE8QwwA-2Z8K7mZUHGGcRdro,484
|
@@ -26,12 +30,11 @@ oafuncs/oa_sign/__init__.py,sha256=QKqTFrJDFK40C5uvk48GlRRbGFzO40rgkYwu6dYxatM,5
|
|
26
30
|
oafuncs/oa_sign/meteorological.py,sha256=mLbupsZSq427HTfVbZMvIlFzDHwSzQAbK3X19o8anFY,6525
|
27
31
|
oafuncs/oa_sign/ocean.py,sha256=xrW-rWD7xBWsB5PuCyEwQ1Q_RDKq2KCLz-LOONHgldU,5932
|
28
32
|
oafuncs/oa_sign/scientific.py,sha256=a4JxOBgm9vzNZKpJ_GQIQf7cokkraV5nh23HGbmTYKw,5064
|
29
|
-
oafuncs/oa_tool/__init__.py,sha256=
|
33
|
+
oafuncs/oa_tool/__init__.py,sha256=AvrCNR2-xad9ZRjthIdAoSk8UX4vOpEWLg6CV1NQNKc,161
|
30
34
|
oafuncs/oa_tool/email.py,sha256=4lJxV_KUzhxgLYfVwYTqp0qxRugD7fvsZkXDe5WkUKo,3052
|
31
|
-
oafuncs/oa_tool/parallel.py,sha256=
|
32
|
-
oafuncs/
|
33
|
-
oafuncs-0.0.97.
|
34
|
-
oafuncs-0.0.97.
|
35
|
-
oafuncs-0.0.97.
|
36
|
-
oafuncs-0.0.97.
|
37
|
-
oafuncs-0.0.97.4.dist-info/RECORD,,
|
35
|
+
oafuncs/oa_tool/parallel.py,sha256=LBFWEKPcILVCbfSulETJE4wGPiOw1P_Fl9DzjYoCqgk,21844
|
36
|
+
oafuncs-0.0.97.6.dist-info/licenses/LICENSE.txt,sha256=rMtLpVg8sKiSlwClfR9w_Dd_5WubTQgoOzE2PDFxzs4,1074
|
37
|
+
oafuncs-0.0.97.6.dist-info/METADATA,sha256=3S9GNpCs63Ef6sJ_IcIoizHyQt10hprfwrv2YwxkmZs,4204
|
38
|
+
oafuncs-0.0.97.6.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
|
39
|
+
oafuncs-0.0.97.6.dist-info/top_level.txt,sha256=bgC35QkXbN4EmPHEveg_xGIZ5i9NNPYWqtJqaKqTPsQ,8
|
40
|
+
oafuncs-0.0.97.6.dist-info/RECORD,,
|
oafuncs/oa_tool/time.py
DELETED
@@ -1,22 +0,0 @@
|
|
1
|
-
#!/usr/bin/env python
|
2
|
-
# coding=utf-8
|
3
|
-
"""
|
4
|
-
Author: Liu Kun && 16031215@qq.com
|
5
|
-
Date: 2025-03-09 13:55:46
|
6
|
-
LastEditors: Liu Kun && 16031215@qq.com
|
7
|
-
LastEditTime: 2025-03-09 13:55:46
|
8
|
-
FilePath: \\Python\\My_Funcs\\OAFuncs\\oafuncs\\oa_tool\\time.py
|
9
|
-
Description:
|
10
|
-
EditPlatform: vscode
|
11
|
-
ComputerInfo: XPS 15 9510
|
12
|
-
SystemInfo: Windows 11
|
13
|
-
Python Version: 3.12
|
14
|
-
"""
|
15
|
-
|
16
|
-
|
17
|
-
import calendar
|
18
|
-
|
19
|
-
__all__ = ["get_days"]
|
20
|
-
|
21
|
-
def get_days(year, month):
|
22
|
-
return calendar.monthrange(year, month)[1]
|
File without changes
|
File without changes
|