oafuncs 0.0.98.35__py3-none-any.whl → 0.0.98.37__py3-none-any.whl

oafuncs/_script/parallel.py

@@ -20,7 +20,7 @@ class ParallelExecutor:
         self,
         max_workers: Optional[int] = None,
         chunk_size: Optional[int] = None,
-        mem_per_process: float = 1.0,  # GB
+        mem_per_process: float = 3.0,  # GB
         timeout_per_task: int = 3600,
         max_retries: int = 3,
     ):
@@ -130,18 +130,50 @@ class ParallelExecutor:
             return self._chunked_execution(func, params, chunk_size)
 
         results = [None] * len(params)
+        
+        # 创建进度条 - 使用 range 作为占位符，手动控制进度
+        progress_bar = pbar(
+            iterable=range(len(params)),  # 使用 range 作为占位符
+            description="Parallel Tasks", 
+            total=len(params),
+            completed=0,
+            next_line=False,
+        )
+        # 手动开始任务
+        progress_bar.task.start()
+        
         with self._get_executor() as executor:
             futures = {executor.submit(func, *args): idx for idx, args in enumerate(params)}
-            for future in pbar(as_completed(futures), "Parallel Tasks", total=len(futures)):
+            
+            for future in as_completed(futures):
                 idx = futures[future]
                 try:
                     results[idx] = future.result(timeout=self.timeout_per_task)
                 except Exception as e:
                     results[idx] = self._handle_error(e, func, params[idx])
+                
+                # 实时更新进度条
+                progress_bar.update(1)
+                progress_bar.refresh()
+        print('\n') # 结束进度条输出
         return results
 
     def _chunked_execution(self, func: Callable, params: List[Tuple], chunk_size: int) -> List[Any]:
+        from oafuncs.oa_tool import pbar
+        
         results = []
+        chunk_count = (len(params) + chunk_size - 1) // chunk_size
+        
+        # 为分块执行创建进度条
+        progress_bar = pbar(
+            iterable=range(chunk_count),  # 使用 range 作为占位符
+            description="Parallel Chunks",
+            total=chunk_count,
+            completed=0,
+            next_line=False,
+        )
+        progress_bar.task.start()
+        
         with self._get_executor() as executor:
             futures = []
             for i in range(0, len(params), chunk_size):
@@ -154,6 +186,11 @@ class ParallelExecutor:
                 except Exception as e:
                     logging.error(f"Chunk failed: {e}")
                     results.extend([None] * chunk_size)
+                
+                # 更新分块进度
+                progress_bar.update(1)
+                progress_bar.refresh()
+        print('\n')
         return results
 
     @staticmethod

oafuncs/_script/parallel_bak.py

@@ -0,0 +1,230 @@
+import logging
+import multiprocessing as mp
+import platform
+import threading
+import time
+from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed
+from typing import Any, Callable, Dict, List, Optional, Tuple
+
+
+import psutil
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+
+__all__ = ["ParallelExecutor"]
+
+
+class ParallelExecutor:
+    
+    def __init__(
+        self,
+        max_workers: Optional[int] = None,
+        chunk_size: Optional[int] = None,
+        mem_per_process: float = 3.0,  # GB
+        timeout_per_task: int = 3600,
+        max_retries: int = 3,
+    ):
+        self.platform = self._detect_platform()
+        self.mem_per_process = mem_per_process
+        self.timeout_per_task = timeout_per_task
+        self.max_retries = max_retries
+        self.running = True
+        self.task_history = []
+        self._executor = None
+        self._shutdown_called = False
+
+        self.mode, default_workers = self._determine_optimal_settings()
+        self.max_workers = max_workers or default_workers
+        self.chunk_size = chunk_size or self._get_default_chunk_size()
+
+        self._init_platform_settings()
+        self._start_resource_monitor()
+
+        logging.info(f"Initialized {self.__class__.__name__} on {self.platform} (mode={self.mode}, workers={self.max_workers})")
+
+    def _detect_platform(self) -> str:
+        system = platform.system().lower()
+        if system == "linux":
+            return "wsl" if "microsoft" in platform.release().lower() else "linux"
+        return system
+
+    def _init_platform_settings(self):
+        if self.platform in ["linux", "wsl"]:
+            self.mp_context = mp.get_context("fork")
+        elif self.platform == "windows":
+            mp.set_start_method("spawn", force=True)
+            self.mp_context = mp.get_context("spawn")
+        else:
+            self.mp_context = None
+
+    def _determine_optimal_settings(self) -> Tuple[str, int]:
+        logical_cores = psutil.cpu_count(logical=True) or 1
+        available_mem = psutil.virtual_memory().available / 1024**3  # GB
+
+        mem_limit = max(1, int(available_mem / self.mem_per_process))
+        return ("process", min(logical_cores, mem_limit))
+
+    def _get_default_chunk_size(self) -> int:
+        return max(10, 100 // (psutil.cpu_count() or 1))
+
+    def _start_resource_monitor(self):
+        def monitor():
+            threshold = self.mem_per_process * 1024**3
+            while self.running:
+                try:
+                    if psutil.virtual_memory().available < threshold:
+                        self._scale_down_workers()
+                    time.sleep(1)
+                except Exception as e:
+                    logging.error(f"Resource monitor error: {e}")
+
+        threading.Thread(target=monitor, daemon=True).start()
+
+    def _scale_down_workers(self):
+        if self.max_workers > 1:
+            new_count = self.max_workers - 1
+            logging.warning(f"Scaling down workers from {self.max_workers} to {new_count}")
+            self.max_workers = new_count
+            self._restart_executor()
+
+    def _restart_executor(self):
+        if self._executor:
+            self._executor.shutdown(wait=False)
+            self._executor = None
+
+    def _get_executor(self):
+        if not self._executor:
+            Executor = ThreadPoolExecutor if self.mode == "thread" else ProcessPoolExecutor
+            self._executor = Executor(max_workers=self.max_workers, mp_context=self.mp_context if self.mode == "process" else None)
+        return self._executor
+
+    def run(self, func: Callable, params: List[Tuple], chunk_size: Optional[int] = None) -> List[Any]:
+        chunk_size = chunk_size or self.chunk_size
+        try:
+            for retry in range(self.max_retries + 1):
+                try:
+                    start_time = time.monotonic()
+                    results = self._execute_batch(func, params, chunk_size)
+                    self._update_settings(time.monotonic() - start_time, len(params))
+                    return results
+                except Exception as e:
+                    logging.error(f"Attempt {retry + 1} failed: {e}")
+                    self._handle_failure()
+            raise RuntimeError(f"Failed after {self.max_retries} retries")
+        finally:
+            # 仅关闭当前 executor，保留资源监控等运行状态
+            if self._executor:
+                try:
+                    self._executor.shutdown(wait=True)
+                except Exception as e:
+                    logging.error(f"Executor shutdown error: {e}")
+                finally:
+                    self._executor = None
+
+    def _execute_batch(self, func: Callable, params: List[Tuple], chunk_size: int) -> List[Any]:
+        from oafuncs.oa_tool import pbar
+        if not params:
+            return []
+
+        if len(params) > chunk_size * 2:
+            return self._chunked_execution(func, params, chunk_size)
+
+        results = [None] * len(params)
+        with self._get_executor() as executor:
+            futures = {executor.submit(func, *args): idx for idx, args in enumerate(params)}
+            for future in pbar(as_completed(futures), "Parallel Tasks", total=len(futures)):
+                idx = futures[future]
+                try:
+                    results[idx] = future.result(timeout=self.timeout_per_task)
+                except Exception as e:
+                    results[idx] = self._handle_error(e, func, params[idx])
+        return results
+
+    def _chunked_execution(self, func: Callable, params: List[Tuple], chunk_size: int) -> List[Any]:
+        results = []
+        with self._get_executor() as executor:
+            futures = []
+            for i in range(0, len(params), chunk_size):
+                chunk = params[i : i + chunk_size]
+                futures.append(executor.submit(self._process_chunk, func, chunk))
+
+            for future in as_completed(futures):
+                try:
+                    results.extend(future.result(timeout=self.timeout_per_task))
+                except Exception as e:
+                    logging.error(f"Chunk failed: {e}")
+                    results.extend([None] * chunk_size)
+        return results
+
+    @staticmethod
+    def _process_chunk(func: Callable, chunk: List[Tuple]) -> List[Any]:
+        return [func(*args) for args in chunk]
+
+    def _update_settings(self, duration: float, task_count: int):
+        self.task_history.append((duration, task_count))
+        self.chunk_size = max(5, min(100, self.chunk_size + (1 if duration < 5 else -1)))
+
+    def _handle_error(self, error: Exception, func: Callable, args: Tuple) -> Any:
+        if isinstance(error, TimeoutError):
+            logging.warning(f"Timeout processing {func.__name__}{args}")
+        elif isinstance(error, MemoryError):
+            logging.warning("Memory error detected")
+            self._scale_down_workers()
+        else:
+            logging.error(f"Error processing {func.__name__}{args}: {str(error)}")
+        return None
+
+    def _handle_failure(self):
+        if self.max_workers > 2:
+            self.max_workers = max(1, self.max_workers // 2)
+            self._restart_executor()
+
+    def shutdown(self):
+        if self._shutdown_called:
+            return
+        self._shutdown_called = True
+        self.running = False
+        # 基类不再打印日志，由子类统一处理
+        if self._executor:
+            try:
+                self._executor.shutdown(wait=True)
+            except Exception as e:
+                logging.error(f"Shutdown error: {e}")
+            finally:
+                self._executor = None
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc_info):
+        self.shutdown()
+
+    def get_stats(self) -> Dict[str, Any]:
+        stats = {
+            "platform": self.platform,
+            "mode": self.mode,
+            "workers": self.max_workers,
+            "chunk_size": self.chunk_size,
+            "total_tasks": sum(count for _, count in self.task_history),
+        }
+        if self.task_history:
+            total_time = sum(time for time, _ in self.task_history)
+            stats["avg_task_throughput"] = stats["total_tasks"] / total_time if total_time else 0
+        return stats
+
+
+def _test_func(a, b):
+    time.sleep(0.01)
+    return a + b
+
+
+if __name__ == "__main__":
+    params = [(i, i * 2) for i in range(1000)]
+
+    with ParallelExecutor() as executor:
+        results = executor.run(_test_func, params)
+
+    # print("Results:", results)
+
+    print(f"Processed {len(results)} tasks")
+    print("Execution stats:", executor.get_stats())

oafuncs/oa_down/hycom_3hourly.py

@@ -98,6 +98,18 @@ def _get_initial_data():
     data_info["hourly"]["dataset"]["GLBu0.08"]["version"]["93.0"]["classification"] = "var_different"
     data_info["hourly"]["dataset"]["GLBy0.08"]["version"]["93.0"]["classification"] = "var_year_different"
     data_info["hourly"]["dataset"]["ESPC_D"]["version"]["V02"]["classification"] = "single_var_year_different"
+    
+    # lon lat
+    data_info["hourly"]["dataset"]["GLBv0.08"]["version"]["53.X"]["lonlat"] = {'lon_min': -180.00, 'lon_max': 179.92, 'lat_min': -80.0, 'lat_max': 90.0}
+    data_info["hourly"]["dataset"]["GLBv0.08"]["version"]["56.3"]["lonlat"] = {'lon_min': -180.00, 'lon_max': 179.92, 'lat_min': -80.0, 'lat_max': 90.0}
+    data_info["hourly"]["dataset"]["GLBv0.08"]["version"]["57.2"]["lonlat"] = {'lon_min': -180.00, 'lon_max': 179.92, 'lat_min': -80.0, 'lat_max': 90.0}
+    data_info["hourly"]["dataset"]["GLBv0.08"]["version"]["92.8"]["lonlat"] = {'lon_min': 0.00, 'lon_max': 359.92, 'lat_min': -80.0, 'lat_max': 90.0}
+    data_info["hourly"]["dataset"]["GLBv0.08"]["version"]["57.7"]["lonlat"] = {'lon_min': -180.00, 'lon_max': 179.92, 'lat_min': -80.0, 'lat_max': 90.0}
+    data_info["hourly"]["dataset"]["GLBv0.08"]["version"]["92.9"]["lonlat"] = {'lon_min': 0.00, 'lon_max': 359.92, 'lat_min': -80.0, 'lat_max': 90.0}
+    data_info["hourly"]["dataset"]["GLBv0.08"]["version"]["93.0"]["lonlat"] = {'lon_min': 0.00, 'lon_max': 359.92, 'lat_min': -80.0, 'lat_max': 90.0}
+    data_info["hourly"]["dataset"]["GLBu0.08"]["version"]["93.0"]["lonlat"] = {'lon_min': 0.00, 'lon_max': 359.92, 'lat_min': -80.0, 'lat_max': 80.0}
+    data_info["hourly"]["dataset"]["GLBy0.08"]["version"]["93.0"]["lonlat"] = {'lon_min': 0.00, 'lon_max': 359.92, 'lat_min': -80.0, 'lat_max': 90.0}
+    data_info["hourly"]["dataset"]["ESPC_D"]["version"]["V02"]["lonlat"] = {'lon_min': 0.00, 'lon_max': 359.92, 'lat_min': -80.0, 'lat_max': 90.0}
 
     # download info
     # base url
@@ -851,6 +863,14 @@ def _get_submit_url_var(var, depth, level_num, lon_min, lon_max, lat_min, lat_ma
         which_mode = "level"
     else:
         which_mode = "full"
+    if lon_min == 0.0:
+        lon_min = data_info["hourly"]["dataset"][dataset_name]["version"][version_name]["lonlat"]["lon_min"]
+    if lon_max == 359.92:
+        lon_max = data_info["hourly"]["dataset"][dataset_name]["version"][version_name]["lonlat"]["lon_max"]
+    if lat_min == -80.0:
+        lat_min = data_info["hourly"]["dataset"][dataset_name]["version"][version_name]["lonlat"]["lat_min"]
+    if lat_max == 90.0:
+        lat_max = data_info["hourly"]["dataset"][dataset_name]["version"][version_name]["lonlat"]["lat_max"]
     query_dict = _get_query_dict(var, lon_min, lon_max, lat_min, lat_max, download_time, download_time_end, which_mode, depth, level_num)
     submit_url = _get_submit_url(dataset_name, version_name, var, ymdh_str, query_dict)
     return submit_url
@@ -1194,10 +1214,10 @@ if __name__ == "__main__":
         "start_time": "2018010100",
         "end_time": "2019063000",
         "output_dir": r"G:\Data\HYCOM\china_sea\hourly_24",
-        "lon_min": 105,
-        "lon_max": 135,
-        "lat_min": 10,
-        "lat_max": 45,
+        # "lon_min": 105,
+        # "lon_max": 135,
+        # "lat_min": 10,
+        # "lat_max": 45,
         "workers": 1,
         "overwrite": False,
         "depth": None,

oafuncs/oa_tool.py

@@ -37,7 +37,7 @@ class PEx(ParallelExecutor):
         self,
         max_workers: Optional[int] = None,
         chunk_size: Optional[int] = None,
-        mem_per_process: float = 1.5,  # 调大默认内存限制
+        mem_per_process: float = 3.0,  # 调大默认内存限制
         timeout_per_task: int = 7200,  # 延长默认超时时间
         max_retries: int = 5,  # 增加默认重试次数
         progress_callback: Optional[Callable[[int, int], None]] = None,

{oafuncs-0.0.98.35.dist-info → oafuncs-0.0.98.37.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: oafuncs
-Version: 0.0.98.35
+Version: 0.0.98.37
 Summary: Oceanic and Atmospheric Functions
 Home-page: https://github.com/Industry-Pays/OAFuncs
 Author: Kun Liu

{oafuncs-0.0.98.35.dist-info → oafuncs-0.0.98.37.dist-info}/RECORD

@@ -7,7 +7,7 @@ oafuncs/oa_file.py,sha256=fLb0gRhq2AiPl-5ASDHMrx6Z267FmhqNcTV7CdCxTdI,16934
 oafuncs/oa_help.py,sha256=0J5VaZX-cB0c090KxgmktQJBc0o00FsY-4wB8l5y00k,4178
 oafuncs/oa_nc.py,sha256=mKNxQ9jPxfRH7xINyrX7tBhitG5gmOKm6Dn7stk5mdw,15279
 oafuncs/oa_python.py,sha256=xYMQnM0cGq9xUCtcoMpnN0LG5Rc_s94tai5nC6CNJ3E,4831
-oafuncs/oa_tool.py,sha256=Zuaoa92wll0YqXGRf0oF_c7wlATtl7bvjCuLt9VLXp0,8046
+oafuncs/oa_tool.py,sha256=VHx15VqpbzNlVXh0-3nJqcDgLVaECMD1FvxJ_CrV39E,8046
 oafuncs/_data/hycom.png,sha256=MadKs6Gyj5n9-TOu7L4atQfTXtF9dvN9w-tdU9IfygI,10945710
 oafuncs/_data/oafuncs.png,sha256=o3VD7wm-kwDea5E98JqxXl04_78cBX7VcdUt7uQXGiU,3679898
 oafuncs/_script/cprogressbar.py,sha256=nIOs42t6zURLTNjJglavqrI2TKO9GWD0AmZ_DOBmXDU,15949
@@ -16,13 +16,13 @@ oafuncs/_script/email.py,sha256=l5xDgdVj8O5V0J2SwjsHKdUuxOH2jZvwdMO_P0dImHU,2684
 oafuncs/_script/netcdf_merge.py,sha256=tM9ePqLiEsE7eIsNM5XjEYeXwxjYOdNz5ejnEuI7xKw,6066
 oafuncs/_script/netcdf_modify.py,sha256=XDlAEToe_lwfAetkBSENqU5df-wnH7MGuxNTjG1gwHY,4178
 oafuncs/_script/netcdf_write.py,sha256=GvyUyUhzMonzSp3y4pT8ZAfbQrsh5J3dLnmINYJKhuE,21422
-oafuncs/_script/parallel.py,sha256=07-BJVHxXJNlrOrhrSGt7qCZiKWq6dBvNDBA1AANYnI,8861
-oafuncs/_script/parallel_test.py,sha256=0GBqZOX7IaCOKF2t1y8N8YYu53GJ33OkfsWgpvZNqM4,372
+oafuncs/_script/parallel.py,sha256=glEeZEg6HU3q1E6kUF-9k4l__KmIa3KlOglbOUcqubU,10047
+oafuncs/_script/parallel_bak.py,sha256=2ySmYZ9e_PLhhMocWCCFWCYZD3Gs_mxl0HxEzbIuQvA,8861
 oafuncs/_script/plot_dataset.py,sha256=QrA4vOCzWbAJp3hf5YYzgIRUZdJB5_ugepgyT_YfnaY,16327
 oafuncs/_script/replace_file_content.py,sha256=wIwvaISFNYWG58BLZHZP9ZgbC5OhoZ-cpR3y25U1EUM,5601
 oafuncs/oa_down/User_Agent-list.txt,sha256=pHaMlElMvZ8TG4vf4BqkZYKqe0JIGkr4kCN0lM1Y9FQ,514295
 oafuncs/oa_down/__init__.py,sha256=IT6oTqaxuV_mC6AwBut0HtkmnVtEu1MyX0x0oS7TKoA,218
-oafuncs/oa_down/hycom_3hourly.py,sha256=y2ZoT1uCpWRJaOMRXzvW1e9f9lXzTTqw5kYVumzms5Q,55266
+oafuncs/oa_down/hycom_3hourly.py,sha256=dFXSC_5o-Dic616KrLXir4tEHvCiZt8vGKPEYpXFMmA,57356
 oafuncs/oa_down/idm.py,sha256=vAhRjt_Sb-rKhzFShmSf29QcFTqsHpHXCvTSD1uSXyQ,1455
 oafuncs/oa_down/literature.py,sha256=7Qy5OphcjdRwY2uZ5hmmgK36U_QtVmEUSW0vQaxihC8,10960
 oafuncs/oa_down/read_proxy.py,sha256=HQpr-Mwn0Z8ICAuf63NUUY9p05E_uNWyWmOK46-73Ec,2866
@@ -37,8 +37,8 @@ oafuncs/oa_sign/__init__.py,sha256=JSx1fcWpmNhQBvX_Bmq3xysfSkkFMrjbJASxV_V6aqE,1
 oafuncs/oa_sign/meteorological.py,sha256=3MSjy7HTcvz2zsITkjUMr_0Y027Gas1LFE9pk99990k,6110
 oafuncs/oa_sign/ocean.py,sha256=3uYEzaq-27yVy23IQoqy-clhWu1I_fhPFBAQyT-OF4M,5562
 oafuncs/oa_sign/scientific.py,sha256=moIl2MEY4uitbXoD596JmXookXGQtQsS-8_1NBBTx84,4689
-oafuncs-0.0.98.35.dist-info/licenses/LICENSE.txt,sha256=rMtLpVg8sKiSlwClfR9w_Dd_5WubTQgoOzE2PDFxzs4,1074
-oafuncs-0.0.98.35.dist-info/METADATA,sha256=s4W9Zkj2_ALKyPBleO3l791AVeekgC508wE7XNRrpcU,4326
-oafuncs-0.0.98.35.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-oafuncs-0.0.98.35.dist-info/top_level.txt,sha256=bgC35QkXbN4EmPHEveg_xGIZ5i9NNPYWqtJqaKqTPsQ,8
-oafuncs-0.0.98.35.dist-info/RECORD,,
+oafuncs-0.0.98.37.dist-info/licenses/LICENSE.txt,sha256=rMtLpVg8sKiSlwClfR9w_Dd_5WubTQgoOzE2PDFxzs4,1074
+oafuncs-0.0.98.37.dist-info/METADATA,sha256=EuBjhFOpWRAcZeADmTIFmg5FUSholH23O3qn5CsCR5U,4326
+oafuncs-0.0.98.37.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+oafuncs-0.0.98.37.dist-info/top_level.txt,sha256=bgC35QkXbN4EmPHEveg_xGIZ5i9NNPYWqtJqaKqTPsQ,8
+oafuncs-0.0.98.37.dist-info/RECORD,,

oafuncs/_script/parallel_test.py

@@ -1,14 +0,0 @@
-#!/usr/bin/env python
-# coding=utf-8
-"""
-Author: Liu Kun && 16031215@qq.com
-Date: 2025-04-08 16:18:49
-LastEditors: Liu Kun && 16031215@qq.com
-LastEditTime: 2025-04-08 16:18:50
-FilePath: \\Python\\My_Funcs\\OAFuncs\\oafuncs\\_script\\parallel_test.py
-Description:
-EditPlatform: vscode
-ComputerInfo: XPS 15 9510
-SystemInfo: Windows 11
-Python Version: 3.12
-"""