oafuncs 0.0.98.3__py3-none-any.whl → 0.0.98.5__py3-none-any.whl

oafuncs/oa_tool.py

@@ -1,50 +1,120 @@
-from typing import Any, Iterable, List, Optional, Union
+import datetime
+import logging
+from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
 
 from rich import print
 
+from ._script.parallel import ParallelExecutor
+
 __all__ = ["PEx", "email", "pbar"]
 
 
-class PEx:
+class PEx(ParallelExecutor):
     """
-    PEx wraps ParallelExecutor and delegates all its methods,
-    allowing direct calls to PEx methods to achieve the same effect as ParallelExecutor.
+    并行执行器扩展类 (ParallelExecutor Extend)
 
-    Example:
-        # Create a PEx instance
-        executor = PEx(max_workers=4)
+    继承自 ParallelExecutor，提供更简洁的接口和增强功能：
 
-        # Use the run method to execute parallel tasks
-        result = executor.run(lambda x: x * x, [(i,) for i in range(5)])
-        print(result)  # Output: [0, 1, 4, 9, 16]
-    """
+    特点：
+    - 自动时间统计 (开始/结束时间、总耗时)
+    - 增强的错误处理机制
+    - 友好的统计输出格式
+    - 支持进度回调函数
 
-    try:
-        from ._script.parallel import ParallelExecutor
-    except ImportError:
-        raise ImportError("[red]ParallelExecutor could not be imported. Ensure the module '_script.parallel' exists and is accessible.[/red]")
+    示例：
+    >>> with PEx() as executor:
+    ...     results = executor.run(lambda x: x*2, [(i,) for i in range(5)])
+    ...     print(executor.format_stats())
+    [2024-06-08 15:30:00] 成功处理5个任务 (耗时0.5秒)
 
-    def __init__(self, *args: Any, **kwargs: Any) -> None:
+    参数调整建议：
+    - 内存密集型任务：增大 mem_per_process
+    - I/O密集型任务：使用 mode='thread'
+    """
+
+    def __init__(
+        self,
+        max_workers: Optional[int] = None,
+        chunk_size: Optional[int] = None,
+        mem_per_process: float = 1.5,  # 调大默认内存限制
+        timeout_per_task: int = 7200,  # 延长默认超时时间
+        max_retries: int = 5,  # 增加默认重试次数
+        progress_callback: Optional[Callable[[int, int], None]] = None,
+    ):
+        """
+        初始化并行执行器
+
+        :param max_workers: 最大工作进程/线程数 (默认自动计算)
+        :param chunk_size: 任务分块大小 (默认自动优化)
+        :param mem_per_process: 单进程内存预估(GB)
+        :param timeout_per_task: 单任务超时时间(秒)
+        :param max_retries: 最大重试次数
+        :param progress_callback: 进度回调函数 (当前完成数, 总数)
         """
-        Initialize a PEx instance, internally creating a ParallelExecutor instance.
+        # 时间记录扩展
+        self.start_dt = datetime.datetime.now()
+        self.end_dt = None
+        self.progress_callback = progress_callback
+
+        super().__init__(max_workers=max_workers, chunk_size=chunk_size, mem_per_process=mem_per_process, timeout_per_task=timeout_per_task, max_retries=max_retries)
 
-        Args:
-            *args: Positional arguments passed to ParallelExecutor.
-            **kwargs: Keyword arguments passed to ParallelExecutor.
+        logging.info(f"PEx initialized at {self.start_dt:%Y-%m-%d %H:%M:%S}")
+
+    def run(self, func: Callable, params: List[Tuple], chunk_size: Optional[int] = None) -> List[Any]:
         """
-        self.executor = self.ParallelExecutor(*args, **kwargs)
+        执行并行任务 (增强版)
 
-    def __getattr__(self, attr: str) -> Any:
+        :param func: 目标函数，需能序列化(pickle)
+        :param params: 参数列表，每个元素为参数元组
+        :param chunk_size: 可选的分块大小
+        :return: 结果列表，与输入顺序一致
         """
-        Delegate all undefined attribute access to the internal ParallelExecutor instance.
+        total_tasks = len(params)
+        if self.progress_callback:
+            self.progress_callback(0, total_tasks)
+
+        results = super().run(func, params, chunk_size)
+
+        if self.progress_callback:
+            self.progress_callback(total_tasks, total_tasks)
+
+        return results
+
+    def shutdown(self):
+        """增强关闭方法，记录结束时间"""
+        self.end_dt = datetime.datetime.now()
+        super().shutdown()
+        logging.info(f"PEx shutdown at {self.end_dt:%Y-%m-%d %H:%M:%S}")
+
+    def get_stats(self) -> Dict[str, Any]:
+        """获取统计信息 (扩展时间数据)"""
+        stats = super().get_stats()
+        stats.update({"start_time": self.start_dt.strftime("%Y-%m-%d %H:%M:%S"), "end_time": self.end_dt.strftime("%Y-%m-%d %H:%M:%S") if self.end_dt else "运行中", "total_seconds": round((self.end_dt - self.start_dt).total_seconds(), 3) if self.end_dt else None})
+        return stats
+
+    def format_stats(self) -> str:
+        """生成友好统计报告"""
+        stats = self.get_stats()
+        report = [
+            f"[{stats['start_time']}] 并行任务统计报告",
+            f"▪ 平台环境：{stats['platform'].upper()} ({stats['mode']}模式)",
+            f"▪ 资源使用：{stats['workers']}个工作进程 | 分块大小{stats['chunk_size']}",
+            f"▪ 任务统计：成功处理{stats['total_tasks']}个任务 (失败{stats['total_tasks'] - sum(1 for r in self.results if r is not None)})",
+            f"▪ 时间统计：总耗时{stats['total_seconds']}秒",
+        ]
+        if "avg_task_throughput" in stats:
+            report.append(f"▪ 吞吐性能：{stats['avg_task_throughput']:.1f} 任务/秒")
+        return "\n".join(report)
+
+    def __enter__(self):
+        """支持上下文管理器"""
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """退出时自动关闭"""
+        self.shutdown()
 
-        Args:
-            attr (str): The name of the attribute to access.
 
-        Returns:
-            Any: The value of the corresponding attribute.
-        """
-        return getattr(self.executor, attr)
 
 
 def email(title: str = "Title", content: Optional[str] = None, send_to: str = "10001@qq.com") -> None:
@@ -91,7 +161,7 @@ def pbar(
 
     Returns:
         Any: An instance of ColorProgressBar.
-    
+
     Example:
         >>> for i in pbar(range(10), description="Processing"):
         ...     time.sleep(0.1)
@@ -117,3 +187,21 @@ def pbar(
         speed_estimate_period=speed_estimate_period,
         next_line=next_line,
     )
+
+
+# 使用示例
+if __name__ == "__main__":
+
+    def sample_task(x):
+        import time
+
+        time.sleep(0.1)
+        return x * 2
+
+    def progress_handler(current, total):
+        print(f"\r进度: {current}/{total} ({current / total:.1%})", end="")
+
+    with PEx(max_workers=4, progress_callback=progress_handler) as executor:
+        results = executor.run(sample_task, [(i,) for i in range(10)])
+
+    print("\n" + executor.format_stats())

{oafuncs-0.0.98.3.dist-info → oafuncs-0.0.98.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: oafuncs
-Version: 0.0.98.3
+Version: 0.0.98.5
 Summary: Oceanic and Atmospheric Functions
 Home-page: https://github.com/Industry-Pays/OAFuncs
 Author: Kun Liu
@@ -25,6 +25,7 @@ Requires-Dist: rich
 Requires-Dist: pathlib
 Requires-Dist: requests
 Requires-Dist: bs4
+Requires-Dist: httpx
 Requires-Dist: matplotlib
 Requires-Dist: netCDF4
 Requires-Dist: xlrd

{oafuncs-0.0.98.3.dist-info → oafuncs-0.0.98.5.dist-info}/RECORD

@@ -7,7 +7,7 @@ oafuncs/oa_file.py,sha256=goF5iRXJFFCIKhIjlkCnYYt0EYlJb_4r8AeYNZ0-SOk,16209
 oafuncs/oa_help.py,sha256=_4AZgRDq5Or0vauNvq5IDDHIBoBfdOQtzak-mG1wwAw,4537
 oafuncs/oa_nc.py,sha256=L1gqXxg93kIDsMOa87M0o-53KVmdqCipnXeF9XfzfY8,10513
 oafuncs/oa_python.py,sha256=NkopwkYFGSEuVljnTBvXCl6o2CeyRNBqRXSsUl3euEE,5192
-oafuncs/oa_tool.py,sha256=8y0X3KXNLbYchr5hBHCwhdsA_1VDEnzsZVqcM5sNeog,4484
+oafuncs/oa_tool.py,sha256=UNdiXKWfk860eb01vSGH8XN_Bd3CP7ihaspRrudday0,7983
 oafuncs/_data/hycom.png,sha256=MadKs6Gyj5n9-TOu7L4atQfTXtF9dvN9w-tdU9IfygI,10945710
 oafuncs/_data/oafuncs.png,sha256=o3VD7wm-kwDea5E98JqxXl04_78cBX7VcdUt7uQXGiU,3679898
 oafuncs/_script/cprogressbar.py,sha256=wRU3SFPFtMI7ER26tTzg223kVKNo5RDWE9CzdIgUsuE,15771
@@ -15,17 +15,16 @@ oafuncs/_script/email.py,sha256=lL4HGKrr524-g0xLlgs-4u7x4-u7DtgNoD9AL8XJKj4,3058
 oafuncs/_script/netcdf_merge.py,sha256=_EPF9Xj4HOVC9sZpi1lt62-Aq6pMlgsgwaajEBLhW6g,5092
 oafuncs/_script/netcdf_modify.py,sha256=sGRUYNhfGgf9JV70rnBzw3bzuTRSXzBTL_RMDnDPeLQ,4552
 oafuncs/_script/netcdf_write.py,sha256=iO1Qv9bp6RLiw1D8Nrv7tX_8X-diUZaX3Nxhk6pJ5Nw,8556
-oafuncs/_script/parallel.py,sha256=FS9FgaByq2yb9j6nL-Y0xP1VLvp4USMLBFMRsJDoqeQ,21848
-oafuncs/_script/parallel_example_usage.py,sha256=uLvE7iwkMn9Cyq6-wk5_RpbQk7PXM9d16-26lTknW9s,2646
+oafuncs/_script/parallel.py,sha256=dRT7w_rBnR3mZkUlO6v6j05SwBTQpTccOna5CXI5Msg,8196
+oafuncs/_script/parallel_test.py,sha256=0GBqZOX7IaCOKF2t1y8N8YYu53GJ33OkfsWgpvZNqM4,372
 oafuncs/_script/plot_dataset.py,sha256=zkSEnO_-biyagorwWXPoihts_cwuvripzEt-l9bHJ2E,13989
 oafuncs/_script/replace_file_content.py,sha256=eCFZjnZcwyRvy6b4mmIfBna-kylSZTyJRfgXd6DdCjk,5982
-oafuncs/oa_down/User_Agent-list.txt,sha256=pazxSip8_lphEBOPHG902zmIBUg8sBKXgmqp_g6j_E4,661062
+oafuncs/oa_down/User_Agent-list.txt,sha256=pHaMlElMvZ8TG4vf4BqkZYKqe0JIGkr4kCN0lM1Y9FQ,514295
 oafuncs/oa_down/__init__.py,sha256=kRX5eTUCbAiz3zTaQM1501paOYS_3fizDN4Pa0mtNUA,585
-oafuncs/oa_down/hycom_3hourly.py,sha256=Xi6LuD3DIiYq2gYYBVxY3OpVvfy3RkSMzNkOX2isSK8,52915
-oafuncs/oa_down/hycom_3hourly_20250407.py,sha256=DQd_NmQgmSqu7jsrfpDB7k23mkUEy9kyWs-dLUg7GDw,64472
+oafuncs/oa_down/hycom_3hourly.py,sha256=ERH24OOT62IiOSk5IiSnqad_c13zL7-uD5lAMdbJDiQ,53576
 oafuncs/oa_down/idm.py,sha256=4z5IvgfTyIKEI1kOtqXZwN7Jnfjwp6qDBOIoVyOLp0I,1823
 oafuncs/oa_down/literature.py,sha256=2bF9gSKQbzcci9LcKE81j8JEjIJwON7jbwQB3gDDA3E,11331
-oafuncs/oa_down/test_ua.py,sha256=0IQq3NjqfNr7KkyjS_U-a4mYu-r-E7gzawwo4IfEa6Y,10851
+oafuncs/oa_down/test_ua.py,sha256=l8MCD6yU2W75zRPTDKUZTJhCWNF9lfk-MiSFqAqKH1M,1398
 oafuncs/oa_down/user_agent.py,sha256=TsPcAxFmMTYAEHRFjurI1bQBJfDhcA70MdHoUPwQmks,785
 oafuncs/oa_model/__init__.py,sha256=__ImltHkP1bSsIpsmKpDE8QwwA-2Z8K7mZUHGGcRdro,484
 oafuncs/oa_model/roms/__init__.py,sha256=g-_iv2vUFUnSsKWc7pzGRfkNUvhxfG-Bn4deqfGwxyo,474
@@ -36,8 +35,8 @@ oafuncs/oa_sign/__init__.py,sha256=QKqTFrJDFK40C5uvk48GlRRbGFzO40rgkYwu6dYxatM,5
 oafuncs/oa_sign/meteorological.py,sha256=8091SHo2L8kl4dCFmmSH5NGVHDku5i5lSiLEG5DLnOQ,6489
 oafuncs/oa_sign/ocean.py,sha256=xrW-rWD7xBWsB5PuCyEwQ1Q_RDKq2KCLz-LOONHgldU,5932
 oafuncs/oa_sign/scientific.py,sha256=a4JxOBgm9vzNZKpJ_GQIQf7cokkraV5nh23HGbmTYKw,5064
-oafuncs-0.0.98.3.dist-info/licenses/LICENSE.txt,sha256=rMtLpVg8sKiSlwClfR9w_Dd_5WubTQgoOzE2PDFxzs4,1074
-oafuncs-0.0.98.3.dist-info/METADATA,sha256=1-5l6aAaOFaZ9U0pSE6f1944hrE01bLGCBLPonbEXeU,4220
-oafuncs-0.0.98.3.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-oafuncs-0.0.98.3.dist-info/top_level.txt,sha256=bgC35QkXbN4EmPHEveg_xGIZ5i9NNPYWqtJqaKqTPsQ,8
-oafuncs-0.0.98.3.dist-info/RECORD,,
+oafuncs-0.0.98.5.dist-info/licenses/LICENSE.txt,sha256=rMtLpVg8sKiSlwClfR9w_Dd_5WubTQgoOzE2PDFxzs4,1074
+oafuncs-0.0.98.5.dist-info/METADATA,sha256=YwSyTAse2NhMIfM_22BNOJCDLh8Mgy3X_G5zvtCKyPc,4242
+oafuncs-0.0.98.5.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+oafuncs-0.0.98.5.dist-info/top_level.txt,sha256=bgC35QkXbN4EmPHEveg_xGIZ5i9NNPYWqtJqaKqTPsQ,8
+oafuncs-0.0.98.5.dist-info/RECORD,,

oafuncs/_script/parallel_example_usage.py

@@ -1,83 +0,0 @@
-#!/usr/bin/env python
-# coding=utf-8
-"""
-Author: Liu Kun && 16031215@qq.com
-Date: 2025-03-18 19:14:19
-LastEditors: Liu Kun && 16031215@qq.com
-LastEditTime: 2025-03-18 19:18:38
-FilePath: \\Python\\My_Funcs\\OAFuncs\\oafuncs\\_script\\parallel_example_usage.py
-Description:
-EditPlatform: vscode
-ComputerInfo: XPS 15 9510
-SystemInfo: Windows 11
-Python Version: 3.12
-"""
-
-import logging
-import time
-from auto_optimized_parallel_executor import ParallelExecutor
-
-# 设置日志
-logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
-
-
-# 示例函数
-def compute_intensive_task(n):
-    """计算密集型任务示例"""
-    result = 0
-    for i in range(n):
-        result += i**0.5
-    return result
-
-
-def io_intensive_task(seconds, value):
-    """IO密集型任务示例"""
-    time.sleep(seconds)  # 模拟IO操作
-    return f"Processed {value}"
-
-
-def main():
-    # 创建自动优化的执行器
-    executor = ParallelExecutor()
-
-    # 打印选择的模式和工作线程/进程数量
-    print(f"自动选择的执行模式: {executor.mode}")
-    print(f"自动选择的工作线程/进程数: {executor.max_workers}")
-    print(f"运行平台: {executor.platform}")
-
-    # 示例1: 计算密集型任务
-    print("\n运行计算密集型任务...")
-    params = [(1000000,) for _ in range(20)]
-    results = executor.run(compute_intensive_task, params)
-    print(f"完成计算密集型任务，结果数量: {len(results)}")
-
-    # 示例2: IO密集型任务
-    print("\n运行IO密集型任务...")
-    io_params = [(0.1, f"item-{i}") for i in range(30)]
-    io_results = executor.run(io_intensive_task, io_params)
-    print(f"完成IO密集型任务，结果示例: {io_results[:3]}")
-
-    # 示例3: 使用map接口
-    print("\n使用map接口...")
-    numbers = list(range(1, 11))
-    squared = list(executor.map(lambda x: x * x, numbers))
-    print(f"Map结果: {squared}")
-
-    # 示例4: 使用imap_unordered接口（乱序返回结果）
-    print("\n使用imap_unordered接口...")
-    for i, result in enumerate(executor.imap_unordered(lambda x: x * x * x, range(1, 11))):
-        print(f"收到结果 #{i}: {result}")
-
-    # 示例5: 使用gather执行不同函数
-    print("\n使用gather接口执行不同函数...")
-    tasks = [(compute_intensive_task, (500000,)), (io_intensive_task, (0.2, "task1")), (io_intensive_task, (0.1, "task2")), (compute_intensive_task, (300000,))]
-    gather_results = executor.gather(tasks)
-    print(f"Gather结果: {gather_results}")
-
-    # 关闭执行器
-    executor.shutdown()
-    print("\n执行器已关闭")
-
-
-if __name__ == "__main__":
-    main()