xutk 0.3.4__tar.gz

xutk-0.3.4/PKG-INFO

@@ -0,0 +1,89 @@
+Metadata-Version: 2.4
+Name: xutk
+Version: 0.3.4
+Summary: Xulab toolkit in python
+Requires-Python: <3.15,>=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: colorama>=0.4.6
+Requires-Dist: psutil>=7.1.0
+
+# Xulab Useful Toolkit in Python (xutk)
+
+![Maintenance](https://img.shields.io/maintenance/Zelin2001/2025)
+![PyPI - Python Version](https://img.shields.io/badge/python-3.11|3.12|3.13|3.14|3.14t-blue.svg)
+
+## 导入项目
+
+```bash
+pip install \
+  --trusted-host "gitea.xulab.ion.ac.cn" \
+  --extra-index-url "http://gitea.xulab.ion.ac.cn/api/packages/zelin2001/pypi/simple/" \
+  xutk
+```
+
+或者使用 uv:
+
+```bash
+uv add --index http://gitea.xulab.ion.ac.cn/api/packages/zelin2001/pypi/simple xutk
+```
+
+## 快速开始
+
+```python
+from xutk import verchk, log, mp_runner
+from pathlib import Path
+
+# 版本检查
+verchk.check_version("0.2.0", "some_package")
+
+# 日志记录
+logger = log.CtxLogger("my_app")
+logger.info({"module": "processor", "file": "data.csv"}, "Processing started")
+
+# 多进程运行器
+runner = mp_runner.mprunner_factory(2, log_file=Path("process.log")) # 进程队列，取 2 个运行
+runner.batch_run(["echo", "-E", f"test{num}"] for num in range(1,25)) # 排队 25 个进程
+```
+
+**建议在项目 `__init__.py` 创建 CtxLogger("my_app")，避免实例化顺序问题。**
+
+## 项目结构
+
+```text
+xutk/
+├── xutk/
+│   ├── __init__.py      # 包初始化
+│   ├── log.py           # 日志记录功能
+│   ├── mp_runner.py     # 多进程运行管理
+│   ├── perf.py          # 快速资源检查
+│   └── verchk.py        # 版本检查功能
+├── test/
+│   ├── integration/     # 集成测试
+│   └── unit/            # 单元测试
+├── gitea/
+│   └── workflows/       # CI 工作流
+├── pyproject.toml       # 项目配置
+└── README.md            # 项目文档
+```
+
+## 开发
+
+### 安装开发依赖
+
+```bash
+uv sync --all-extras
+```
+
+### 运行测试
+
+```bash
+uv run pytest
+```
+
+### 代码检查
+
+```bash
+uv run ruff check      # 代码风格检查
+uv run ty check        # 类型检查
+uv run pytest          # 样例测试
+```

xutk-0.3.4/README.md

@@ -0,0 +1,80 @@
+# Xulab Useful Toolkit in Python (xutk)
+
+![Maintenance](https://img.shields.io/maintenance/Zelin2001/2025)
+![PyPI - Python Version](https://img.shields.io/badge/python-3.11|3.12|3.13|3.14|3.14t-blue.svg)
+
+## 导入项目
+
+```bash
+pip install \
+  --trusted-host "gitea.xulab.ion.ac.cn" \
+  --extra-index-url "http://gitea.xulab.ion.ac.cn/api/packages/zelin2001/pypi/simple/" \
+  xutk
+```
+
+或者使用 uv:
+
+```bash
+uv add --index http://gitea.xulab.ion.ac.cn/api/packages/zelin2001/pypi/simple xutk
+```
+
+## 快速开始
+
+```python
+from xutk import verchk, log, mp_runner
+from pathlib import Path
+
+# 版本检查
+verchk.check_version("0.2.0", "some_package")
+
+# 日志记录
+logger = log.CtxLogger("my_app")
+logger.info({"module": "processor", "file": "data.csv"}, "Processing started")
+
+# 多进程运行器
+runner = mp_runner.mprunner_factory(2, log_file=Path("process.log")) # 进程队列，取 2 个运行
+runner.batch_run(["echo", "-E", f"test{num}"] for num in range(1,25)) # 排队 25 个进程
+```
+
+**建议在项目 `__init__.py` 创建 CtxLogger("my_app")，避免实例化顺序问题。**
+
+## 项目结构
+
+```text
+xutk/
+├── xutk/
+│   ├── __init__.py      # 包初始化
+│   ├── log.py           # 日志记录功能
+│   ├── mp_runner.py     # 多进程运行管理
+│   ├── perf.py          # 快速资源检查
+│   └── verchk.py        # 版本检查功能
+├── test/
+│   ├── integration/     # 集成测试
+│   └── unit/            # 单元测试
+├── gitea/
+│   └── workflows/       # CI 工作流
+├── pyproject.toml       # 项目配置
+└── README.md            # 项目文档
+```
+
+## 开发
+
+### 安装开发依赖
+
+```bash
+uv sync --all-extras
+```
+
+### 运行测试
+
+```bash
+uv run pytest
+```
+
+### 代码检查
+
+```bash
+uv run ruff check      # 代码风格检查
+uv run ty check        # 类型检查
+uv run pytest          # 样例测试
+```

xutk-0.3.4/pyproject.toml

@@ -0,0 +1,23 @@
+[project]
+name = "xutk"
+version = "0.3.4"
+description = "Xulab toolkit in python"
+readme = "README.md"
+requires-python = ">=3.11,<3.15"
+dependencies = [
+    "colorama>=0.4.6",
+    "psutil>=7.1.0",
+]
+
+[dependency-groups]
+dev = [
+    "pytest>=8.4.1",
+    "ruff>=0.12.4",
+    "ty>=0.0.1a14",
+]
+
+# [[tool.uv.index]]
+# name = "gitea-zelin2001"
+# url = "http://gitea.xulab.ion.ac.cn/api/packages/zelin2001/pypi/simple"
+# publish-url = "http://gitea.xulab.ion.ac.cn/api/packages/zelin2001/pypi"
+

xutk-0.3.4/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

xutk-0.3.4/xutk/__init__.py

@@ -0,0 +1,18 @@
+"""Xulab Useful Toolkits for Python.
+
+- module `log`:
+  - class CtxLogger
+  - set_log_level()
+- module `mp_runner`:
+  - class MPRunner
+  - class ThreadPoolRunner
+  - dataclass SLURMConfig
+  - class SLURMRunner
+  - mprunner_factory()
+- module `perf`:
+  - class Perf
+  - fadvise_remove_cache()
+- module `verchk`:
+  - check_version()
+
+"""

xutk-0.3.4/xutk/log.py

@@ -0,0 +1,175 @@
+"""Logging for A package with level-specific, contex aware messages.
+
+- Use `XUTK_LOG_LEVEL` environment variable to set logging level
+  level: One of 'DEBUG', 'INFO', 'ARNING', 'ERROR'
+  - Alternatively, use function `set_log_level()`
+- Use `CtxLogger` to initialize specific logger
+"""
+
+import colorama
+import logging
+import os
+import sys
+from typing import Dict, Any
+
+
+class _ColorfulFormatter(logging.Formatter):
+    """A custom logging formatter that adds color to log messages.
+
+    This formatter uses colorama to add ANSI color codes to log messages
+    based on their severity level, making them easier to distinguish
+    in terminal output.
+    """
+
+    def formatMessage(self, record: logging.LogRecord) -> str:
+        """Format the message part of the log record with color.
+
+        This method is called by the parent format() method and only
+        adds color to the levelname while preserving all other standard
+        formatting behavior.
+
+        Args:
+            record: The log record to format
+
+        Returns:
+            A formatted string with colored level name
+        """
+        # Color mapping for log levels
+        log_colors = {
+            "NOTSET": colorama.Fore.WHITE,
+            "DEBUG": colorama.Fore.CYAN,
+            "INFO": colorama.Fore.GREEN,
+            "WARNING": colorama.Fore.YELLOW + colorama.Style.BRIGHT,
+            "ERROR": colorama.Fore.RED + colorama.Style.BRIGHT,
+            "CRITICAL": colorama.Fore.RED + colorama.Style.BRIGHT,
+        }
+
+        # Passing colored levelname to parent's formatMessage
+        original_levelname = record.levelname
+        record.levelname = (
+            f"{log_colors.get(original_levelname, '')}"
+            + f"{original_levelname}{colorama.Style.RESET_ALL}"
+        )
+        try:
+            # Use parent's formatMessage to handle the actual formatting
+            result = super().formatMessage(record)
+        finally:
+            # Restore original levelname
+            record.levelname = original_levelname
+
+        return result
+
+
+class CtxLogger:
+    """Logger with built-in context formatting and level methods."""
+
+    caller_names: set[str]
+    _logger: logging.Logger
+    _color_flag: bool
+
+    def __init__(self, name: str = "xutk") -> None:
+        """Initialize the CtxLogger with basic configuration.
+
+        Args:
+            name: Logger name (defaults to 'xutk'). Creates a new logger or returns
+                  existing one if name matches.
+
+        Configures logging level from XUTK_LOG_LEVEL (PKG_LOG_LEVEL)
+        environment variable or set_log_level() (defaults to INFO)
+        and sets up a basic console handler with standard formatting.
+
+        Also override color output setting by environment variable
+        XUTK_LOG_COLOR (PKG_LOG_COLOR), True (true) or False (false).
+        """
+        self._logger = logging.getLogger(name)
+        self.caller_names = {"caller", "plotter", "loader", "processor"}
+        self._init_handler_and_color(name)
+
+    def _init_handler_and_color(self, name: str) -> None:
+        color_opt = os.getenv(name.upper() + "_LOG_COLOR", "auto").lower()
+        color_flag = False
+        if "auto" == color_opt:
+            color_flag = sys.stdout.isatty()
+        elif color_opt in {"true", "1", "yes", "y"}:
+            color_flag = True
+        if color_flag:
+            colorama.init()
+        self._color_flag = color_flag
+
+        if not self._logger.handlers:
+            # Set level from environment or default to INFO
+            logger_level = name.upper() + "_LOG_LEVEL"
+            level = os.getenv(logger_level, "INFO").upper()
+            self._logger.setLevel(getattr(logging, level, logging.INFO))
+
+            handler = logging.StreamHandler()
+            formatter = (
+                _ColorfulFormatter(
+                    "%(asctime)s [%(levelname)s] %(message)s", datefmt="%H:%M:%S"
+                )
+                if self._color_flag
+                else logging.Formatter(
+                    "%(asctime)s [%(levelname)s] %(message)s", datefmt="%H:%M:%S"
+                )
+            )
+            handler.setFormatter(formatter)
+            self._logger.addHandler(handler)
+
+    def _log_with_context(
+        self, level: int, context: Dict[str, Any], message: str
+    ) -> None:
+        """Add to log internally."""
+        callers = [(k, v) for k, v in context.items() if k in self.caller_names]
+        caller_str = ""
+        try:
+            caller_str = callers[0][1]
+        except IndexError:
+            pass
+
+        if self._color_flag:
+            message = (
+                str(colorama.Style.BRIGHT) + message + str(colorama.Style.RESET_ALL)
+            )
+
+        ctx_items = [
+            f"{k}: {v}" for k, v in context.items() if k not in self.caller_names
+        ]
+        ctx_str = "; ".join(ctx_items)
+
+        self._logger.log(
+            level,
+            f"{caller_str}: {message}"
+            + f"{' | ' if len(ctx_str) > 0 else ''}{ctx_str}",
+        )
+
+    def debug(self, context: Dict[str, Any], message: str) -> None:
+        """Log debug message with context."""
+        self._log_with_context(logging.DEBUG, context, message)
+
+    def info(self, context: Dict[str, Any], message: str) -> None:
+        """Log info message with context."""
+        self._log_with_context(logging.INFO, context, message)
+
+    def warning(self, context: Dict[str, Any], message: str) -> None:
+        """Log warning message with context."""
+        self._log_with_context(logging.WARNING, context, message)
+
+    def error(self, context: Dict[str, Any], message: str) -> None:
+        """Log error message with context."""
+        self._log_with_context(logging.ERROR, context, message)
+
+    def get_log_level(self) -> int:
+        """Get log level (number)."""
+        return self._logger.getEffectiveLevel()
+
+
+def set_log_level(level: str, logger_name: str = "xutk") -> None:
+    """Set log level programmatically.
+
+    Args:
+        level: One of 'DEBUG', 'INFO', 'WARNING', 'ERROR'
+               (Neither 'NOTSET' nor 'CRITICAL' are provided)
+        logger_name: Name of the logger to configure (defaults to 'xutk')
+    """
+    logger = logging.getLogger(logger_name)
+    logger.setLevel(getattr(logging, level.upper(), logging.INFO))

xutk-0.3.4/xutk/mp_runner.py

@@ -0,0 +1,464 @@
+"""Parallel computing runner module.
+
+This module provides a flexible interface for executing tasks
+using different parallel computing backends. It implements a factory pattern
+through mprunner_factory that creates appropriate runners based on the client
+type provided.
+
+The module supports multiple execution modes:
+1. ThreadPoolExecutor: For local multi-threaded processing
+2. SLURMConfig: For job submission to SLURM clusters
+3. DaskClient (future): For distributed computing with Dask
+
+Key Components:
+- MPRunner: Abstract base class defining the interface for all runners
+- ThreadPoolRunner: Implementation for local thread-based parallelism
+- SLURMRunner: Implementation for SLURM cluster job submission
+- SLURMConfig: Configuration class for SLURM job parameters
+- mprunner_factory: Factory function to create appropriate runner instances
+- MPClient: Type alias for Union[ThreadPoolExecutor, SLURMConfig]
+
+The API provides two main execution methods:
+- batch_run: For executing command-line operations
+- batch_function_run: For executing Python functions
+
+Each runner implementation handles the execution details while providing
+a consistent interface. The abstract methods _submit_command, _submit_function,
+_join_futures, and _cancel_futures provide extension points for new runners.
+
+Usage:
+```python
+# For local processing with limited workers
+executor = ThreadPoolExecutor(max_workers=4)
+runner = mprunner_factory(executor, log_file)
+runner.batch_run(cmd_args_list)
+runner.batch_function_run(process_video, video_paths, presets)
+
+# For SLURM cluster processing
+config = SLURMConfig(partition="batch", num_cpus=12, memory="8G", output=log_file)
+runner = mprunner_factory(config, log_file)
+runner.batch_run(cmd_args_list)
+
+# For future Dask distributed processing
+client = Client("scheduler-address:8786")
+runner = mprunner_factory(client, log_file)
+runner.batch_function_run(process_video, video_paths, presets)
+
+# Using the MPClient type alias for better type hints
+def process_videos(client: MPClient, videos: list[str]) -> None:
+    runner = mprunner_factory(client)
+    runner.batch_run(create_ffmpeg_commands(videos))
+```
+"""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from concurrent.futures import Future, ThreadPoolExecutor
+from pathlib import Path
+import queue
+import shlex
+from subprocess import run, CompletedProcess
+import threading
+from typing import Callable
+
+from xutk.log import CtxLogger as Logger
+
+vidtidy_logger = Logger("vidtidy")
+
+
+class MPRunner(ABC):
+    """Handles video processing with string parameters."""
+
+    def __init__(self, client, log_file: Path | None) -> None:  # noqa: ANN001
+        """Initialize with basic file patterns and optional config.
+
+        Args:
+            client: Configuration or executor for the runner
+            log_file: Path to the log file (optional, if None logging is disabled)
+
+        """
+        self._log_queue = queue.Queue()
+        self._log_lock = threading.Lock()
+
+        if log_file:
+            self._log_file = log_file
+            self._log_thread = threading.Thread(target=self._log_writer, daemon=True)
+            self._log_thread.start()
+        else:
+            self._log_file = Path()
+            self._log_thread = None
+
+        self.client = client
+
+    def _log_writer(self) -> None:
+        """Background thread that writes logs sequentially.
+
+        This method runs as a daemon thread, continuously processing log messages
+        from the queue and writing them to the log file. Only active when log_file
+        is not None.
+        """
+        if not self._log_file:
+            return
+
+        while True:
+            log_msg = self._log_queue.get()
+            try:
+                with self._log_file.open("a") as f:
+                    f.write(log_msg)
+            except Exception as e:
+                vidtidy_logger.error(
+                    {"caller": self.__class__.__name__}, f"Failed to write log: {e}"
+                )
+            self._log_queue.task_done()
+
+    @abstractmethod
+    def batch_run(self, cmd_args_list: list[list[str]]) -> list:
+        """Run multiple commands synchronously, blocking until completion.
+
+        Args:
+            cmd_args_list: List of command argument lists to execute
+
+        """
+        ...
+
+    @abstractmethod
+    def batch_function_run(
+        self,
+        func: Callable,
+        *args_lists: list,
+        **common_kwargs,  # noqa: ANN003
+    ) -> list:
+        """Run multiple instances of a function together, blocking until completion.
+
+        Supports two calling conventions:
+        1. Multiple argument lists: batch_function_run
+           (func, arg1_list, arg2_list, kwarg=value)
+        2. List of argument tuples: batch_function_run
+           (func, [(arg1, arg2), ...], kwarg=value)
+
+        Args:
+            func: The function to execute
+            *args_lists: Variable length argument lists
+            **common_kwargs: Common keyword arguments for all function calls
+
+        """
+        ...
+
+
+class ThreadPoolRunner(MPRunner):
+    """Handles video processing with string parameters."""
+
+    client: ThreadPoolExecutor
+
+    def batch_run(self, cmd_args_list: list[list[str]]) -> list[CompletedProcess[str]]:  # noqa: D102
+        futures: list[Future[CompletedProcess[str]]] = []
+        results: list[CompletedProcess[str]] = []
+        try:
+            with self.client as executor:
+                for cmd_item in cmd_args_list:
+                    future = executor.submit(self._thread_pool_execute, cmd_item)
+                    futures.append(future)
+                # Wait for all futures and logs to complete
+                try:
+                    results.extend(future.result() for future in futures)
+                    self._log_queue.join()
+                except KeyboardInterrupt:
+                    vidtidy_logger.warning(
+                        {"caller": self.__class__.__name__},
+                        "\nProcessing interrupted by user. Cleaning up...",
+                    )
+                    # Cancel all pending futures on interrupt
+                    for future in futures:
+                        future.cancel()
+                    # Wait for any running tasks to complete
+                    for future in futures:
+                        if not future.done():
+                            future.result()
+                    self._log_queue.join()
+                    raise
+        except KeyboardInterrupt:
+            vidtidy_logger.warning(
+                {"caller": self.__class__.__name__},
+                "\nJob scheduling interrupted by user. Cleaning up...",
+            )
+            raise
+        return results
+
+    def _thread_pool_execute(self, cmd_args: list[str]) -> CompletedProcess[str]:
+        process = run(cmd_args, check=True, capture_output=True, text=True)
+        with self._log_lock:
+            # FFmpeg uses stderr for normal output and stdout for progress
+            # Use the last argument as the processing file if available,
+            # otherwise use command name
+            target_file = cmd_args[-1] if len(cmd_args) > 1 else cmd_args[0]
+            self._log_queue.put(
+                f"=== Processing {target_file} ===\n"
+                f"Output:\n{process.stdout}\n"
+                f"Errors:\n{process.stderr}\n\n"
+            )
+        return process
+
+    def batch_function_run(  # noqa: D102
+        self,
+        func: Callable,
+        *args_lists: list,
+        **common_kwargs,  # noqa: ANN003
+    ) -> list:
+        if not args_lists:
+            vidtidy_logger.warning(
+                {"caller": self.__class__.__name__},
+                "\nBatch runner got args_list with different length.",
+            )
+            return []
+
+        arg_list_tuple = self._batch_function_parse_args(args_lists)
+
+        # === Build Tasks and Submit ===
+        futures: list[Future] = []
+        results: list = []
+        try:
+            with self.client as executor:
+                for args in arg_list_tuple:
+                    future = executor.submit(func, *args, **common_kwargs)
+                    futures.append(future)
+
+                # Wait for completion and collect results in order
+                try:
+                    # Wait for all futures and collect results in original order
+                    results.extend(future.result() for future in futures)
+                    self._log_queue.join()
+                except KeyboardInterrupt:
+                    vidtidy_logger.warning(
+                        {"caller": self.__class__.__name__},
+                        "\nProcessing interrupted by user. Cancelling tasks...",
+                    )
+                    for future in futures:
+                        future.cancel()
+                    # Allow running tasks to finish cleanly
+                    for future in futures:
+                        if not future.done():
+                            try:
+                                future.result()
+                            except:  # noqa: E722
+                                pass
+                    self._log_queue.join()
+                    raise
+
+            return results
+
+        except KeyboardInterrupt:
+            vidtidy_logger.warning(
+                {"caller": self.__class__.__name__},
+                "\nJob Scheduling interrupted by user. Cleaning up...",
+            )
+            raise
+
+    @staticmethod
+    def _batch_function_parse_args(args_lists: tuple[list, ...]) -> list[tuple]:
+        """Parse input arguments into list of argument tuples.
+
+        Supports two calling conventions:
+        1. Multiple argument lists: [a1,a2], [b1,b2] -> [(a1,b1), (a2,b2)]
+        2. List of tuples: [(a1,b1), (a2,b2)] -> [(a1,b1), (a2,b2)]
+
+        Args:
+            args_lists: Tuple of argument lists (from *args)
+
+        Returns:
+            List of argument tuples for function calls
+
+        Raises:
+            ValueError: If argument lists have different lengths or invalid format
+
+        """
+        if not args_lists:
+            return []
+
+        first_arg = args_lists[0]
+
+        if len(args_lists) == 1 and isinstance(first_arg, list) and first_arg:
+            # Case 1: Single list of tuples - [(arg1, arg2), ...]
+            if isinstance(first_arg[0], tuple):
+                return first_arg  # Already in (arg1, arg2) format
+            else:
+                # Case 2: Single list of non-tuples, treat as single argument
+                return [(x,) for x in first_arg]
+        else:
+            # Case 3: Multiple argument lists - [a1,a2], [b1,b2]
+            n_args = len(args_lists[0])
+            if any(len(lst) != n_args for lst in args_lists):
+                raise ValueError("All argument lists must have the same length.")
+            return list(zip(*args_lists))
+
+
+@dataclass
+class SLURMConfig:
+    """Configuration for SLURM job submission.
+
+    Attributes:
+        output: Path for stdout log file
+        error: Path for stderr log file
+        partition: SLURM partition to use
+        node: Specific node to run on (optional)
+        num_cpus: Number of CPU cores to allocate per task
+        memory: Memory allocation in GB
+        job_name: Name for the SLURM job
+        time_limit: Time limit for the job (HH:MM:SS format)
+
+    """
+
+    output: Path | None = None
+    error: Path | None = None
+    partition: str = "batch"
+    node: str | None = None
+    num_cpus: int = 12
+    memory: str = "8G"
+    job_name: str = "vidtidy_job"
+    time_limit: str = "02:00:00"
+
+
+class SLURMRunner(MPRunner):
+    """Handles SLURM job submission for video processing tasks."""
+
+    client: SLURMConfig
+
+    def batch_run(self, cmd_args_list: list[list[str]]) -> list[int]:
+        """Submit multiple commands as SLURM jobs using the client configuration.
+
+        Args:
+            cmd_args_list: List of command argument lists to execute
+
+        Raises:
+            RuntimeError: If any job submission fails
+
+        """
+        job_ids: list[int] = []
+        try:
+            for i, cmd_args in enumerate(cmd_args_list):
+                # Create unique job name for each task
+                job_name = f"{self.client.job_name}_{i}"
+
+                # Build sbatch command with options from config
+                sbatch_cmd = [
+                    "sbatch",
+                    "--job-name",
+                    job_name,
+                    "--time",
+                    self.client.time_limit,
+                    "--partition",
+                    self.client.partition,
+                    "--cpus-per-task",
+                    str(self.client.num_cpus),
+                    "--mem",
+                    self.client.memory,
+                ]
+
+                # Add node specification if provided
+                if self.client.node:
+                    sbatch_cmd.extend(["--nodelist", self.client.node])
+
+                # Add output and error log paths if specified
+                if self.client.output:
+                    sbatch_cmd.extend(
+                        [
+                            "--output",
+                            str(
+                                self.client.output.with_suffix(
+                                    f".{i}{self.client.output.suffix}"
+                                )
+                            ),
+                        ]
+                    )
+                if self.client.error:
+                    sbatch_cmd.extend(
+                        [
+                            "--error",
+                            str(
+                                self.client.error.with_suffix(
+                                    f".{i}{self.client.error.suffix}"
+                                )
+                            ),
+                        ]
+                    )
+
+                # Add the actual command to run
+                sbatch_cmd.extend(["--wrap", shlex.join(cmd_args)])
+
+                # Submit the job
+                result = run(sbatch_cmd, capture_output=True, text=True, check=True)
+
+                if result.returncode != 0:
+                    raise RuntimeError(f"SLURM job submission failed: {result.stderr}")
+
+                # Extract job ID from output (format: "Submitted batch job <job_id>")
+                job_id_line = result.stdout.strip()
+                if "Submitted batch job" in job_id_line:
+                    job_id = int(job_id_line.split()[-1])
+                    job_ids.append(job_id)
+                    self._log_queue.put(
+                        f"Submitted SLURM job {job_id} for "
+                        + f"command: {shlex.join(cmd_args)}\n"
+                    )
+                else:
+                    raise RuntimeError(
+                        f"Could not parse SLURM job ID from: {job_id_line}"
+                    )
+
+        except Exception:
+            # Cancel all submitted jobs if there's an error
+            for job_id in job_ids:
+                run(["scancel", str(job_id)], capture_output=True, check=True)
+            raise
+
+        # Wait for all jobs to complete and logs to be written
+        try:
+            self._log_queue.join()
+        except KeyboardInterrupt:
+            # Cancel all jobs on interrupt
+            for job_id in job_ids:
+                run(["scancel", str(job_id)], capture_output=True, check=True)
+            self._log_queue.join()
+            raise
+
+        return job_ids
+
+    def batch_function_run(  # noqa: D102
+        self,
+        func: Callable,
+        *args_lists: list,
+        **common_kwargs,  # noqa: ANN003
+    ) -> None:
+        raise NotImplementedError(
+            """
+            SLURMRunner does not support function execution.
+            Use batch_run with shell commands.
+            """
+        )
+
+
+MPClient = int | ThreadPoolExecutor | SLURMConfig
+
+
+def mprunner_factory(mp_client: MPClient, log_file: Path | None = None) -> MPRunner:
+    """Create appropriate runner based on client type.
+
+    Args:
+        mp_client: A multi-processing implement instance
+        log_file: Path to the log file
+
+    Returns:
+        Appropriate runner instance
+        (ThreadPoolRunner, SLURMRunner, or future DaskRunner)
+
+    Raises:
+        NotImplementedError: If client type is not supported
+
+    """
+    if isinstance(mp_client, int) and mp_client > 0:
+        return ThreadPoolRunner(ThreadPoolExecutor(max_workers=mp_client), log_file)
+    elif isinstance(mp_client, ThreadPoolExecutor):
+        return ThreadPoolRunner(mp_client, log_file)
+    if isinstance(mp_client, SLURMConfig):
+        return SLURMRunner(mp_client, log_file)
+    else:
+        raise NotImplementedError(f"Unsupported client type: {type(mp_client)}")

xutk-0.3.4/xutk/perf.py

@@ -0,0 +1,89 @@
+"""Performance measurement utilities.
+
+This module provides tools for measuring and analyzing code performance,
+including execution time and memory usage.
+"""
+
+import os
+import time
+import tracemalloc
+import psutil
+from contextlib import ContextDecorator
+from pathlib import Path
+
+
+class Perf(ContextDecorator):
+    """Performance measurement context manager and decorator.
+
+    Measures wall time, CPU time, memory usage and allocations.
+    """
+
+    def __init__(self, label: str | None = None, print_output: bool = True) -> None:
+        """Initialize performance measurement context.
+
+        Args:
+            label: Optional label for the measurement
+            print_output: Whether to print results automatically
+
+        """
+        self.label = label
+        self.cpu = 0
+        self.wall = 0
+        self.ratio = 0
+        self.rss = 0
+        self.peak = 0
+        self.final_rss = 0
+        self.print = print_output
+
+        self.label_str = ""
+
+    def __enter__(self) -> "Perf":
+        """Enter the performance measurement context.
+
+        Returns:
+            The Perf instance itself
+
+        """
+        self.proc = psutil.Process(os.getpid())
+        self.t0_cpu = time.process_time()
+        self.t0_wall = time.time()
+        self.t0_rss = self.proc.memory_info().rss
+        tracemalloc.start()
+        return self
+
+    def __exit__(self, *exc) -> None:  # noqa: ANN002
+        """Exit the performance measurement context.
+
+        Args:
+            *exc: Exception information if any
+
+        """
+        self.cpu = time.process_time() - self.t0_cpu
+        self.wall = time.time() - self.t0_wall
+        self.ratio = self.cpu / self.wall if self.wall > 0 else 0
+        self.rss = self.proc.memory_info().rss - self.t0_rss
+        self.peak = tracemalloc.get_traced_memory()[1] / 1_048_576
+        self.final_rss = self.proc.memory_info().rss / 1_048_576
+        tracemalloc.stop()
+
+        self.label_str = f"[{self.label}]" if self.label is not None else ""
+        self.label_str += f"\tWall Time: {self.wall:.3f}s"
+        self.label_str += f"\tCPU Time: {self.cpu:.3f}s({self.ratio:.2f}x)"
+        self.label_str += f"\tΔRSS: {self.rss / 1_048_576:.1f} MiB"
+        self.label_str += f"\tPeak alloc: {self.peak:.1f} MiB"
+        self.label_str += f"\tFinal RSS: {self.final_rss:.1f} MiB"
+
+        if self.print:
+            print(self.label_str)
+
+
+def fadvise_remove_cache(path: str) -> None:
+    """Advise OS to remove file from cache.
+
+    Args:
+        path: Path to the file to remove from cache
+
+    """
+    with Path(path).open("rb") as f:
+        fd = f.fileno()
+        os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED)

xutk-0.3.4/xutk/verchk.py

@@ -0,0 +1,34 @@
+"""Manage packages and versions."""
+
+from importlib import metadata
+import packaging.version as pv
+
+
+def check_version(version_need: str, package_name: str) -> None:
+    """Check if the installed version of a package satisfies the required version.
+
+    Args:
+        version_need (str): The minimum required version string (e.g., '0.3.0a1').
+        package_name (str): The name of the package to check (e.g., 'plotma').
+
+    Raises:
+        ImportError: If the package is not installed or the version is incompatible.
+
+    """
+    # Get versions: required, installed
+    required = pv.parse(version_need)
+    try:
+        installed = pv.parse(metadata.version(package_name))
+    except metadata.PackageNotFoundError:
+        raise ImportError(f"错误: {package_name} 包未安装。")
+
+    # Check version compatibility
+    if installed >= required:  # satisfied minimum version required
+        if installed.release[0] == required.release[0]:  # satisfied MAJOR
+            if required.release[0] > 0 or installed.release[1] == required.release[1]:
+                # version compatible
+                return
+    raise ImportError(
+        f"版本不兼容：需要 {package_name} = {version_need}, 当前为 {installed}"
+    )
+    return

xutk-0.3.4/xutk.egg-info/PKG-INFO

@@ -0,0 +1,89 @@
+Metadata-Version: 2.4
+Name: xutk
+Version: 0.3.4
+Summary: Xulab toolkit in python
+Requires-Python: <3.15,>=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: colorama>=0.4.6
+Requires-Dist: psutil>=7.1.0
+
+# Xulab Useful Toolkit in Python (xutk)
+
+![Maintenance](https://img.shields.io/maintenance/Zelin2001/2025)
+![PyPI - Python Version](https://img.shields.io/badge/python-3.11|3.12|3.13|3.14|3.14t-blue.svg)
+
+## 导入项目
+
+```bash
+pip install \
+  --trusted-host "gitea.xulab.ion.ac.cn" \
+  --extra-index-url "http://gitea.xulab.ion.ac.cn/api/packages/zelin2001/pypi/simple/" \
+  xutk
+```
+
+或者使用 uv:
+
+```bash
+uv add --index http://gitea.xulab.ion.ac.cn/api/packages/zelin2001/pypi/simple xutk
+```
+
+## 快速开始
+
+```python
+from xutk import verchk, log, mp_runner
+from pathlib import Path
+
+# 版本检查
+verchk.check_version("0.2.0", "some_package")
+
+# 日志记录
+logger = log.CtxLogger("my_app")
+logger.info({"module": "processor", "file": "data.csv"}, "Processing started")
+
+# 多进程运行器
+runner = mp_runner.mprunner_factory(2, log_file=Path("process.log")) # 进程队列，取 2 个运行
+runner.batch_run(["echo", "-E", f"test{num}"] for num in range(1,25)) # 排队 25 个进程
+```
+
+**建议在项目 `__init__.py` 创建 CtxLogger("my_app")，避免实例化顺序问题。**
+
+## 项目结构
+
+```text
+xutk/
+├── xutk/
+│   ├── __init__.py      # 包初始化
+│   ├── log.py           # 日志记录功能
+│   ├── mp_runner.py     # 多进程运行管理
+│   ├── perf.py          # 快速资源检查
+│   └── verchk.py        # 版本检查功能
+├── test/
+│   ├── integration/     # 集成测试
+│   └── unit/            # 单元测试
+├── gitea/
+│   └── workflows/       # CI 工作流
+├── pyproject.toml       # 项目配置
+└── README.md            # 项目文档
+```
+
+## 开发
+
+### 安装开发依赖
+
+```bash
+uv sync --all-extras
+```
+
+### 运行测试
+
+```bash
+uv run pytest
+```
+
+### 代码检查
+
+```bash
+uv run ruff check      # 代码风格检查
+uv run ty check        # 类型检查
+uv run pytest          # 样例测试
+```

xutk-0.3.4/xutk.egg-info/SOURCES.txt

@@ -0,0 +1,12 @@
+README.md
+pyproject.toml
+xutk/__init__.py
+xutk/log.py
+xutk/mp_runner.py
+xutk/perf.py
+xutk/verchk.py
+xutk.egg-info/PKG-INFO
+xutk.egg-info/SOURCES.txt
+xutk.egg-info/dependency_links.txt
+xutk.egg-info/requires.txt
+xutk.egg-info/top_level.txt

xutk-0.3.4/xutk.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

xutk-0.3.4/xutk.egg-info/requires.txt

@@ -0,0 +1,2 @@
+colorama>=0.4.6
+psutil>=7.1.0

xutk-0.3.4/xutk.egg-info/top_level.txt

@@ -0,0 +1 @@
+xutk