gpu-burn 2.2.1__py3-none-any.whl

gpu_burn/__init__.py

@@ -0,0 +1,3 @@
+"""PaddlePaddle GPU utilization workload."""
+
+__version__ = "2.2.1"

gpu_burn/__main__.py

@@ -0,0 +1,8 @@
+import sys
+
+from .cli import main
+
+
+if __name__ == "__main__":
+    sys.argv[0] = "gpu-burn"
+    raise SystemExit(main())

gpu_burn/cli.py

@@ -0,0 +1,998 @@
+"""Run and manage adaptive per-user PaddlePaddle GPU workloads."""
+
+from __future__ import annotations
+
+import argparse
+from dataclasses import dataclass
+import fcntl
+import json
+import math
+import multiprocessing as mp
+import os
+from pathlib import Path
+import signal
+import subprocess
+import sys
+import time
+from typing import Any, TextIO
+
+from . import __version__
+
+
+PROGRAM_NAME = "gpu-burn"
+DEFAULT_UTILIZATION = 100.0
+DEFAULT_STOP_TIMEOUT = 15.0
+DEFAULT_PAUSE_PERCENT = 5.0
+DEFAULT_RESUME_PERCENT = 2.0
+DEFAULT_IDLE_MINUTES = 5.0
+DEFAULT_POLL_INTERVAL = 1.0
+NVIDIA_SMI = os.environ.get("GPU_BURN_NVIDIA_SMI", "nvidia-smi")
+SPINNER_FRAMES = ("⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏")
+
+
+class Spinner:
+    def __init__(self, message: str, stream: TextIO = sys.stdout) -> None:
+        self.message = message
+        self.stream = stream
+        self.frame = 0
+        self.active = False
+        disabled = os.environ.get("GPU_BURN_NO_SPINNER", "").lower()
+        self.animated = bool(stream.isatty()) and disabled not in {"1", "true", "yes"}
+
+    def start(self) -> None:
+        self.active = True
+        if self.animated:
+            self.stream.write("\x1b[?25l")
+            self.tick()
+        else:
+            self.stream.write(f"{self.message}\n")
+            self.stream.flush()
+
+    def tick(self) -> None:
+        if not self.active or not self.animated:
+            return
+        glyph = SPINNER_FRAMES[self.frame % len(SPINNER_FRAMES)]
+        self.frame += 1
+        self.stream.write(f"\r\x1b[2K\x1b[36m{glyph}\x1b[0m {self.message}")
+        self.stream.flush()
+
+    def finish(self, message: str, success: bool = True) -> None:
+        if not self.active:
+            return
+        if self.animated:
+            glyph = "✓" if success else "✗"
+            color = "32" if success else "31"
+            self.stream.write(
+                f"\r\x1b[2K\x1b[{color}m{glyph}\x1b[0m {message}\x1b[?25h\n"
+            )
+            self.stream.flush()
+        elif not success:
+            self.stream.write(f"{message}\n")
+            self.stream.flush()
+        self.active = False
+
+    def close(self) -> None:
+        if self.active and self.animated:
+            self.stream.write("\r\x1b[2K\x1b[?25h")
+            self.stream.flush()
+        self.active = False
+
+
+@dataclass
+class WorkerState:
+    device_id: int
+    process: mp.Process | None = None
+    stop_event: Any | None = None
+    idle_since: float | None = None
+    external_memory_percent: float | None = None
+
+
+@dataclass(frozen=True)
+class GPUInfo:
+    uuid: str
+    total_memory_mib: float
+
+
+def state_dir() -> Path:
+    override = os.environ.get("GPU_BURN_STATE_DIR")
+    if override:
+        path = Path(override).expanduser()
+    elif os.environ.get("XDG_RUNTIME_DIR"):
+        path = Path(os.environ["XDG_RUNTIME_DIR"]) / PROGRAM_NAME
+    else:
+        path = Path(f"/tmp/{PROGRAM_NAME}-{os.getuid()}")
+
+    path.mkdir(mode=0o700, parents=True, exist_ok=True)
+    if path.stat().st_uid != os.getuid():
+        raise RuntimeError(f"state directory is not owned by uid {os.getuid()}: {path}")
+    path.chmod(0o700)
+    return path
+
+
+def paths() -> dict[str, Path]:
+    directory = state_dir()
+    return {
+        "directory": directory,
+        "record": directory / "instance.json",
+        "lifetime_lock": directory / "instance.lock",
+        "control_lock": directory / "control.lock",
+        "log": directory / "gpu-burn.log",
+    }
+
+
+def proc_start_time(pid: int) -> int | None:
+    try:
+        stat = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8")
+        fields_after_name = stat[stat.rfind(")") + 2 :].split()
+        return int(fields_after_name[19])
+    except (FileNotFoundError, IndexError, OSError, ValueError):
+        return None
+
+
+def pid_matches(pid: int, uid: int, start_time: int) -> bool:
+    try:
+        process_uid = Path(f"/proc/{pid}").stat().st_uid
+    except (FileNotFoundError, OSError):
+        return False
+    return process_uid == uid == os.getuid() and proc_start_time(pid) == start_time
+
+
+def process_matches(record: dict[str, Any]) -> bool:
+    try:
+        return pid_matches(
+            int(record["pid"]), int(record["uid"]), int(record["proc_start_time"])
+        )
+    except (KeyError, TypeError, ValueError):
+        return False
+
+
+def matching_worker_pids(record: dict[str, Any]) -> list[int]:
+    try:
+        uid = int(record["uid"])
+        return [
+            int(worker["pid"])
+            for worker in record.get("workers", [])
+            if pid_matches(int(worker["pid"]), uid, int(worker["proc_start_time"]))
+        ]
+    except (KeyError, TypeError, ValueError):
+        return []
+
+
+def instance_is_alive(record: dict[str, Any]) -> bool:
+    return process_matches(record) or bool(matching_worker_pids(record))
+
+
+def read_record(record_path: Path) -> dict[str, Any] | None:
+    try:
+        value = json.loads(record_path.read_text(encoding="utf-8"))
+    except (FileNotFoundError, json.JSONDecodeError, OSError):
+        return None
+    return value if isinstance(value, dict) else None
+
+
+def remove_stale_record(record_path: Path) -> None:
+    record = read_record(record_path)
+    if record is not None and instance_is_alive(record):
+        return
+    try:
+        record_path.unlink()
+    except FileNotFoundError:
+        pass
+
+
+def write_record(record_path: Path, record: dict[str, Any]) -> None:
+    temporary = record_path.with_suffix(".tmp")
+    temporary.write_text(json.dumps(record, indent=2) + "\n", encoding="utf-8")
+    temporary.chmod(0o600)
+    temporary.replace(record_path)
+
+
+def acquire_lock(path: Path, nonblocking: bool = False):
+    lock_file = path.open("a+")
+    operation = fcntl.LOCK_EX | (fcntl.LOCK_NB if nonblocking else 0)
+    try:
+        fcntl.flock(lock_file.fileno(), operation)
+    except BlockingIOError:
+        lock_file.close()
+        return None
+    return lock_file
+
+
+def validate_positive(value: float, option: str, allow_zero: bool = False) -> None:
+    valid = math.isfinite(value) and (value >= 0 if allow_zero else value > 0)
+    if not valid:
+        relation = "non-negative" if allow_zero else "greater than zero"
+        raise ValueError(f"{option} must be finite and {relation}")
+
+
+def validate_adaptive_thresholds(pause_percent: float, resume_percent: float) -> None:
+    if not math.isfinite(pause_percent) or not 0 < pause_percent <= 100:
+        raise ValueError("--pause must be greater than 0 and at most 100")
+    if not math.isfinite(resume_percent) or not 0 <= resume_percent < pause_percent:
+        raise ValueError("--resume must be non-negative and lower than --pause")
+
+
+def validate_utilization(value: float) -> None:
+    if not math.isfinite(value) or not 0 < value <= 100:
+        raise ValueError("--util must be greater than 0 and at most 100")
+
+
+def calculate_idle_seconds(compute_seconds: float, utilization: float) -> float:
+    """Return the idle time needed to approximate the requested compute duty cycle."""
+    validate_utilization(utilization)
+    if utilization == 100:
+        return 0.0
+    return max(0.0, compute_seconds * (100.0 / utilization - 1.0))
+
+
+def parse_csv_rows(output: str, columns: int) -> list[list[str]]:
+    rows: list[list[str]] = []
+    for line in output.splitlines():
+        if not line.strip():
+            continue
+        values = [value.strip() for value in line.split(",")]
+        if len(values) != columns:
+            raise RuntimeError(f"unexpected nvidia-smi output: {line}")
+        rows.append(values)
+    return rows
+
+
+def run_nvidia_smi(query: str) -> str:
+    executable = find_nvidia_smi()
+    try:
+        result = subprocess.run(
+            [executable, f"--query-{query}", "--format=csv,noheader,nounits"],
+            check=False,
+            capture_output=True,
+            text=True,
+            timeout=10,
+        )
+    except FileNotFoundError as error:
+        raise RuntimeError(f"nvidia-smi not found: {executable}") from error
+    except subprocess.TimeoutExpired as error:
+        raise RuntimeError("nvidia-smi timed out") from error
+    if result.returncode != 0:
+        details = result.stderr.strip() or f"exit code {result.returncode}"
+        raise RuntimeError(f"nvidia-smi failed: {details}")
+    return result.stdout
+
+
+def find_nvidia_smi() -> str:
+    if NVIDIA_SMI != "nvidia-smi":
+        return NVIDIA_SMI
+    for candidate in (
+        "/usr/bin/nvidia-smi",
+        "/usr/local/bin/nvidia-smi",
+        "/home/opt/cuda_tools/nvidia-smi",
+    ):
+        if Path(candidate).is_file() and os.access(candidate, os.X_OK):
+            return candidate
+    return NVIDIA_SMI
+
+
+def parse_visible_device_tokens(value: str | None) -> list[str] | None:
+    if value is None or not value.strip():
+        return None
+    tokens = [token.strip() for token in value.split(",") if token.strip()]
+    return tokens or None
+
+
+def query_gpu_info(visible_devices: str | None = None) -> dict[int, GPUInfo]:
+    output = run_nvidia_smi("gpu=index,uuid,memory.total")
+    try:
+        physical = {
+            int(index): GPUInfo(uuid, float(total_memory_mib))
+            for index, uuid, total_memory_mib in parse_csv_rows(output, 3)
+        }
+    except ValueError as error:
+        raise RuntimeError("nvidia-smi returned invalid GPU information") from error
+    if any(info.total_memory_mib <= 0 for info in physical.values()):
+        raise RuntimeError("nvidia-smi returned non-positive total GPU memory")
+    tokens = parse_visible_device_tokens(visible_devices)
+    if tokens is None:
+        return physical
+    by_uuid = {info.uuid: info for info in physical.values()}
+    resolved: dict[int, GPUInfo] = {}
+    for logical_index, token in enumerate(tokens):
+        if token in by_uuid:
+            resolved[logical_index] = by_uuid[token]
+            continue
+        try:
+            resolved[logical_index] = physical[int(token)]
+        except (KeyError, ValueError) as error:
+            raise RuntimeError(
+                f"CUDA_VISIBLE_DEVICES references an unknown GPU: {token}"
+            ) from error
+    return resolved
+
+
+def query_gpu_uuids(visible_devices: str | None = None) -> dict[int, str]:
+    return {
+        device: info.uuid for device, info in query_gpu_info(visible_devices).items()
+    }
+
+
+def query_external_memory_mib(
+    device_uuids: dict[int, str], ignored_pids: set[int]
+) -> dict[int, float]:
+    by_uuid = {uuid: device_id for device_id, uuid in device_uuids.items()}
+    usage = {device_id: 0.0 for device_id in device_uuids}
+    output = run_nvidia_smi("compute-apps=gpu_uuid,pid,used_gpu_memory")
+    for uuid, pid_text, memory_text in parse_csv_rows(output, 3):
+        device_id = by_uuid.get(uuid)
+        if device_id is None:
+            continue
+        try:
+            pid = int(pid_text)
+            memory_mib = float(memory_text)
+        except ValueError as error:
+            raise RuntimeError("nvidia-smi returned invalid process memory data") from error
+        if pid not in ignored_pids and math.isfinite(memory_mib):
+            usage[device_id] += max(0.0, memory_mib)
+    return usage
+
+
+def query_external_memory_percent(
+    device_info: dict[int, GPUInfo], ignored_pids: set[int]
+) -> dict[int, float]:
+    usage_mib = query_external_memory_mib(
+        {device: info.uuid for device, info in device_info.items()}, ignored_pids
+    )
+    return {
+        device: usage_mib[device] / info.total_memory_mib * 100.0
+        for device, info in device_info.items()
+    }
+
+
+def query_external_percent_with_retry(
+    device_info: dict[int, GPUInfo], ignored_pids: set[int]
+) -> dict[int, float]:
+    first = query_external_memory_percent(device_info, ignored_pids)
+    if not any(first.values()):
+        return first
+    time.sleep(0.1)
+    return query_external_memory_percent(device_info, ignored_pids)
+
+
+def burn_gpu(
+    device_id: int,
+    matrix_size: int,
+    dtype: str,
+    log_interval: float,
+    utilization: float,
+    stop_event,
+) -> None:
+    os.environ["FLAGS_allocator_strategy"] = "auto_growth"
+
+    import paddle
+
+    paddle.set_device(f"gpu:{device_id}")
+    paddle.seed(2026 + device_id)
+    x = paddle.randn([matrix_size, matrix_size], dtype=dtype)
+    y = paddle.randn([matrix_size, matrix_size], dtype=dtype)
+    iterations = 0
+    started_at = time.monotonic()
+    last_log = started_at
+
+    print(
+        f"[gpu:{device_id}] started pid={os.getpid()} "
+        f"shape={matrix_size}x{matrix_size} dtype={dtype} "
+        f"utilization={utilization:g}%",
+        flush=True,
+    )
+    while not stop_event.is_set():
+        compute_started = time.perf_counter()
+        z = paddle.matmul(x, y)
+        if iterations % 8 == 7:
+            x = paddle.tanh(z)
+        paddle.device.synchronize(f"gpu:{device_id}")
+        compute_seconds = time.perf_counter() - compute_started
+        iterations += 1
+        idle_seconds = calculate_idle_seconds(compute_seconds, utilization)
+        if idle_seconds > 0:
+            stop_event.wait(idle_seconds)
+        now = time.monotonic()
+        if now - last_log >= log_interval:
+            elapsed = now - started_at
+            print(
+                f"[gpu:{device_id}] iterations={iterations} "
+                f"average={iterations / elapsed:.3f} matmul/s",
+                flush=True,
+            )
+            last_log = now
+    paddle.device.synchronize(f"gpu:{device_id}")
+    print(f"[gpu:{device_id}] stopped after {iterations} iterations", flush=True)
+
+
+def parse_device_list(spec: str, available: int) -> list[int]:
+    devices: list[int] = []
+    for part in spec.split(","):
+        part = part.strip()
+        if "-" in part:
+            lo, hi = part.split("-", 1)
+            lo_value, hi_value = int(lo.strip()), int(hi.strip())
+            if lo_value > hi_value:
+                raise ValueError(f"invalid device range: {part}")
+            devices.extend(range(lo_value, hi_value + 1))
+        else:
+            devices.append(int(part))
+    unique: list[int] = []
+    for device in devices:
+        if device < 0 or device >= available:
+            raise ValueError(
+                f"device {device} is out of range; available GPUs are 0..{available - 1}"
+            )
+        if device not in unique:
+            unique.append(device)
+    if not unique:
+        raise ValueError("device list is empty")
+    return unique
+
+
+def resolve_devices(args: argparse.Namespace, available: int) -> list[int]:
+    if available < 1:
+        raise RuntimeError("PaddlePaddle did not detect any CUDA GPUs")
+    if args.devices is not None:
+        return parse_device_list(args.devices, available)
+    gpu_count = available if args.gpus is None else args.gpus
+    if not 1 <= gpu_count <= available:
+        raise ValueError(f"--gpus must be between 1 and {available}, got {gpu_count}")
+    return list(range(gpu_count))
+
+
+def query_paddle_gpu_count() -> int:
+    code = (
+        "import paddle; "
+        "assert paddle.is_compiled_with_cuda(), "
+        "'the installed PaddlePaddle package has no CUDA support'; "
+        "print(paddle.device.cuda.device_count())"
+    )
+    try:
+        result = subprocess.run(
+            [sys.executable, "-c", code],
+            check=False,
+            capture_output=True,
+            text=True,
+            timeout=30,
+        )
+    except subprocess.TimeoutExpired as error:
+        raise RuntimeError("PaddlePaddle GPU preflight timed out") from error
+    if result.returncode != 0:
+        details = result.stderr.strip().splitlines()
+        message = details[-1] if details else f"exit code {result.returncode}"
+        raise RuntimeError(f"PaddlePaddle GPU preflight failed: {message}")
+    try:
+        return int(result.stdout.strip().splitlines()[-1])
+    except (IndexError, ValueError) as error:
+        raise RuntimeError("PaddlePaddle GPU preflight returned an invalid count") from error
+
+
+def validate_workload(args: argparse.Namespace) -> list[int]:
+    if args.matrix_size <= 0:
+        raise ValueError("--size must be greater than zero")
+    validate_positive(args.log_interval, "--log-int")
+    validate_adaptive_thresholds(args.pause_percent, args.resume_percent)
+    validate_positive(args.idle_minutes, "--idle")
+    validate_positive(args.poll_interval, "--poll")
+    validate_utilization(args.utilization)
+    devices = resolve_devices(args, query_paddle_gpu_count())
+    detected = query_gpu_info(os.environ.get("CUDA_VISIBLE_DEVICES"))
+    missing = [device for device in devices if device not in detected]
+    if missing:
+        raise RuntimeError(f"nvidia-smi did not report selected devices: {missing}")
+    return devices
+
+
+def start_worker(state: WorkerState, args: argparse.Namespace, context) -> None:
+    stop_event = context.Event()
+    process = context.Process(
+        target=burn_gpu,
+        args=(
+            state.device_id,
+            args.matrix_size,
+            args.dtype,
+            args.log_interval,
+            args.utilization,
+            stop_event,
+        ),
+        name=f"gpu-burn-{state.device_id}",
+    )
+    process.start()
+    state.process = process
+    state.stop_event = stop_event
+    state.idle_since = None
+    state.external_memory_percent = None
+    print(f"[gpu:{state.device_id}] worker launched pid={process.pid}", flush=True)
+
+
+def stop_worker(state: WorkerState, timeout: float = DEFAULT_STOP_TIMEOUT) -> None:
+    process = state.process
+    if process is None:
+        return
+    if state.stop_event is not None:
+        state.stop_event.set()
+    process.join(timeout=timeout)
+    if process.is_alive():
+        process.terminate()
+        process.join(timeout=5)
+    if process.is_alive():
+        process.kill()
+        process.join(timeout=5)
+    state.process = None
+    state.stop_event = None
+
+
+def worker_record(state: WorkerState) -> dict[str, Any] | None:
+    process = state.process
+    if process is None or process.pid is None or not process.is_alive():
+        return None
+    start_time = proc_start_time(process.pid)
+    if start_time is None:
+        return None
+    return {
+        "device": state.device_id,
+        "pid": process.pid,
+        "proc_start_time": start_time,
+    }
+
+
+def update_record(
+    record_path: Path, record: dict[str, Any], states: dict[int, WorkerState]
+) -> None:
+    workers = [worker_record(state) for state in states.values()]
+    record["workers"] = [worker for worker in workers if worker is not None]
+    record["device_states"] = {
+        str(device): {
+            "state": "running" if state.process is not None else "paused",
+            "external_memory_percent": state.external_memory_percent,
+            "idle_since": state.idle_since,
+        }
+        for device, state in states.items()
+    }
+    write_record(record_path, record)
+
+
+def reconcile_workers(
+    states: dict[int, WorkerState],
+    external_percent: dict[int, float],
+    args: argparse.Namespace,
+    context,
+    now: float,
+) -> bool:
+    failed = False
+    for device, state in states.items():
+        memory_percent = external_percent.get(device, 0.0)
+        process = state.process
+        if process is not None and process.exitcode is not None:
+            print(
+                f"[gpu:{device}] worker exited unexpectedly: {process.exitcode}",
+                file=sys.stderr,
+                flush=True,
+            )
+            state.process = None
+            state.stop_event = None
+            state.idle_since = now
+            failed = True
+            continue
+
+        if process is not None and memory_percent >= args.pause_percent:
+            print(
+                f"[gpu:{device}] pausing: external memory {memory_percent:.3f}% "
+                f">= pause threshold {args.pause_percent:g}%",
+                flush=True,
+            )
+            stop_worker(state)
+            state.external_memory_percent = memory_percent
+            state.idle_since = None
+            continue
+
+        if state.process is None:
+            state.external_memory_percent = memory_percent
+            if memory_percent > args.resume_percent:
+                state.idle_since = None
+            else:
+                if state.idle_since is None:
+                    state.idle_since = now
+                    print(
+                        f"[gpu:{device}] idle timer started; restart in "
+                        f"{args.idle_minutes:g} min if memory remains at or below "
+                        f"{args.resume_percent:g}%",
+                        flush=True,
+                    )
+                if now - state.idle_since >= args.idle_minutes * 60.0:
+                    print(f"[gpu:{device}] idle period reached; restarting", flush=True)
+                    start_worker(state, args, context)
+    return failed
+
+
+def command_run(args: argparse.Namespace) -> int:
+    runtime_paths = paths()
+    lifetime_lock = acquire_lock(runtime_paths["lifetime_lock"], nonblocking=True)
+    if lifetime_lock is None:
+        print(f"{PROGRAM_NAME} is already running for uid {os.getuid()}", file=sys.stderr)
+        return 1
+    remove_stale_record(runtime_paths["record"])
+    states: dict[int, WorkerState] = {}
+    record: dict[str, Any] | None = None
+    stopping = False
+
+    def request_stop(_signum=None, _frame=None) -> None:
+        nonlocal stopping
+        stopping = True
+
+    signal.signal(signal.SIGINT, request_stop)
+    signal.signal(signal.SIGTERM, request_stop)
+    context = mp.get_context("spawn")
+    try:
+        devices = validate_workload(args)
+        device_info = {
+            device: info
+            for device, info in query_gpu_info(
+                os.environ.get("CUDA_VISIBLE_DEVICES")
+            ).items()
+            if device in devices
+        }
+        start_time = proc_start_time(os.getpid())
+        if start_time is None:
+            raise RuntimeError("could not read this process start time from /proc")
+        states = {device: WorkerState(device) for device in devices}
+        record = {
+            "pid": os.getpid(),
+            "uid": os.getuid(),
+            "proc_start_time": start_time,
+            "started_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
+            "gpus": len(devices),
+            "devices": devices,
+            "matrix_size": args.matrix_size,
+            "dtype": args.dtype,
+            "log_interval": args.log_interval,
+            "utilization": args.utilization,
+            "pause_percent": args.pause_percent,
+            "resume_percent": args.resume_percent,
+            "idle_minutes": args.idle_minutes,
+            "poll_interval": args.poll_interval,
+        }
+        initial_usage = query_external_percent_with_retry(device_info, {os.getpid()})
+        now = time.monotonic()
+        for device, state in states.items():
+            memory_percent = initial_usage.get(device, 0.0)
+            if memory_percent < args.pause_percent:
+                start_worker(state, args, context)
+            else:
+                state.external_memory_percent = memory_percent
+                print(
+                    f"[gpu:{device}] initially paused: external memory "
+                    f"{memory_percent:.3f}% >= pause threshold {args.pause_percent:g}%",
+                    flush=True,
+                )
+        update_record(runtime_paths["record"], record, states)
+        print(f"Managing GPU devices {devices}; supervisor pid={os.getpid()}", flush=True)
+
+        exit_code = 0
+        while not stopping:
+            time.sleep(args.poll_interval)
+            ignored_pids = {os.getpid()} | {
+                state.process.pid
+                for state in states.values()
+                if state.process is not None and state.process.pid is not None
+            }
+            usage = query_external_percent_with_retry(device_info, ignored_pids)
+            now = time.monotonic()
+            if reconcile_workers(states, usage, args, context, now):
+                exit_code = 1
+            update_record(runtime_paths["record"], record, states)
+        return exit_code
+    finally:
+        for state in states.values():
+            stop_worker(state)
+        current = read_record(runtime_paths["record"])
+        if record is not None and current is not None and current.get("pid") == record["pid"]:
+            try:
+                runtime_paths["record"].unlink()
+            except FileNotFoundError:
+                pass
+        lifetime_lock.close()
+        print(f"{PROGRAM_NAME} stopped", flush=True)
+
+
+def workload_command(command: str, args: argparse.Namespace) -> list[str]:
+    result = [sys.executable, "-m", "gpu_burn", command]
+    if args.devices is not None:
+        result.extend(["--devs", args.devices])
+    elif args.gpus is not None:
+        result.extend(["--gpus", str(args.gpus)])
+    result.extend(
+        [
+            "--size", str(args.matrix_size),
+            "--type", args.dtype,
+            "--log-int", str(args.log_interval),
+            "--util", str(args.utilization),
+            "--pause", str(args.pause_percent),
+            "--resume", str(args.resume_percent),
+            "--idle", str(args.idle_minutes),
+            "--poll", str(args.poll_interval),
+        ]
+    )
+    return result
+
+
+def print_start_summary(
+    record: dict[str, Any], log_path: Path, status: str = "started"
+) -> None:
+    devices = record.get("devices", [])
+    device_states = record.get("device_states", {})
+    worker_pids = {
+        int(worker["device"]): int(worker["pid"])
+        for worker in record.get("workers", [])
+        if "device" in worker and "pid" in worker
+    }
+    running = sum(
+        1
+        for device in devices
+        if device_states.get(str(device), {}).get("state") == "running"
+    )
+    paused = len(devices) - running
+
+    print(f"\n{PROGRAM_NAME} {__version__} {status}")
+    print(f"  supervisor : pid {record['pid']}")
+    print(
+        f"  devices    : {len(devices)} selected, {running} running, {paused} paused"
+    )
+    for device in devices:
+        state = device_states.get(str(device), {})
+        if state.get("state") == "running":
+            pid = worker_pids.get(device)
+            detail = "running" if pid is None else f"running (pid {pid})"
+        else:
+            percent = state.get("external_memory_percent")
+            detail = "paused"
+            if percent is not None:
+                detail += f" (external memory {percent:.3f}%)"
+        print(f"  gpu:{device:<6} : {detail}")
+    print(
+        f"  workload   : {record['matrix_size']}x{record['matrix_size']} "
+        f"{record['dtype']}, util {record.get('utilization', 100):g}%"
+    )
+    print(
+        f"  adaptive   : pause >= {record.get('pause_percent', 0):g}%, "
+        f"resume <= {record.get('resume_percent', 0):g}%, "
+        f"idle {record.get('idle_minutes', 0):g} min, "
+        f"poll {record.get('poll_interval', 0):g}s"
+    )
+    print(f"  log        : {log_path}\n")
+
+
+def command_start(args: argparse.Namespace) -> int:
+    runtime_paths = paths()
+    control_lock = acquire_lock(runtime_paths["control_lock"])
+    assert control_lock is not None
+    try:
+        record = read_record(runtime_paths["record"])
+        if record is not None and instance_is_alive(record):
+            print_start_summary(record, runtime_paths["log"], "already running")
+            return 0
+        remove_stale_record(runtime_paths["record"])
+        spinner = Spinner(f"Starting {PROGRAM_NAME} {__version__} in the background")
+        spinner.start()
+        log_handle = runtime_paths["log"].open("a", encoding="utf-8")
+        try:
+            process = subprocess.Popen(
+                workload_command("run", args),
+                stdin=subprocess.DEVNULL,
+                stdout=log_handle,
+                stderr=subprocess.STDOUT,
+                start_new_session=True,
+                close_fds=True,
+            )
+        finally:
+            log_handle.close()
+        deadline = time.monotonic() + args.start_timeout
+        try:
+            while time.monotonic() < deadline:
+                record = read_record(runtime_paths["record"])
+                if record is not None and process_matches(record):
+                    spinner.finish(f"{PROGRAM_NAME} is ready")
+                    print_start_summary(record, runtime_paths["log"])
+                    return 0
+                return_code = process.poll()
+                if return_code is not None:
+                    message = f"{PROGRAM_NAME} failed to start (exit code {return_code})"
+                    spinner.finish(message, success=False)
+                    print(f"See log: {runtime_paths['log']}", file=sys.stderr)
+                    return 1
+                spinner.tick()
+                time.sleep(0.1)
+            process.send_signal(signal.SIGTERM)
+            message = (
+                f"{PROGRAM_NAME} was not ready within {args.start_timeout:g}s"
+            )
+            spinner.finish(message, success=False)
+            print(f"See log: {runtime_paths['log']}", file=sys.stderr)
+            return 1
+        finally:
+            spinner.close()
+    finally:
+        control_lock.close()
+
+
+def command_stop(args: argparse.Namespace, quiet: bool = False) -> int:
+    runtime_paths = paths()
+    control_lock = acquire_lock(runtime_paths["control_lock"])
+    assert control_lock is not None
+    try:
+        record = read_record(runtime_paths["record"])
+        if record is None or not instance_is_alive(record):
+            remove_stale_record(runtime_paths["record"])
+            if not quiet:
+                print(f"{PROGRAM_NAME} is not running")
+            return 0
+        pid = int(record["pid"])
+        targets = matching_worker_pids(record)
+        if process_matches(record):
+            targets.append(pid)
+        for target in targets:
+            try:
+                os.kill(target, signal.SIGTERM)
+            except ProcessLookupError:
+                pass
+        deadline = time.monotonic() + args.timeout
+        while time.monotonic() < deadline and instance_is_alive(record):
+            time.sleep(0.1)
+        if instance_is_alive(record):
+            if not args.force:
+                print(
+                    f"{PROGRAM_NAME} did not stop within {args.timeout:g}s; "
+                    "retry with stop --force",
+                    file=sys.stderr,
+                )
+                return 1
+            force_targets = matching_worker_pids(record)
+            if process_matches(record):
+                force_targets.append(pid)
+            for target in force_targets:
+                try:
+                    os.kill(target, signal.SIGKILL)
+                except ProcessLookupError:
+                    pass
+            force_deadline = time.monotonic() + 5
+            while time.monotonic() < force_deadline and instance_is_alive(record):
+                time.sleep(0.1)
+        if instance_is_alive(record):
+            print(f"failed to stop {PROGRAM_NAME}; managed processes remain", file=sys.stderr)
+            return 1
+        remove_stale_record(runtime_paths["record"])
+        if not quiet:
+            print(f"{PROGRAM_NAME} stopped")
+        return 0
+    finally:
+        control_lock.close()
+
+
+def command_status(_args: argparse.Namespace) -> int:
+    runtime_paths = paths()
+    record = read_record(runtime_paths["record"])
+    if record is None or not instance_is_alive(record):
+        remove_stale_record(runtime_paths["record"])
+        print(f"status: stopped\nstate: {runtime_paths['directory']}")
+        return 3
+    active_workers = matching_worker_pids(record)
+    device_states = record.get("device_states", {})
+    managed = sum(
+        1 for state in device_states.values() if state.get("state") == "running"
+    )
+    status = "running" if process_matches(record) else "degraded"
+    print(f"status: {status}")
+    print(f"pid: {record['pid']}")
+    print(f"started: {record['started_at']}")
+    print(f"workers: {len(active_workers)}/{record['gpus']} ({managed} running)")
+    print(f"worker_pids: {' '.join(str(pid) for pid in active_workers)}")
+    for device in record.get("devices", []):
+        state = device_states.get(str(device), {})
+        memory = state.get("external_memory_percent")
+        suffix = "" if memory is None else f", external_memory={memory:.3f}%"
+        print(f"device {device}: {state.get('state', 'unknown')}{suffix}")
+    print(
+        f"workload: {record['matrix_size']}x{record['matrix_size']} {record['dtype']}, "
+        f"utilization {record.get('utilization', 100):g}%"
+    )
+    print(
+        f"adaptive: pause {record.get('pause_percent', 0):g}%, "
+        f"resume {record.get('resume_percent', 0):g}%, "
+        f"idle {record.get('idle_minutes', 0):g} min"
+    )
+    print(f"log: {runtime_paths['log']}")
+    return 0
+
+
+def command_logs(args: argparse.Namespace) -> int:
+    log_path = paths()["log"]
+    if not log_path.exists():
+        print(f"no log file: {log_path}", file=sys.stderr)
+        return 1
+    command = ["tail", "-n", str(args.lines)]
+    if args.follow:
+        command.append("-f")
+    command.append(str(log_path))
+    try:
+        return subprocess.run(command, check=False).returncode
+    except KeyboardInterrupt:
+        return 130
+
+
+def command_restart(args: argparse.Namespace) -> int:
+    stop_args = argparse.Namespace(timeout=args.timeout, force=args.force)
+    if command_stop(stop_args, quiet=True) != 0:
+        return 1
+    return command_start(args)
+
+
+def add_workload_arguments(parser: argparse.ArgumentParser) -> None:
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument("-g", "--gpus", type=int, default=None, help="GPU count (default: all)")
+    group.add_argument(
+        "-d", "--devs", dest="devices", type=str, default=None,
+        help="GPU IDs or ranges, e.g. '0,2,5' or '1-3,7'",
+    )
+    parser.add_argument("-m", "--size", dest="matrix_size", type=int, default=16384)
+    parser.add_argument("-t", "--type", dest="dtype", choices=("float16", "float32"), default="float16")
+    parser.add_argument("-l", "--log-int", dest="log_interval", type=float, default=30.0)
+    parser.add_argument(
+        "-u", "--util", dest="utilization", type=float, default=DEFAULT_UTILIZATION,
+        metavar="PERCENT", help=f"compute utilization percentage (default: {DEFAULT_UTILIZATION:g})",
+    )
+    parser.add_argument(
+        "-P", "--pause", dest="pause_percent", type=float, default=DEFAULT_PAUSE_PERCENT,
+        metavar="PERCENT", help=f"pause at this external memory percentage (default: {DEFAULT_PAUSE_PERCENT:g})",
+    )
+    parser.add_argument(
+        "-R", "--resume", dest="resume_percent", type=float, default=DEFAULT_RESUME_PERCENT,
+        metavar="PERCENT", help=f"start idle timer at or below this percentage (default: {DEFAULT_RESUME_PERCENT:g})",
+    )
+    parser.add_argument(
+        "-i", "--idle", dest="idle_minutes", type=float, default=DEFAULT_IDLE_MINUTES,
+        metavar="MINUTES", help=f"continuous idle minutes before restart (default: {DEFAULT_IDLE_MINUTES:g})",
+    )
+    parser.add_argument(
+        "-p", "--poll", dest="poll_interval", type=float, default=DEFAULT_POLL_INTERVAL,
+        metavar="SECONDS", help=f"GPU memory polling interval (default: {DEFAULT_POLL_INTERVAL:g})",
+    )
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("-V", "--version", action="version", version=f"%(prog)s {__version__}")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+    run_parser = subparsers.add_parser("run", help="run in the foreground")
+    add_workload_arguments(run_parser)
+    run_parser.set_defaults(handler=command_run)
+    start_parser = subparsers.add_parser("start", help="start in the background")
+    add_workload_arguments(start_parser)
+    start_parser.add_argument("--start-wait", dest="start_timeout", type=float, default=30.0)
+    start_parser.set_defaults(handler=command_start)
+    stop_parser = subparsers.add_parser("stop", help="stop this user's instance")
+    stop_parser.add_argument("-T", "--wait", dest="timeout", type=float, default=DEFAULT_STOP_TIMEOUT)
+    stop_parser.add_argument("-f", "--force", action="store_true")
+    stop_parser.set_defaults(handler=command_stop)
+    status_parser = subparsers.add_parser("status", help="show instance status")
+    status_parser.set_defaults(handler=command_status)
+    logs_parser = subparsers.add_parser("logs", help="show this user's log")
+    logs_parser.add_argument("-n", "--lines", type=int, default=20)
+    logs_parser.add_argument("-f", "--follow", action="store_true")
+    logs_parser.set_defaults(handler=command_logs)
+    restart_parser = subparsers.add_parser("restart", help="stop and start an instance")
+    add_workload_arguments(restart_parser)
+    restart_parser.add_argument("--start-wait", dest="start_timeout", type=float, default=30.0)
+    restart_parser.add_argument("-T", "--wait", dest="timeout", type=float, default=DEFAULT_STOP_TIMEOUT)
+    restart_parser.add_argument("-f", "--force", action="store_true")
+    restart_parser.set_defaults(handler=command_restart)
+    return parser
+
+
+def main() -> int:
+    parser = build_parser()
+    args = parser.parse_args()
+    try:
+        return args.handler(args)
+    except (OSError, RuntimeError, ValueError) as error:
+        print(f"{PROGRAM_NAME}: {error}", file=sys.stderr)
+        return 1

gpu_burn-2.2.1.dist-info/METADATA

@@ -0,0 +1,193 @@
+Metadata-Version: 2.4
+Name: gpu-burn
+Version: 2.2.1
+Summary: Control GPU compute utilization with a managed PaddlePaddle workload
+License-Expression: MIT
+Project-URL: Source, https://github.com/cangtianhuang/gpu-burn
+Project-URL: Issues, https://github.com/cangtianhuang/gpu-burn/issues
+Project-URL: PaddlePaddle, https://www.paddlepaddle.org.cn/
+Keywords: gpu,paddlepaddle,cuda,stress-test,utilization
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: System Administrators
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: System :: Hardware
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: paddlepaddle-gpu<4.0.0,>=2.6.2
+Dynamic: license-file
+
+# gpu-burn
+
+`gpu-burn` runs a configurable matrix-multiplication workload on one or more
+NVIDIA GPUs using PaddlePaddle. It dynamically yields each GPU when another
+process starts using a configured percentage of device memory, then resumes after
+that GPU has remained idle for the configured period.
+
+This project is not related to the CUDA/C++ project
+[`wilicc/gpu-burn`](https://github.com/wilicc/gpu-burn). It does not validate
+calculation results and should not be used as a hardware fault or ECC diagnostic.
+
+## Requirements
+
+- Linux and Python 3.10 or newer
+- NVIDIA GPU and a compatible CUDA driver
+- **PaddlePaddle with CUDA support**
+
+The PyPI package explicitly depends on `paddlepaddle-gpu>=2.6.2,<4.0.0`.
+PaddlePaddle publishes different packages and indexes for some CUDA versions, so
+install the build recommended by the
+[official PaddlePaddle installation guide](https://www.paddlepaddle.org.cn/install/quick)
+when the default PyPI build does not match your CUDA environment.
+
+## Installation
+
+Install from PyPI:
+
+```bash
+python -m pip install gpu-burn
+```
+
+For a CUDA-specific PaddlePaddle build, install PaddlePaddle first and then avoid
+letting pip replace it:
+
+```bash
+# Install the appropriate paddlepaddle-gpu build from the official Paddle index.
+python -m pip install --no-deps gpu-burn
+```
+
+Install from a source checkout:
+
+```bash
+python -m pip install .
+```
+
+## Usage
+
+Run on all detected GPUs in the foreground at full utilization:
+
+```bash
+gpu-burn run
+```
+
+Select devices and target an approximate compute duty cycle:
+
+```bash
+gpu-burn run --devs 0,2-3 --util 60
+```
+
+Configure adaptive yielding with hysteresis. This pauses only the affected GPU
+when external compute processes use at least 5% of its total memory. Its worker
+restarts after usage remains at or below 2% for 5 continuous minutes:
+
+```bash
+gpu-burn run -d 0,2-3 -u 60 -P 5 -R 2 -i 5 -p 1
+```
+
+Run as a per-user background process:
+
+```bash
+gpu-burn start --gpus 2 --util 80
+gpu-burn status
+gpu-burn logs -f
+gpu-burn stop
+```
+
+With no options, `gpu-burn start` selects every Paddle-visible GPU and uses:
+
+- `16384x16384` float16 matrix multiplication
+- 100% compute duty-cycle target
+- pause at 5% external memory, resume at or below 2%
+- 5 continuous idle minutes before restart
+- 1-second monitoring interval
+
+After the supervisor becomes ready, startup output includes the resolved GPU
+count, each device's running or paused state, worker PIDs, workload, adaptive
+policy, supervisor PID, and log path:
+
+```text
+✓ gpu-burn is ready
+
+gpu-burn 2.2.1 started
+  supervisor : pid 42000
+  devices    : 2 selected, 2 running, 0 paused
+  gpu:0      : running (pid 42010)
+  gpu:1      : running (pid 42011)
+  workload   : 16384x16384 float16, util 100%
+  adaptive   : pause >= 5%, resume <= 2%, idle 5 min, poll 1s
+  log        : /tmp/gpu-burn-1000/gpu-burn.log
+```
+
+On an interactive terminal, the first line is an animated spinner while Paddle
+and NVIDIA devices are checked. Redirected output automatically uses plain text.
+Set `GPU_BURN_NO_SPINNER=1` to disable animation explicitly.
+
+The main workload options are:
+
+| Option | Meaning | Default |
+| --- | --- | --- |
+| `-g`, `--gpus N` | Use the first `N` visible GPUs | all |
+| `-d`, `--devs LIST` | Device IDs/ranges such as `0,2-3` | all |
+| `-u`, `--util PERCENT` | Approximate compute duty cycle in `(0, 100]` | `100` |
+| `-m`, `--size N` | Square matrix dimension | `16384` |
+| `-t`, `--type TYPE` | `float16` or `float32` | `float16` |
+| `-l`, `--log-int SECONDS` | Worker progress log interval | `30` |
+| `-P`, `--pause PERCENT` | Pause at this external memory percentage | `5` |
+| `-R`, `--resume PERCENT` | Start the idle timer at or below this percentage | `2` |
+| `-i`, `--idle MINUTES` | Continuous idle minutes before restart | `5` |
+| `-p`, `--poll SECONDS` | Memory polling interval | `1` |
+
+`--gpus` and `--devs` are mutually exclusive. Lower utilization is
+implemented by alternating synchronized matrix multiplication with idle time, so
+the value observed by monitoring tools can vary with sampling intervals and GPU
+power-management behavior.
+
+The supervisor queries `nvidia-smi` compute applications and excludes its own
+worker PIDs. Memory from other compute processes is summed per selected GPU and
+divided by that GPU's total physical memory. At `--pause`, only that GPU's
+worker is stopped, which releases its PaddlePaddle tensors and GPU memory. Once
+usage reaches `--resume` or lower, it must remain there for `--idle` minutes
+before a new worker starts. `--resume` must be lower than `--pause`; this
+hysteresis prevents repeated stop/start cycles near one threshold. Display-only
+contexts absent from NVIDIA's compute-app list are not considered.
+Numeric and UUID-based `CUDA_VISIBLE_DEVICES` remapping is honored, so CLI device
+IDs remain the same logical IDs that PaddlePaddle exposes.
+
+## State and cleanup
+
+Only one managed instance is allowed per user. Runtime state and logs are stored
+under `$GPU_BURN_STATE_DIR`, `$XDG_RUNTIME_DIR/gpu-burn`, or
+`/tmp/gpu-burn-$UID`, in that order. `gpu-burn stop --force` sends `SIGKILL` if a
+worker does not stop before the configured timeout.
+
+## Development
+
+The PEP 517 build requirement uses a compatible lower bound
+(`setuptools>=77`) instead of pinning one exact backend release. An exact pin
+makes builds byte-for-byte easier to reproduce, but also forces every source
+installer to download that precise release and can unnecessarily break offline
+or constrained environments. This pure-Python project does not need `wheel` in
+`build-system.requires`; setuptools implements the wheel build hook itself.
+Release CI pins the user-facing `build` and `twine` tools separately.
+
+```bash
+python -m pip install --no-deps -e .
+python -m pytest
+python -m build
+```
+
+Run the opt-in adaptive lifecycle test on a real GPU:
+
+```bash
+GPU_BURN_TEST_DEVICE=0 python tests/real_gpu_dynamic.py
+```
+
+## License
+
+MIT

gpu_burn-2.2.1.dist-info/RECORD

@@ -0,0 +1,9 @@
+gpu_burn/__init__.py,sha256=DNInW4C8-cRGoEVAkZI82RBdmodZVoPVLXsMhdg2E0M,68
+gpu_burn/__main__.py,sha256=XiQVCf1vRCFHcKwUEtDlvg8ZoL0INNbirGwZ7_009D8,121
+gpu_burn/cli.py,sha256=Q_BnaJ2bOaefDyu99_fUS_y-Egu1Np0AyE5uhSx54rY,37222
+gpu_burn-2.2.1.dist-info/licenses/LICENSE,sha256=DfEWpDJkKpy0NF2NBXgCHShN0qh8qG0EI-FtytAxf3w,1079
+gpu_burn-2.2.1.dist-info/METADATA,sha256=fFNndy567rhDY8q9BYBXnUX3DLNvhksNhAb5OHNz6ow,6943
+gpu_burn-2.2.1.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+gpu_burn-2.2.1.dist-info/entry_points.txt,sha256=F298jeoHF2kwKhjHztcAoek7ks6tqzwbyQrHXAVZ8kg,47
+gpu_burn-2.2.1.dist-info/top_level.txt,sha256=J6_AYXWsvy93qHY6NGhvJWnzLPuWn3o_nDdk38er9w0,9
+gpu_burn-2.2.1.dist-info/RECORD,,

gpu_burn-2.2.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (83.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

gpu_burn-2.2.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+gpu-burn = gpu_burn.cli:main

gpu_burn-2.2.1.dist-info/licenses/LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2026 gpu-burn contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

gpu_burn-2.2.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+gpu_burn