rexprun 0.2.0__tar.gz → 0.2.2__tar.gz

{rexprun-0.2.0 → rexprun-0.2.2}/MANIFEST.in

@@ -6,5 +6,6 @@ exclude .agents
 
 # Exclude caches and build leftovers from sdists
 global-exclude __pycache__ *.py[cod]
+global-exclude .DS_Store
 recursive-exclude * __pycache__
 exclude HANDOFF.md

{rexprun-0.2.0/rexprun.egg-info → rexprun-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rexprun
-Version: 0.2.0
+Version: 0.2.2
 Summary: Research experiment orchestration with resource-aware scheduling, isolated environments, checkpoint recovery, and Ray Tune searches on GPUs and Ascend NPUs.
 Author: REXP contributors
 License: MIT

{rexprun-0.2.0 → rexprun-0.2.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rexprun"
-version = "0.2.0"
+version = "0.2.2"
 description = "Research experiment orchestration with resource-aware scheduling, isolated environments, checkpoint recovery, and Ray Tune searches on GPUs and Ascend NPUs."
 readme = "README.md"
 requires-python = ">=3.10"

{rexprun-0.2.0 → rexprun-0.2.2}/rexp/cli/main.py

@@ -284,7 +284,7 @@ def _gpu_summary(node: dict) -> str:
             return f"{sum(names.values())} {_accelerator_label(node)}s ({models})"
     count = node.get("gpu_count") or 0
     if not count:
-        return "none"
+        return "probe failed; run rexp doctor" if node.get("accelerator_probe_error") else "none"
     name = node.get("gpu_name") or "unknown model"
     return f"{count} × {name}"
 
@@ -478,6 +478,10 @@ def init(
             env_setup.prepare_runtime(paths, manager)
             node.environment_manager = manager
             save_nodes_config(paths, node)
+            command = env_setup.register_command(manager.runtime_python)
+            console.print(f"Global command: {command['path']}")
+            if command.get("new_terminal_required"):
+                console.print("Open a new terminal to load the updated PATH; the absolute command above works now.")
             console.print(f"Platform runtime: {manager.runtime_python}")
             console.print(f"Managed experiment directory: {paths.root / 'env'}")
             console.print("Preparing default experiment environment...")
@@ -536,6 +540,8 @@ def doctor(json_output: bool = typer.Option(False, "--json")):
     ):
         table.add_row(item, str(value))
     console.print(table)
+    if hardware.get("accelerator_probe_error"):
+        console.print(f"Accelerator probe failed: {hardware['accelerator_probe_error']}", markup=False)
     _print_environment_report(info["environments"])
     _print_default_env_note(info.get("default_experiment_environment"))
 

{rexprun-0.2.0 → rexprun-0.2.2}/rexp/cli/templates/default/README.md

@@ -51,7 +51,7 @@
 ## 使用
 
 ```bash
-rexp submit {{project}} {{version}} --config configs/smoke.yaml
+rexp submit {{project}} {{version}} --config <PLATFORM_ROOT>/research/{{project}}/{{version}}/configs/smoke.yaml
 rexp start
 rexp status
 rexp run eval <run_id>

{rexprun-0.2.0 → rexprun-0.2.2}/rexp/cli/templates/dlm_kv_staleness/README.md

@@ -16,7 +16,7 @@ Diffusion steps 之间的 KV state 是否存在 temporal locality / sparsity？
 
 1. 确认 CUDA 环境（`rexp doctor`，环境需 cu128 wheel 的 PyTorch）。
 2. 把 LLaDA checkpoint 放到 `PLATFORM_ROOT/models_store/llada`。
-3. `rexp submit {{project}} {{version}} --config configs/smoke.yaml`
+3. `rexp submit {{project}} {{version}} --config <PLATFORM_ROOT>/research/{{project}}/{{version}}/configs/smoke.yaml`
 4. `rexp start` / `rexp status`
 5. Smoke 通过后跑 `configs/full.yaml`。
 

{rexprun-0.2.0 → rexprun-0.2.2}/rexp/cli/templates/ray_tune/README.md

@@ -4,7 +4,7 @@
 登记该环境并在配置中填写 `experiment.environment`，然后：
 
 ```sh
-rexp submit {{project}} {{version}} --config configs/smoke.yaml
+rexp submit {{project}} {{version}} --config <PLATFORM_ROOT>/research/{{project}}/{{version}}/configs/smoke.yaml
 rexp start
 rexp tune list
 rexp tune show run_001

{rexprun-0.2.0 → rexprun-0.2.2}/rexp/cli/templates/ray_tune/experiment/trainable.py

@@ -20,6 +20,9 @@ def trainable(config):
         state = artifacts / 'checkpoint-staging'
         state.mkdir(exist_ok=True)
         (state / 'state.json').write_text(json.dumps({'step': step + 1}))
-        tune.report({'score': score, 'samples_completed': step + 1},
-                    checkpoint=tune.Checkpoint.from_directory(str(state)))
+        metrics = {'score': score, 'samples_completed': step + 1}
+        if step + 1 == 4:
+            tune.report(metrics, checkpoint=tune.Checkpoint.from_directory(str(state)))
+        else:
+            tune.report(metrics)
     (artifacts / 'result.json').write_text(json.dumps({'x': config['x'], 'score': score}))

{rexprun-0.2.0 → rexprun-0.2.2}/rexp/cli/templates/ray_tune_npu/README.md

@@ -5,7 +5,7 @@
 此示例执行真实 NPU 标量运算和梯度校验，搜索 x=2。`resources.gpu_count: 2` 是搜索池的两个**逻辑设备**；A3 一张物理卡可能正好暴露这两个设备。`tune.gpus_per_trial: 1` 表示每个尝试使用一个逻辑 NPU，内部使用 `npu:0`。单逻辑设备实例将池、并发都改成 1。
 
 ```sh
-rexp submit {{project}} {{version}} --config configs/smoke.yaml
+rexp submit {{project}} {{version}} --config <PLATFORM_ROOT>/research/{{project}}/{{version}}/configs/smoke.yaml
 rexp start
 rexp tune show run_001
 ```

{rexprun-0.2.0 → rexprun-0.2.2}/rexp/core/accelerators.py

@@ -6,9 +6,50 @@ Importing this module does not import PyTorch or initialize a device runtime.
 from __future__ import annotations
 
 import json
+import os
+import platform
+from pathlib import Path
 import re
 
 
+def ascend_runtime_environment(inherited=None):
+    """Use installed vendor libraries for non-login SSH, without running shell profiles.
+
+    Preserve user-supplied paths and container masks. Only existing standard
+    installation directories are appended; no driver/framework installation.
+    """
+    env = dict(os.environ if inherited is None else inherited)
+    root = Path('/usr/local/Ascend')
+    # CANN 9 images use cann/; older installations use ascend-toolkit/latest.
+    # Select one installation, preserving an explicitly configured version.
+    toolkit = next((path for path in [
+        Path(env[key]) for key in ('ASCEND_HOME_PATH', 'ASCEND_TOOLKIT_HOME') if env.get(key)
+    ] + [root / 'cann', root / 'ascend-toolkit/latest'] if path.is_dir()), None)
+    paths = {
+        'PATH': [root / 'driver/tools', root / 'driver/bin'],
+        'LD_LIBRARY_PATH': [root / 'driver/lib64', root / 'driver/lib64/common',
+                            root / 'driver/lib64/driver'],
+    }
+    if toolkit:
+        paths['PATH'] += [toolkit / 'bin', toolkit / 'tools/ccec_compiler/bin']
+        paths['LD_LIBRARY_PATH'] += [toolkit / 'lib64', toolkit / 'lib64/plugin/opskernel',
+            toolkit / 'lib64/plugin/nnengine',
+            toolkit / f'opp/built-in/op_impl/ai_core/tbe/op_tiling/lib/linux/{platform.machine()}']
+        paths['PYTHONPATH'] = [toolkit / 'python/site-packages', toolkit / 'opp/built-in/op_impl/ai_core/tbe']
+        for key, path in {'ASCEND_HOME_PATH': toolkit, 'ASCEND_TOOLKIT_HOME': toolkit,
+                          'ASCEND_OPP_PATH': toolkit / 'opp', 'ASCEND_AICPU_PATH': toolkit}.items():
+            if path.is_dir():
+                env.setdefault(key, str(path))
+    for key, candidates in paths.items():
+        values = [v for v in env.get(key, '').split(os.pathsep) if v]
+        for path in candidates:
+            if path.is_dir() and str(path) not in values:
+                values.append(str(path))
+        if values:
+            env[key] = os.pathsep.join(values)
+    return env
+
+
 def ascend_visible_ids(mask: str | None) -> set[int] | None:
     """Accept complete, increasing runtime masks; malformed masks expose nothing.
 

{rexprun-0.2.0 → rexprun-0.2.2}/rexp/core/provenance.py

@@ -14,6 +14,8 @@ from rexp.core.storage import write_json, write_text_atomic
 NPU_PROBE = r"""
 out.update(torch_npu=None, npu_available=False, npu_device_count=0, npu_error=None)
 try:
+    if out.get("torch_error"):
+        raise RuntimeError(out["torch_error"])
     import torch_npu
     import torch
     out["torch_npu"] = getattr(torch_npu, "__version__", "unknown")
@@ -36,8 +38,9 @@ try:
     out["torch"] = torch.__version__
     out["cuda"] = torch.version.cuda
     out["cuda_available"] = bool(torch.cuda.is_available())
-except Exception:
+except Exception as exc:
     out["torch"] = None
+    out["torch_error"] = type(exc).__name__ + ": " + str(exc)
     out["cuda"] = None
     out["cuda_available"] = False
 """ + NPU_PROBE + r"""
@@ -99,12 +102,14 @@ def get_git_commit(repo: Optional[str | Path]) -> Optional[str]:
 def environment_snapshot(python_executable: Optional[str] = None) -> dict:
     """Probe a python interpreter (subprocess) or the current one."""
     if python_executable:
+        from rexp.core.accelerators import ascend_runtime_environment
         try:
             out = subprocess.run(
                 [str(python_executable), "-c", ENV_PROBE],
                 capture_output=True,
                 text=True,
                 timeout=180,
+                env=ascend_runtime_environment(),
             )
             if out.returncode == 0:
                 lines = [ln for ln in out.stdout.strip().splitlines() if ln.strip()]
@@ -121,8 +126,9 @@ def environment_snapshot(python_executable: Optional[str] = None) -> dict:
         info["torch"] = torch.__version__
         info["cuda"] = torch.version.cuda
         info["cuda_available"] = bool(torch.cuda.is_available())
-    except Exception:
+    except Exception as exc:
         info["torch"] = None
+        info["torch_error"] = type(exc).__name__ + ": " + str(exc)
         info["cuda"] = None
         info["cuda_available"] = False
     exec(NPU_PROBE, {"out": info})

{rexprun-0.2.0 → rexprun-0.2.2}/rexp/core/state_store.py

@@ -130,6 +130,8 @@ class StateStore:
                 self._conn.execute("ALTER TABLE nodes ADD COLUMN gpu_devices TEXT")
             if "accelerator_backend" not in cols:
                 self._conn.execute("ALTER TABLE nodes ADD COLUMN accelerator_backend TEXT")
+            if "accelerator_probe_error" not in cols:
+                self._conn.execute("ALTER TABLE nodes ADD COLUMN accelerator_probe_error TEXT")
             run_cols = {r[1] for r in self._conn.execute("PRAGMA table_info(runs)").fetchall()}
             for column, definition in (("queue_order", "INTEGER"), ("tune_yield", "INTEGER")):
                 if column not in run_cols:

{rexprun-0.2.0 → rexprun-0.2.2}/rexp/docs/schemas.md

@@ -1,5 +1,7 @@
 # Schema 说明
 
+0.2.1：节点新增可空 `accelerator_probe_error`，SQLite 使用增量迁移。Tune counts/trials 新增 CANCELLED 展示状态；父任务终止且实际进程退出后，从旧快照投影终态，不改写 Ray checkpoint。CPU-only runtime 的 `accelerator_backend` 为 cpu。
+
 Handfree REXP 插件通过 SSH 调用被控端 `rexp platform info --json`。返回值中的
 `platform_root` 必须与该连接配置的绝对 `platformRoot` 一致；否则插件拒绝上传。
 `connectionName` 只由 Handfree 的 SSH 主机配置解析，不是平台事实 schema 的字段。

{rexprun-0.2.0 → rexprun-0.2.2}/rexp/docs//344/275/277/347/224/250/350/257/264/346/230/216/344/271/246.md

@@ -1,5 +1,15 @@
 # REXP 使用说明书（v0.1）
 
+## 0.2.1 部署修复
+
+独立环境安装后执行同根目录的 `rexp init`，自动登记全局命令：Linux root 使用 `/usr/local/bin/rexp`，普通用户使用 `~/.local/bin/rexp`；Windows 创建用户级 `rexp.cmd` 并登记 PATH。不会覆盖无关同名程序；需要加载新 PATH 时打开新终端。验收必须直接输入 `rexp version` 和 `rexp doctor --json`，无需激活实验环境。`.platform/bootstrap` 是 uv 引导环境，不是实验环境。
+
+非登录 SSH 的 Ascend 探测、实验解释器探测与 runner 自动补齐已存在的标准库路径，保留用户配置和可见设备。优先采用显式 `ASCEND_HOME_PATH` / `ASCEND_TOOLKIT_HOME`，再识别 CANN 9 的 `/usr/local/Ascend/cann` 或旧版 `ascend-toolkit/latest`，每次只选一套工具包。探测失败返回 `accelerator_probe_error`，不再无声显示为没有设备；Torch 导入失败保留原始原因，不重复导入产生误导错误。自定义驱动安装仍需自行配置兼容路径。
+
+取消任务后以 `management.process_alive=false` 确认退出，再更新结束时间与 Tune 的取消状态。停止调度器会处理已经完成但尚未自动评测的记录。日志按节点时区显示并标明时区名称；JSON 时间仍为 UTC。CPU-only Tune runtime 标为 cpu。
+
+SSH-MCP 安装清单可通过 `dependencyIndex` 选择依赖镜像，REXP wheel 本体始终取自正式 PyPI 并校验。安装支持 MCP progress；超时先检查目标实际状态，再显式重试。清单与已安装 skill 应同步更新。
+
 Research Experiment Platform：单节点、离线优先的个人科研实验平台。
 目标：Idea → Research Project → Experiment → Run → Artifact → Evaluation → Analysis → Conclusion，
 全程可恢复、可复现、可追溯，且不用手动管理一堆脚本和文件。
@@ -41,6 +51,10 @@ Windows 安装后的命令通常是 `<Conda环境>\Scripts\rexp.exe`。
 其他实验环境用 `rexp node env create NAME --python 3.11 --description "中文用途及限制"` 创建、登记在 `<根目录>/env/NAME`；uv/Conda 可按版本创建，venv 需要已安装的 Python 可执行文件（可用 `--python /path/to/python`）。优先复用已有兼容环境；云镜像的 TorchNPU/CANN 环境也可用 `node env add --python` 登记，不重装驱动。Conda prefix 环境与 uv/venv 一样由平台记录绝对解释器路径；既有 Conda 命名环境继续支持 `--conda-env` 登记。`env/platform` 保留给平台，不能登记为普通实验环境。
 
 环境设置失败保留数据根目录，修复原因后重跑相同 init。依赖下载失败但解释器已建好时可重试安装；解释器创建中断留下不完整目录时会提示先检查，不自动删除。首次初始化保存用户默认根目录；已有默认根目录时用 `--save-default` 显式切换。安装后分别检查插件内和不设置 PLATFORM_ROOT 的普通终端 `rexp doctor --json`，确认根目录、绑定管理器、平台解释器及 default 环境一致。
+
+Agent 首次询问管理器时应给出完整三项：安装 uv（推荐）、安装 Conda、使用现有 Python venv（不安装管理器）。推荐 uv 不会移除 Conda 选项。环境创建/安装显示当前步骤；pip/uv 默认每次网络请求超时 15 秒、重试一次，用户显式配置优先，整个初始化仍有独立总时限。
+
+离线安装先准备匹配远端 OS/架构/Python ABI 的完整 wheelhouse，然后设置 `UV_OFFLINE=true UV_FIND_LINKS=<目录> PIP_NO_INDEX=1 PIP_FIND_LINKS=<目录>` 执行相同 init。`uv venv --seed` 也需要种子包（所选 Python 所需的 pip/setuptools/wheel）；集合应另包含 default 的 NumPy 等依赖。仅设置 `UV_NO_INDEX=1` 不能让 uv 离线。已有 wheelhouse 时直接离线使用；失败后检查环境，不重复长时间联网尝试。
 `rexp doctor --json` 与 `rexp platform info --json` 的 `experiment_setup` 返回登记状态：
 `needs_environment` 表示没有实验环境，`registered` 只代表已登记，实际实验仍须检查项目依赖。
 至少登记一个可用实验环境；未登记名为 `default` 的环境时，在实验 YAML 中显式指定 `environment:`，

{rexprun-0.2.0 → rexprun-0.2.2}/rexp/server/environments.py

@@ -41,7 +41,21 @@ def select_manager(available, requested="auto", existing=None):
 
 
 def run(command, timeout=900):
-    result = subprocess.run([str(x) for x in command], capture_output=True, text=True, timeout=timeout)
+    # A failed index request should not consume pip/uv's long default retries.
+    # Explicit operator settings (including offline wheelhouses) still win.
+    child_env = os.environ.copy()
+    for key, value in {"PIP_DEFAULT_TIMEOUT": "15", "PIP_RETRIES": "1",
+                       "PIP_DISABLE_PIP_VERSION_CHECK": "1",
+                       "UV_HTTP_TIMEOUT": "15", "UV_HTTP_RETRIES": "1"}.items():
+        child_env.setdefault(key, value)
+    argv = [str(x) for x in command]
+    print(f"Environment step: {Path(argv[0]).name} {' '.join(argv[1:3])}", flush=True)
+    try:
+        result = subprocess.run(argv, capture_output=True, text=True, timeout=timeout, env=child_env)
+    except subprocess.TimeoutExpired as exc:
+        raise RuntimeError(f"Environment step timed out after {timeout}s. Inspect the existing environment before retrying; "
+                           "for a prepared wheelhouse use UV_OFFLINE=true UV_FIND_LINKS=<directory> "
+                           "and PIP_NO_INDEX=1 PIP_FIND_LINKS=<directory>.") from exc
     if result.returncode:
         raise RuntimeError(f"Environment command failed: {(result.stderr or result.stdout)[-1500:]}")
     return result.stdout
@@ -178,3 +192,57 @@ def configured_runtime(args):
         return None
     os.environ["REXP_RUNTIME_FORWARDED"] = "1"
     return target
+
+
+def register_command(runtime_python, bin_directory=None, *, update_path=True):
+    """Expose the configured runtime outside its virtual environment.
+
+    Never replace an unrelated command. User installs use ~/.local/bin; root
+    installs use /usr/local/bin. A Windows .cmd launcher uses the user PATH.
+    """
+    import shlex
+    if bin_directory is None:
+        if os.name == 'nt':
+            bin_directory = Path(os.environ.get('LOCALAPPDATA', Path.home() / 'AppData/Local')) / 'rexprun/bin'
+        else:
+            bin_directory = Path('/usr/local/bin') if hasattr(os, 'geteuid') and os.geteuid() == 0 else Path.home() / '.local/bin'
+    directory = Path(bin_directory)
+    directory.mkdir(parents=True, exist_ok=True)
+    command = directory / ('rexp.cmd' if os.name == 'nt' else 'rexp')
+    marker = 'Managed by rexprun'
+    text = (f'@echo off\nrem {marker}\n"{runtime_python}" -m rexp %*\n' if os.name == 'nt' else
+            f'#!/bin/sh\n# {marker}\nexec {shlex.quote(str(runtime_python))} -m rexp "$@"\n')
+    if command.exists() or command.is_symlink():
+        try:
+            existing = command.read_text(encoding='utf-8')
+        except (OSError, UnicodeError):
+            existing = ''
+        if marker not in existing:
+            # An already globally installed console entry is valid, too.
+            if 'rexp.cli.main' in existing:
+                return {'path': str(command), 'created': False}
+            raise ValueError(f'Command {command} already exists and is not managed by rexprun; it was not overwritten.')
+    command.write_text(text, encoding='utf-8')
+    if os.name != 'nt':
+        command.chmod(0o755)
+    in_path = str(directory) in os.environ.get('PATH', '').split(os.pathsep)
+    if update_path and not in_path:
+        if os.name == 'nt':
+            import winreg
+            with winreg.CreateKey(winreg.HKEY_CURRENT_USER, 'Environment') as key:
+                try:
+                    value, _ = winreg.QueryValueEx(key, 'Path')
+                except FileNotFoundError:
+                    value = ''
+                if str(directory).lower() not in value.lower().split(';'):
+                    winreg.SetValueEx(key, 'Path', 0, winreg.REG_EXPAND_SZ, value.rstrip(';') + ';' + str(directory))
+        else:
+            line = f'\n# {marker}\nexport PATH={shlex.quote(str(directory))}:"$PATH"\n'
+            profiles = ['.zprofile', '.zshrc'] if 'zsh' in os.environ.get('SHELL', '') else ['.profile', '.bashrc']
+            for name in profiles:
+                profile = Path.home() / name
+                current = profile.read_text() if profile.exists() else ''
+                if line.strip() not in current:
+                    with profile.open('a') as output:
+                        output.write(line)
+    return {'path': str(command), 'created': True, 'new_terminal_required': not in_path}

{rexprun-0.2.0 → rexprun-0.2.2}/rexp/server/node.py

@@ -12,12 +12,12 @@ import re
 import socket
 import subprocess
 
-from rexp.core.accelerators import ascend_visible_ids
+from rexp.core.accelerators import ascend_visible_ids, ascend_runtime_environment
 
 
 HARDWARE_FIELDS = (
     "hostname", "os", "cpu_cores", "ram_gb", "gpu_name", "gpu_count",
-    "gpu_vram_mb", "gpu_driver", "gpu_devices", "accelerator_backend",
+    "gpu_vram_mb", "gpu_driver", "gpu_devices", "accelerator_backend", "accelerator_probe_error",
 )
 
 
@@ -33,7 +33,8 @@ def detect_local_node(platform_version: str = "") -> dict:
     hostname = socket.gethostname()
     cpu_cores = _available_cpu_cores()
     ram_gb = _available_ram_gb()
-    gpu_name, gpu_count, gpu_vram_mb, driver, devices = _detect_gpus()
+    diagnostics = {}
+    gpu_name, gpu_count, gpu_vram_mb, driver, devices = _detect_gpus(diagnostics)
     return {
         "hostname": hostname,
         "os": _platform.platform(),
@@ -45,6 +46,7 @@ def detect_local_node(platform_version: str = "") -> dict:
         "gpu_driver": driver,
         "gpu_devices": json.dumps(devices),
         "accelerator_backend": devices[0].get("backend", "cuda") if devices else None,
+        "accelerator_probe_error": diagnostics.get("error"),
         # Keep legacy DB columns cleared; CUDA/PyTorch belong to experiment envs.
         "cuda": None,
         "torch": None,
@@ -156,9 +158,9 @@ def _total_ram_gb() -> float:
     return 0.0
 
 
-def _detect_gpus() -> tuple[str, int, int, str, list[dict]]:
+def _detect_gpus(diagnostics=None) -> tuple[str, int, int, str, list[dict]]:
     detected = _detect_nvidia_gpus()
-    return detected if detected[1] else _detect_npus()
+    return detected if detected[1] else _detect_npus(diagnostics)
 
 
 def _detect_nvidia_gpus() -> tuple[str, int, int, str, list[dict]]:
@@ -191,16 +193,19 @@ def _detect_nvidia_gpus() -> tuple[str, int, int, str, list[dict]]:
         return "", 0, 0, "", []
 
 
-def _detect_npus() -> tuple[str, int, int, str, list[dict]]:
+def _detect_npus(diagnostics=None) -> tuple[str, int, int, str, list[dict]]:
     """Read Ascend board/chip/runtime logical mappings without importing torch.
 
     First delivery targets homogeneous, unpartitioned Ascend Linux nodes.
     Unknown topology is not guessed: no fabricated contiguous device IDs.
     """
+    diagnostics = diagnostics if diagnostics is not None else {}
+    runtime_env = ascend_runtime_environment()
     try:
         result = subprocess.run(["npu-smi", "info", "-m"], capture_output=True,
-                                text=True, timeout=15)
+                                text=True, timeout=15, env=runtime_env)
         if result.returncode != 0:
+            diagnostics["error"] = "npu-smi failed: " + (getattr(result, "stderr", "") or result.stdout)[-1000:]
             return "", 0, 0, "", []
         devices = []
         seen = set()
@@ -221,7 +226,7 @@ def _detect_npus() -> tuple[str, int, int, str, list[dict]]:
             try:
                 memory = subprocess.run(
                     ["npu-smi", "info", "-t", "memory", "-i", str(card), "-c", str(chip)],
-                    capture_output=True, text=True, timeout=10)
+                    capture_output=True, text=True, timeout=10, env=runtime_env)
                 if memory.returncode == 0:
                     capacity = re.search(r"(?:HBM|Memory)\s+Capacity\s*\(\s*MB\s*\)\s*:\s*(\d+)",
                                          memory.stdout, re.IGNORECASE)
@@ -232,8 +237,13 @@ def _detect_npus() -> tuple[str, int, int, str, list[dict]]:
             devices.append({"index": logical, "uuid": None, "name": " ".join(fields[name_start:]),
                             "memory_mb": memory_mb, "backend": "npu",
                             "card_id": card, "chip_id": chip})
+        if not devices and allowed is None:
+            diagnostics["error"] = "npu-smi returned no recognized logical devices; inspect npu-smi info -m"
         devices.sort(key=lambda device: device["index"])
         return (", ".join(d["name"] for d in devices), len(devices),
                 max((d["memory_mb"] for d in devices), default=0), "", devices)
-    except (OSError, subprocess.TimeoutExpired):
+    except FileNotFoundError:
+        return "", 0, 0, "", []
+    except (OSError, subprocess.TimeoutExpired) as exc:
+        diagnostics["error"] = f"npu-smi probe failed: {exc}"
         return "", 0, 0, "", []

{rexprun-0.2.0 → rexprun-0.2.2}/rexp/server/runner.py

@@ -10,6 +10,8 @@ import argparse
 import importlib.util
 import json
 import logging
+from datetime import datetime
+from zoneinfo import ZoneInfo
 import sys
 import threading
 import traceback
@@ -46,7 +48,12 @@ class ExperimentContext:
         logger = logging.getLogger(f"rexp.run.{self.run_id}")
         logger.setLevel(logging.INFO)
         if not logger.handlers:
-            fmt = logging.Formatter("%(asctime)s %(levelname)s %(message)s")
+            from rexp.core.config import load_nodes_config
+            tz = ZoneInfo(load_nodes_config(self.paths).schedule.timezone)
+            class LocalFormatter(logging.Formatter):
+                def formatTime(self, record, datefmt=None):
+                    return datetime.fromtimestamp(record.created, tz).strftime("%Y-%m-%d %H:%M:%S %Z")
+            fmt = LocalFormatter("%(asctime)s %(levelname)s %(message)s")
             fh = logging.FileHandler(self.run_dir / "logs" / "experiment.log", encoding="utf-8")
             fh.setFormatter(fmt)
             logger.addHandler(fh)
@@ -146,6 +153,7 @@ def main(argv: Optional[list[str]] = None) -> int:
             experiment.setup(ctx)
             experiment.run(ctx)
         if (store.get_run(args.run_id) or {}).get("status") == ExperimentStatus.CANCELLED.value:
+            store.update_run(args.run_id, finished_at=now_iso(), exit_code=0)
             ctx.logger.info("Run cancelled before finalization")
             return 0
         if hasattr(experiment, "finalize"):
@@ -173,6 +181,7 @@ def main(argv: Optional[list[str]] = None) -> int:
     except Exception as exc:  # noqa: BLE001
         traceback.print_exc(file=sys.stderr)
         if (store.get_run(args.run_id) or {}).get("status") == ExperimentStatus.CANCELLED.value:
+            store.update_run(args.run_id, finished_at=now_iso(), exit_code=0)
             ctx.logger.info("Run cancelled during shutdown: %s", exc)
             return 0
         store.update_run(

{rexprun-0.2.0 → rexprun-0.2.2}/rexp/server/scheduler.py

@@ -213,6 +213,8 @@ class Scheduler:
                 except Exception:
                     pass
             self._procs.clear()
+            self._release_finished_allocations()
+            self._maybe_evaluate_all(load_platform_config(self.paths))
             if (self.store.get_platform_status() or {}).get("scheduler_pid") == os.getpid():
                 self.store.set_platform_status(PlatformStatus.OFF.value, stopped_at=now_iso())
 
@@ -385,6 +387,8 @@ class Scheduler:
             if not alive:
                 self.store.update_trial_lease(lease['lease_id'], status='RELEASED')
         for run in self.store.list_runs():
+            if run["status"] == "CANCELLED" and not is_process_group_alive(run.get("pid")) and not run.get("finished_at"):
+                self.store.update_run(run["run_id"], finished_at=now_iso())
             if run.get("allocated_gpu_ids") is not None and run.get("pid"):
                 if not is_process_group_alive(run["pid"]):
                     self.store.update_run(run["run_id"], pid=None)
@@ -542,7 +546,8 @@ class Scheduler:
                    "--run-id", run["run_id"]]
             if run.get("resume_requested"):
                 cmd.append("--resume")
-            env = dict(os.environ)
+            from rexp.core.accelerators import ascend_runtime_environment
+            env = ascend_runtime_environment()
             env["PYTHONPATH"] = str(import_root) + os.pathsep + env.get("PYTHONPATH", "")
             env["REXP_PLATFORM_ROOT"] = str(self.paths.root)
             env = (search_controller_environment(node_info, env) if exp_config.tune

{rexprun-0.2.0 → rexprun-0.2.2}/rexp/server/services.py

@@ -262,7 +262,11 @@ def cancel_run(paths: Paths, run_id: str) -> dict:
         pause_file.write_text(now_iso(), encoding="utf-8")
         result = store.get_run(run_id)
         from rexp.core.process import is_process_group_alive
-        result["cancellation"] = {"requested": True, "process_alive": is_process_group_alive(run.get("pid"))}
+        alive = is_process_group_alive(run.get("pid"))
+        if not alive:
+            store.update_run(run_id, finished_at=now_iso())
+            result = store.get_run(run_id)
+        result["cancellation"] = {"requested": True, "process_alive": alive}
         return result
     finally:
         store.close()

{rexprun-0.2.0 → rexprun-0.2.2}/rexp/server/tuning.py

@@ -93,6 +93,11 @@ def search_details(paths, run_id, *, include_trials=True):
     enabled = bool(run['tune_yield']) if run.get('tune_yield') is not None else data['search'].get('yield_to_normal', False)
     data['search']['yield_to_normal'] = enabled
     data['trial_priority'] = int(enabled)
+    from rexp.core.process import is_process_group_alive, is_process_alive
+    parent_exited = not is_process_group_alive(run.get("pid")) and not any(is_process_alive(l["pid"]) or is_process_group_alive(l["pid"]) for l in leases)
+    parent_terminal = run["status"] in {"COMPLETED", "FAILED", "CANCELLED"}
+    inactive_status = "CANCELLED" if run["status"] == "CANCELLED" else "ERROR" if run["status"] in {"COMPLETED", "FAILED"} else "PAUSED"
+    stopped = parent_exited and (parent_terminal or run["status"] in {"PAUSED", "RECOVERABLE"})
     active_devices = set()
     if include_trials:
         planned_samples = ExperimentConfig.from_yaml_text(run['config_snapshot']).total_samples
@@ -104,17 +109,25 @@ def search_details(paths, run_id, *, include_trials=True):
             if lease and row['status'] not in {'TERMINATED', 'ERROR', 'PAUSED'}:
                 row['status'] = 'PENDING' if lease['status'] == 'REQUESTED' else 'RUNNING'
                 row['wait_reason'] = lease['wait_reason']
-            if lease and lease['status'] == 'GRANTED':
+            if stopped and row["status"] not in {"TERMINATED", "ERROR", "CANCELLED"} and not (lease and is_process_alive(lease["pid"])):
+                row["status"] = inactive_status
+                row["finished_at"] = run.get("finished_at") or row.get("finished_at")
+                row["wait_reason"] = run.get("wait_reason")
+            if lease and lease['status'] == 'GRANTED' and not stopped:
                 row['allocated_gpu_ids'] = json.loads(lease['allocated_gpu_ids'] or '[]')
         data['trials'] = sorted_trials(data['trials'], data['search']['metrics'], data['pareto_trial_ids'])
         data['counts'] = {status: sum(r['status'] == status for r in data['trials'])
-                          for status in ('PENDING', 'RUNNING', 'PAUSED', 'TERMINATED', 'ERROR')}
+                          for status in ('PENDING', 'RUNNING', 'PAUSED', 'TERMINATED', 'ERROR', 'CANCELLED')}
     else:
-        waiting = sum(l['status'] == 'REQUESTED' for l in leases)
+        if stopped:
+            remaining = sum(data["counts"].pop(k, 0) for k in ("RUNNING", "PENDING"))
+            data["counts"]["RUNNING"] = data["counts"]["PENDING"] = 0
+            data["counts"][inactive_status] = data["counts"].get(inactive_status, 0) + remaining
+        waiting = 0 if stopped else sum(l['status'] == 'REQUESTED' for l in leases)
         data['counts']['RUNNING'] = max(0, data['counts'].get('RUNNING', 0) - waiting)
         data['counts']['PENDING'] = data['counts'].get('PENDING', 0) + waiting
     for lease in leases:
-        if lease['status'] in {'GRANTED', 'RELEASING'}:
+        if lease['status'] in {'GRANTED', 'RELEASING'} and not stopped:
             active_devices.update(json.loads(lease['allocated_gpu_ids'] or '[]'))
     return {**data, "status": run["status"], "project": run["project"], "version": run["version"],
             "allocated_gpu_ids": sorted(active_devices),
@@ -248,13 +261,13 @@ def _trial_entry(parameters, *, entry_file, run_dir, experiment, root, run_id, c
         artifacts.mkdir(parents=True, exist_ok=True)
         backend = os.environ.get('REXP_ACCELERATOR_BACKEND', 'cuda')
         visible = os.environ.get('ASCEND_RT_VISIBLE_DEVICES' if backend == 'npu' else 'CUDA_VISIBLE_DEVICES', '')
-        write_json(artifacts / 'runtime.json', {'accelerator_backend': backend, 'visible_devices': visible, 'allocated_gpu_ids': devices, 'pid': os.getpid(), 'execution_started_at': now_iso()})
+        write_json(artifacts / 'runtime.json', {'accelerator_backend': backend if devices else 'cpu', 'visible_devices': visible, 'allocated_gpu_ids': devices, 'pid': os.getpid(), 'execution_started_at': now_iso()})
         spec = importlib.util.spec_from_file_location('rexp_tune_trainable', entry_file)
         module = importlib.util.module_from_spec(spec)
         sys.path.insert(0, str(Path(entry_file).parent))
         sys.path.insert(0, str(Path(entry_file).parent.parent))
         spec.loader.exec_module(module)
-        config = {**parameters, '_rexp': {'artifact_dir': str(artifacts), 'trial_id': trial_id, 'experiment': experiment, 'accelerator_backend': backend, 'allocated_device_ids': devices}}
+        config = {**parameters, '_rexp': {'artifact_dir': str(artifacts), 'trial_id': trial_id, 'experiment': experiment, 'accelerator_backend': backend if devices else 'cpu', 'allocated_device_ids': devices}}
         result = module.trainable(config)
         if result is not None:
             tune.report(result)
@@ -378,7 +391,7 @@ class TuneExperiment:
                                            sampler=optuna.samplers.TPESampler(seed=self.config.seed))
                 else:
                     from ray.tune.search.basic_variant import BasicVariantGenerator
-                    searcher = BasicVariantGenerator(random_state=self.config.seed)
+                    searcher = BasicVariantGenerator(random_state=self.config.seed, max_concurrent=cfg.max_concurrent_trials)
                 scheduler = None
                 if cfg.scheduler == "asha":
                     from ray.tune.schedulers import ASHAScheduler
@@ -387,7 +400,7 @@ class TuneExperiment:
                 tuner = tune.Tuner(trainable, param_space=space,
                     tune_config=tune.TuneConfig(metric=keys[0] if len(keys) == 1 else None,
                         mode=modes[0] if len(keys) == 1 else None, search_alg=searcher, scheduler=scheduler,
-                        num_samples=cfg.num_samples, max_concurrent_trials=cfg.max_concurrent_trials, reuse_actors=False,
+                        num_samples=cfg.num_samples, max_concurrent_trials=cfg.max_concurrent_trials if cfg.search_algorithm == "optuna" else None, reuse_actors=False,
                         time_budget_s=cfg.time_budget_seconds),
                     run_config=tune.RunConfig(name="search", storage_path=str(storage), callbacks=[monitor], verbose=0,
                         failure_config=tune.FailureConfig(max_failures=cfg.max_failures),

{rexprun-0.2.0 → rexprun-0.2.2}/rexp/skills/experiment-authoring/SKILL.md

@@ -5,6 +5,8 @@ description: 为 REXP 设计实验、样本进度、适度频率的断点与评
 
 # Experiment Authoring Skill
 
+0.2.1 补充：在实验解释器内验证 TorchNPU，不给平台解释器安装科研框架。`doctor` 的 `accelerator_probe_error` 表示探测失败，不能当成没有设备。自定义 CANN 安装仍须准备兼容环境。Ray CPU 模板只在结束时保存计算 checkpoint；真实实验按代价选择保存间隔。真卡数值短测与持续负载展示用途不同，瞬时 0% 不能证明未使用 NPU。模板中的 `<PLATFORM_ROOT>` 必须替换成查询得到的绝对根目录，不依赖 SSH 当前目录。
+
 本 Skill 供科研 AI Agent 使用：把用户的科研 Idea 变成规范的 Research Project、
 Experiment、Evaluation、Analysis 与 Config，并确保平台规定的目录与资源纪律。
 
@@ -150,7 +152,7 @@ PAUSED/RECOVERABLE 等可续跑任务若确需放弃，应先明确取消，不
 以下每行是独立命令示例，尖括号中的值由实际项目替换：
 
 ```text
-rexp project create <project> --version v1 --domain dlm
+rexp project create <project> --version v1
 rexp submit <project> v1 --config <PLATFORM_ROOT>/research/<project>/v1/configs/smoke.yaml
 rexp start
 rexp status