acprof 0.3.0__py3-none-any.whl

acprof/_bundle/acprof/host/env_utils.py

@@ -0,0 +1,233 @@
+"""Small helpers for local project environment bootstrap."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import stat
+import tempfile
+from collections.abc import Iterable, Mapping, MutableMapping
+from pathlib import Path
+
+from acprof.config import (
+    CONTAINER_HF_HOME,
+    CONTAINER_MODEL_LOCAL_PATH,
+)
+from acprof.hf_endpoints import hf_endpoints
+
+CONFIGURABLE_ENV_KEYS = (
+    "HF_TOKEN", "HF_ENDPOINT", "HF_FALLBACK_ENDPOINTS", "HF_DOWNLOAD_MODE",
+    "ACPROF_MAX_DOWNLOAD", "ACPROF_MODEL_MAX_DOWNLOAD",
+    "ACPROF_MODEL_STORE", "ACPROF_MODEL_STORE_MAX",
+    "ACPROF_WECOM_WEBHOOK_URL",
+)
+_ENV_ALIASES = {
+    "HF_TOKEN": "HUGGING_FACE_HUB_TOKEN", "HF_ENDPOINT": "HF_HUB_ENDPOINT",
+}
+_RETIRED_PROXY_KEYS = frozenset({"http_proxy", "https_proxy", "all_proxy", "no_proxy"})
+
+
+def _parse_env_line(raw_line: str) -> tuple[str, str] | None:
+    line = raw_line.strip()
+    if line.startswith("export "):
+        line = line[7:].lstrip()
+    if not line or line.startswith("#") or "=" not in line:
+        return None
+    key, value = line.split("=", 1)
+    key, value = key.strip(), value.strip()
+    if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key):
+        return None
+    if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
+        quote, value = value[0], value[1:-1]
+        if quote == '"':
+            value = re.sub(r'\\([\\"])', r'\1', value)
+    return key, value
+
+
+def _read_env_values(path: Path) -> dict[str, str]:
+    if not path.exists():
+        return {}
+    return dict(item for line in path.read_text(encoding="utf-8").splitlines()
+                if (item := _parse_env_line(line)) is not None)
+
+
+def project_env_values(project_dir: str | os.PathLike[str]) -> dict[str, str]:
+    """Read local values without changing the process; .env.local has priority."""
+    values: dict[str, str] = {}
+    for path in _iter_env_files(project_dir):
+        values.update(_read_env_values(path))
+    return values
+
+
+def configurable_env_values(project_dir, *, environ=None) -> dict[str, str]:
+    values = project_env_values(project_dir)
+    values.update(os.environ if environ is None else environ)
+    return {key: values.get(key, values.get(_ENV_ALIASES.get(key, ""), ""))
+            for key in CONFIGURABLE_ENV_KEYS}
+
+
+def _private_env_text(path: Path) -> str:
+    if path.is_symlink() or (path.exists() and not path.is_file()):
+        raise ValueError("Local configuration must be a regular file, not a symbolic link")
+    return path.read_text(encoding="utf-8") if path.exists() else ""
+
+
+def _atomic_private_text(path: Path, text: str) -> None:
+    _private_env_text(path)
+    fd, name = tempfile.mkstemp(prefix=".env.local.", suffix=".tmp", dir=path.parent)
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as stream:
+            os.fchmod(stream.fileno(), stat.S_IRUSR | stat.S_IWUSR)
+            stream.write(text)
+            stream.flush()
+            os.fsync(stream.fileno())
+        os.replace(name, path)
+        if os.name == "posix":
+            directory_fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY)
+            try:
+                os.fsync(directory_fd)
+            finally:
+                os.close(directory_fd)
+    finally:
+        if os.path.exists(name):
+            os.unlink(name)
+
+
+def save_project_env(
+    project_dir: str | os.PathLike[str], values: Mapping[str, str], *,
+    environ: MutableMapping[str, str] | None = None, expected_text: str | None = None,
+) -> Path:
+    """Save explicit TUI edits privately, preserving other settings and a backup.
+
+    No shell evaluation or variable interpolation is performed. Only an explicit
+    save updates the current process and its children; ordinary bootstrap still
+    gives inherited environment variables priority.
+    """
+    if set(values) - set(CONFIGURABLE_ENV_KEYS):
+        raise ValueError("Unsupported environment setting")
+    updates = {}
+    for key, value in values.items():
+        if not isinstance(value, str) or any(ord(char) < 32 or ord(char) == 127 for char in value):
+            raise ValueError(f"{key}: enter a single line without control characters")
+        updates[key] = value.strip()
+    hf_endpoints({**os.environ, **updates})
+    from acprof.network_policy import parse_bytes
+    for key in ("ACPROF_MAX_DOWNLOAD", "ACPROF_MODEL_MAX_DOWNLOAD", "ACPROF_MODEL_STORE_MAX"):
+        parse_bytes(updates.get(key))
+    if updates.get("ACPROF_WECOM_WEBHOOK_URL"):
+        from acprof.notifications import NotificationConfigError, validate_wecom_webhook_url
+        try:
+            validate_wecom_webhook_url(updates["ACPROF_WECOM_WEBHOOK_URL"])
+        except (ValueError, NotificationConfigError):
+            raise ValueError("ACPROF_WECOM_WEBHOOK_URL: invalid WeCom webhook URL") from None
+    for key, alias in _ENV_ALIASES.items():
+        if key in updates:
+            updates[alias] = updates[key]
+    path = Path(project_dir) / ".env.local"
+    original = _private_env_text(path)
+    if expected_text is not None and original != expected_text:
+        raise ValueError(".env.local changed since this form opened; close and reopen before saving")
+    remaining = dict(updates)
+    lines = []
+    for line in original.splitlines(keepends=True):
+        item = _parse_env_line(line)
+        if item is not None and item[0] in updates:
+            key = item[0]
+            if key in remaining:
+                lines.append(f"{key}={json.dumps(remaining.pop(key), ensure_ascii=False)}\n")
+        else:
+            lines.append(line)
+    if lines and not lines[-1].endswith("\n"):
+        lines[-1] += "\n"
+    lines.extend(f"{key}={json.dumps(value, ensure_ascii=False)}\n" for key, value in remaining.items())
+    if path.exists():
+        _atomic_private_text(path.with_name(".env.local.bak"), original)
+    _atomic_private_text(path, "".join(lines))
+    (os.environ if environ is None else environ).update(updates)
+    return path
+
+
+def _iter_env_files(project_dir: str | os.PathLike[str]) -> Iterable[Path]:
+    root = Path(project_dir)
+    yield root / ".env"
+    yield root / ".env.local"
+
+
+def load_project_env(
+    project_dir: str | os.PathLike[str],
+    *,
+    environ: MutableMapping[str, str] | None = None,
+) -> None:
+    """Load local KEY=VALUE pairs into the process or an isolated environment."""
+    target_environ = os.environ if environ is None else environ
+    retired_setting = "ACPROF_SUDO_PASSWORD"
+    migration = f"{retired_setting} is no longer supported; remove it and follow docs/getting_started/installation.md#最小权限安装"
+    if retired_setting in target_environ:
+        raise ValueError(migration)
+    values = project_env_values(project_dir)
+    if retired_setting in values:
+        raise ValueError(migration)
+    for key, value in values.items():
+        # Historical TUI proxy fields remain untouched on disk. Only the user's
+        # inherited system environment configures HTTP clients now.
+        if key.lower() not in _RETIRED_PROXY_KEYS:
+            target_environ.setdefault(key, value)
+
+
+def _set_default_if_blank(key: str, value: str) -> None:
+    if not os.environ.get(key, "").strip():
+        os.environ[key] = value
+
+
+def configure_hf_network() -> str:
+    """Normalize the selected endpoint while preserving the user's proxy policy."""
+    endpoint = hf_endpoints()[0]
+    os.environ["HF_ENDPOINT"] = endpoint
+    _set_default_if_blank("HF_HUB_ENDPOINT", endpoint)
+    from acprof.hf_transport import configure_hf_transport
+    configure_hf_transport()
+    return endpoint
+
+
+def resolve_hf_token() -> str | None:
+    """Populate HF_TOKEN from env or local Hugging Face login when available."""
+    token = (
+        os.environ.get("HF_TOKEN", "").strip()
+        or os.environ.get("HUGGING_FACE_HUB_TOKEN", "").strip()
+    )
+    if not token:
+        try:
+            from huggingface_hub.utils import get_token
+
+            token = (get_token() or "").strip()
+        except Exception:
+            token = ""
+
+    if not token:
+        return None
+
+    _set_default_if_blank("HF_TOKEN", token)
+    _set_default_if_blank("HUGGING_FACE_HUB_TOKEN", token)
+    return token
+
+
+def bootstrap_project_env(project_dir: str | os.PathLike[str]) -> str | None:
+    """Load project env files and normalize Hugging Face auth env vars."""
+    load_project_env(project_dir)
+    configure_hf_network()
+    return resolve_hf_token()
+
+
+def hf_offline_docker_env_args() -> list[str]:
+    """Return the shared Docker environment for network-free model loading."""
+    return [
+        "-e", "HF_HUB_DISABLE_TELEMETRY=1",
+        "-e", "HF_HUB_OFFLINE=1",
+        "-e", "HF_HUB_DISABLE_XET=1",
+        "-e", "TRANSFORMERS_OFFLINE=1",
+        "-e", f"HF_HOME={CONTAINER_HF_HOME}",
+        "-e", f"HF_HUB_CACHE={CONTAINER_HF_HOME}",
+        "-e", f"TRANSFORMERS_CACHE={CONTAINER_HF_HOME}",
+        "-e", f"MODEL_LOCAL_PATH={CONTAINER_MODEL_LOCAL_PATH}",
+    ]

acprof/_bundle/acprof/host/execution_conditions.py

@@ -0,0 +1,87 @@
+"""Restore recorded execution conditions outside measurement windows."""
+from __future__ import annotations
+
+import math
+import os
+from collections.abc import Mapping
+from contextlib import contextmanager
+from dataclasses import dataclass
+
+from acprof.cpu_affinity import normalize_cpu_set, parse_cpu_set
+from acprof.host.gpu_device import gpu_device_scope, pin_gpu_device
+from acprof.runtime_settings import RUNTIME_ENV_NAMES
+
+MEASUREMENT_ENV_NAMES = (
+    "AUTO_WARMUP_REQUESTS", "SLOW_LATENCY_THRESHOLD_S", "IDLE_DEBUG_TRACE_INTERVAL_S",
+    "DEVICE_INDEX", "CUDA_VISIBLE_DEVICES", "OMP_NUM_THREADS", "MKL_NUM_THREADS",
+)
+DEVICE_ENV_NAMES = ("ACPROF_GPU_DEVICE", "GPU_DEVICE_UUID", "NVIDIA_VISIBLE_DEVICES")
+
+
+def measurement_environment() -> dict:
+    return {**{name: os.environ.get(name) for name in MEASUREMENT_ENV_NAMES},
+            **{name: os.environ[name] for name in RUNTIME_ENV_NAMES if name in os.environ}}
+
+
+@contextmanager
+def source_runtime_environment(recorded: Mapping):
+    """Clear unrecorded overrides and restore the caller's environment on every exit."""
+    names = (*MEASUREMENT_ENV_NAMES, *RUNTIME_ENV_NAMES, *DEVICE_ENV_NAMES)
+    previous = {name: os.environ.get(name) for name in names}
+    try:
+        for name in names:
+            value = recorded.get(name)
+            if value is None:
+                os.environ.pop(name, None)
+            else:
+                os.environ[name] = str(value)
+        yield
+    finally:
+        for name, value in previous.items():
+            if value is None:
+                os.environ.pop(name, None)
+            else:
+                os.environ[name] = value
+
+
+@dataclass(frozen=True)
+class ExecutionConditions:
+    gpu_mode: str
+    gpu_uuid: str
+    cpuset_cpus: str
+    request_timeout_seconds: float
+    environment: dict
+
+    @classmethod
+    def from_options(cls, options: Mapping, *, gpu: str) -> ExecutionConditions:
+        if gpu not in {"off", "on"} or gpu not in options.get("gpus", "").split(","):
+            raise ValueError("source does not include the requested GPU mode")
+        environment = options.get("measurement_environment")
+        if not isinstance(environment, dict):
+            raise ValueError("source measurement_environment is missing")
+        uuid = str(options.get("gpu_device", "")) if gpu == "on" else ""
+        if gpu == "on" and not uuid.startswith("GPU-"):
+            raise ValueError("source GPU UUID is missing; refusing index-based replay")
+        timeout = float(options["request_timeout_seconds"])
+        if not math.isfinite(timeout) or timeout <= 0:
+            raise ValueError("source request timeout must be finite and positive")
+        return cls(gpu, uuid, normalize_cpu_set(options.get("cpuset_cpus", "")), timeout, dict(environment))
+
+    @property
+    def container_options(self) -> dict:
+        return {"cpuset_cpus": self.cpuset_cpus, "request_timeout_seconds": self.request_timeout_seconds}
+
+    @contextmanager
+    def activate(self):
+        with source_runtime_environment(self.environment), gpu_device_scope():
+            requested = parse_cpu_set(self.cpuset_cpus)
+            if requested and not requested <= os.sched_getaffinity(0):
+                raise ValueError("source CPU affinity is unavailable on this host")
+            device = {}
+            if self.gpu_mode == "on":
+                device = pin_gpu_device(self.gpu_uuid)
+                if device["uuid"] != self.gpu_uuid:
+                    raise ValueError("resolved GPU UUID differs from the source")
+                os.environ.update(ACPROF_GPU_DEVICE=self.gpu_uuid, GPU_DEVICE_UUID=self.gpu_uuid,
+                                  DEVICE_INDEX=str(device["index"]))
+            yield device

acprof/_bundle/acprof/host/gpu_device.py

@@ -0,0 +1,66 @@
+"""在测量窗口外解析物理 GPU；容器与主机采集器共享同一 UUID。"""
+from __future__ import annotations
+
+import csv
+import os
+from contextlib import contextmanager
+from contextvars import ContextVar
+
+from acprof.host.command import run_command
+
+_SELECTED: ContextVar[dict | None] = ContextVar("acprof_gpu_device", default=None)
+
+
+@contextmanager
+def gpu_device_scope():
+    """Limit a pinned selection to one CLI invocation, including embedded callers."""
+    token = _SELECTED.set(None)
+    try:
+        yield
+    finally:
+        _SELECTED.reset(token)
+
+
+def selected_gpu_device() -> dict:
+    return dict(_SELECTED.get() or {})
+
+
+def resolve_gpu_device(selector: str | None = None) -> dict:
+    if selector is None and _SELECTED.get() is not None:
+        return selected_gpu_device()
+    selector = str(selector if selector is not None else
+                   os.environ.get("ACPROF_GPU_DEVICE", os.environ.get("DEVICE_INDEX", "0"))).strip()
+    if not (selector.isdecimal() or selector.startswith("GPU-")) or any(c in selector for c in ",\n\r "):
+        raise ValueError("GPU selector must be one physical GPU index or UUID; all and MIG are unsupported")
+    result = run_command(
+        ["nvidia-smi", f"--id={selector}",
+         "--query-gpu=uuid,index,pci.bus_id,name,memory.total,mig.mode.current",
+         "--format=csv,noheader,nounits"],
+        capture_output=True, text=True, check=False, timeout=10,
+    )
+    if result.returncode:
+        raise RuntimeError(f"Cannot resolve GPU {selector}: {result.stderr.strip()}")
+    rows = list(csv.reader(result.stdout.strip().splitlines(), skipinitialspace=True))
+    if len(rows) != 1 or len(rows[0]) != 6:
+        raise RuntimeError(f"Expected exactly one physical GPU for {selector}")
+    uuid, index, bus, name, memory_mib, mig = (value.strip() for value in rows[0])
+    if not uuid.startswith("GPU-") or mig.lower() == "enabled":
+        raise RuntimeError("MIG devices cannot be attributed using physical-device NVML energy")
+    return {"uuid": uuid, "index": int(index), "pci_bus_id": bus, "name": name,
+            "memory_total_bytes": int(float(memory_mib) * 1024 ** 2)}
+
+
+def pin_gpu_device(selector: str | None = None) -> dict:
+    device = resolve_gpu_device(selector)
+    _SELECTED.set(device)
+    return dict(device)
+
+
+def gpu_docker_args(device: dict | None = None) -> list[str]:
+    device = resolve_gpu_device() if device is None else device
+    return ["--gpus", f"device={device['uuid']}",
+            "-e", f"NVIDIA_VISIBLE_DEVICES={device['uuid']}", "-e", "CUDA_VISIBLE_DEVICES=0"]
+
+
+def normalize_gpu_mode(gpu: str) -> str:
+    return "on" if str(gpu).lower() == "on" else "off"

acprof/_bundle/acprof/host/hardware_conditions.py

@@ -0,0 +1,130 @@
+"""Observe comparison conditions before capture; never poll in measurement windows."""
+from __future__ import annotations
+
+import csv
+import hashlib
+import os
+import re
+import subprocess
+from pathlib import Path
+
+from acprof.artifact_layout import ArtifactLayout
+from acprof.artifacts import atomic_write_json, read_json_object
+from acprof.cpu_affinity import normalize_cpu_set, parse_cpu_set
+from acprof.hardware_conditions import HARDWARE_FIELDS, conditions_path
+from acprof.host.command import run_command
+
+
+def _read(path) -> str | None:
+    try:
+        return Path(path).read_text().strip() or None
+    except OSError:
+        return None
+
+
+def _run(command) -> str | None:
+    try:
+        result = run_command(command, capture_output=True, text=True, timeout=10, check=False)
+        return result.stdout.strip() if result.returncode == 0 else None
+    except (OSError, subprocess.TimeoutExpired):
+        return None
+
+
+def observe_conditions(session) -> dict:
+    record = dict.fromkeys(HARDWARE_FIELDS)
+    errors = []
+    machine_id = _read("/etc/machine-id")
+    record["host_id"] = hashlib.sha256(machine_id.encode()).hexdigest() if machine_id else None
+    cpuinfo = _read("/proc/cpuinfo") or ""
+    record["cpu_model"] = sorted(set(re.findall(r"^model name\s*:\s*(.+)$", cpuinfo, re.M))) or None
+    if not re.fullmatch(r"[0-9a-f]{64}", str(getattr(session, "container_id", ""))):
+        return {**record, "errors": ["container identity unavailable"]}
+    raw = _run(["docker", "top", session.container_id, "-eo", "pid"])
+    try:
+        pids = [int(value.strip()) for value in (raw or "").splitlines()[1:]]
+        masks = set()
+        allowed = set()
+        for pid in pids:
+            try:
+                tasks = Path(f"/proc/{pid}/task").iterdir()
+                for task in tasks:
+                    try:
+                        affinity = os.sched_getaffinity(int(task.name))
+                    except ProcessLookupError:
+                        # Threads may exit between /proc enumeration and the
+                        # affinity read. A vanished thread cannot participate in
+                        # the later measurement window, so keep evidence from
+                        # the live siblings instead of discarding the snapshot.
+                        continue
+                    allowed.update(affinity)
+                    masks.add(normalize_cpu_set(",".join(str(cpu) for cpu in sorted(affinity))))
+            except FileNotFoundError:
+                # docker top is a snapshot; a process can legitimately exit
+                # before its task directory is opened.
+                continue
+        record["cpu_affinity"] = sorted(masks) or None
+        governors = {str(cpu): _read(f"/sys/devices/system/cpu/cpu{cpu}/cpufreq/scaling_governor")
+                     for cpu in sorted(allowed)}
+        policy = {
+            "governors": governors or None,
+            "boost": _read("/sys/devices/system/cpu/cpufreq/boost"),
+            "intel_no_turbo": _read("/sys/devices/system/cpu/intel_pstate/no_turbo"),
+        }
+        # Only one of the boost interfaces may apply; absent interfaces are
+        # reported distinctly from missing evidence for every applicable one.
+        if policy["boost"] is not None:
+            policy.pop("intel_no_turbo")
+        elif policy["intel_no_turbo"] is not None:
+            policy.pop("boost")
+        record["cpu_policy"] = policy
+    except (OSError, ValueError, AttributeError) as exc:
+        errors.append(f"CPU affinity: {exc}")
+    gpu = session.gpu_device
+    if gpu:
+        raw = _run(["nvidia-smi", f"--id={gpu.get('uuid', '')}",
+                    "--query-gpu=driver_version,power.limit,persistence_mode", "--format=csv,noheader,nounits"])
+        raw_values = next(csv.reader([raw], skipinitialspace=True), []) if raw else []
+        values: list[str | None] = [None if value.strip().lower() in {"", "n/a", "[n/a]", "not supported", "[not supported]"}
+                                    else value.strip() for value in raw_values]
+        record["gpu"] = {"model": gpu.get("name"), "uuid": gpu.get("uuid"),
+                         "driver": values[0] if len(values) == 3 else None,
+                         "power_limit_w": values[1] if len(values) == 3 else None,
+                         "persistence_mode": values[2] if len(values) == 3 else None}
+    else:
+        record["gpu"] = "not_applicable"
+    import requests
+    try:
+        response = requests.get(session.base_url + "/meta", timeout=5, headers={"Connection": "close"})
+        response.raise_for_status()
+        observed = response.json().get("runtime_parameters", {}).get("effective", {}).get("threads")
+        record["runtime_threads"] = observed if type(observed) is int and observed > 0 else None
+    except (requests.RequestException, ValueError, AttributeError, TypeError) as exc:
+        errors.append(f"runtime threads: {exc}")
+    return {**record, "errors": errors}
+
+
+def record_case_conditions(output_dir, case_id, session, *, cpuset_cpus="") -> None:
+    cpuset_cpus = normalize_cpu_set(cpuset_cpus)
+    path = conditions_path(ArtifactLayout.discover(output_dir))
+    payload = (
+        read_json_object(path, label="hardware conditions")
+        if path.exists()
+        else {"schema_version": 1, "cases": {}}
+    )
+    if not isinstance(payload, dict) or payload.get("schema_version") != 1 or not isinstance(payload.get("cases"), dict):
+        raise ValueError("unsupported hardware conditions schema")
+    observed = observe_conditions(session)
+    observed["requested_cpuset"] = cpuset_cpus or "unrestricted"
+    # Runtimes may pin individual workers more narrowly than Docker's CPU set.
+    # Every observed thread must stay inside the requested boundary.
+    affinities = observed["cpu_affinity"]
+    requested = parse_cpu_set(cpuset_cpus)
+    verified = bool(affinities) and all(
+        parse_cpu_set(mask) and parse_cpu_set(mask) <= requested for mask in affinities
+    ) if cpuset_cpus else True
+    if not verified:
+        observed["errors"].append("thread affinities are unknown or outside the requested CPU set")
+    payload["cases"][case_id] = observed
+    atomic_write_json(path, payload)
+    if not verified:
+        raise RuntimeError("cannot verify requested CPU set on the running container")

acprof/_bundle/acprof/host/images/__init__.py

@@ -0,0 +1 @@
+"""Host image planning, dependency evidence, inventory and runtime assembly."""