pantheon-gpu 1.2.0__py3-none-any.whl

pantheon_gpu/Makefile

@@ -0,0 +1,207 @@
+# PANTHEON Build System
+
+# --- Compiler Configuration ---
+HIPCC ?= hipcc
+CXXFLAGS := -O3 -std=c++14 -DNDEBUG -Ikernels/common
+BUILD_DIR := build
+ROCM_PATH ?= /opt/rocm
+BUILD_STAMP := $(BUILD_DIR)/.platform
+
+# Platform Detection
+ifeq ($(PLATFORM), MOCK)
+    COMPILER := g++
+    # Enable Mock Flag and suppress warnings about GPU-specific pragmas
+    CXXFLAGS += -DPANTHEON_MOCK -Wno-unknown-pragmas -pthread
+
+else ifeq ($(PLATFORM), CUDA)
+    COMPILER := nvcc
+    
+    # 1. Detect physical GPU architecture
+    # nvidia-smi writes "NVIDIA-SMI has failed because it couldn't communicate
+    # with the NVIDIA driver" to STDOUT, so 2>/dev/null does not suppress it.
+    # Without the numeric filter that sentence becomes DETECTED_ARCH: it is
+    # non-empty, so the failsafe below never fires, and it reaches the compiler
+    # as sm_<sentence> -- whose apostrophe breaks shell quoting, producing
+    # "/bin/sh: Syntax error: Unterminated quoted string" instead of the
+    # readable fallback this block was written to provide.
+    DETECTED_ARCH := $(strip $(shell nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -n 1 | tr -d '.' | grep -E '^[0-9]+$$'))
+    # Print it immediately during the parsing phase
+    $(info [DEBUG] The detected architecture is: $(DETECTED_ARCH))
+
+    # 2. Hopper WGMMA Promotion
+    # Hopper requires the 'a' suffix (sm_90a) to unlock WGMMA PTX instructions
+    ifeq ($(DETECTED_ARCH), 90)
+        DETECTED_ARCH := 90a
+    endif
+
+    # 3. The Ultimate Failsafe
+    # If nvidia-smi fails (e.g., compiling on a CPU-only login node), 
+    # default to Hopper (90a) so the build doesn't crash with an empty 'sm_' flag.
+    ifeq ($(DETECTED_ARCH),)
+        $(info [DEBUG] Hardware detection failed. Falling back to default sm_86...)
+        DETECTED_ARCH := 86 
+    endif
+
+    # --- Vendored Headers ---
+    # OptiX headers are NVIDIA-proprietary and are not redistributable, so this
+    # tree does not require them. Set OPTIX_PATH to an OptiX SDK include
+    # directory to build the real RT kernel; otherwise rt_virus builds its dummy
+    # kernel, exactly as it already does when HIP-RT is missing on AMD.
+    OPTIX_PATH ?= $(wildcard kernels/common/optix)
+    ifneq ($(OPTIX_PATH),)
+        OPTIX_INC := -I $(OPTIX_PATH)
+    else
+        OPTIX_INC :=
+        $(info [DEBUG] OptiX headers not found; rt_virus will build its dummy kernel. Set OPTIX_PATH to enable it.)
+    endif
+    NVENC_INC := -I kernels/common/nvenc
+
+    CXXFLAGS += -x cu --gpu-architecture=sm_$(DETECTED_ARCH) -Wno-deprecated-gpu-targets $(OPTIX_INC) $(NVENC_INC)
+    
+    # --- Link the CUDA Driver API and Dynamic Loader ---
+    LDFLAGS += -lcuda -ldl
+
+else
+    COMPILER := $(HIPCC)
+    
+    # 1. Detect physical GPU architecture (Modern ROCm uses amdgpu-arch)
+    DETECTED_GFX := $(strip $(shell amdgpu-arch 2>/dev/null | head -n 1))
+    ifeq ($(DETECTED_GFX),)
+        DETECTED_GFX := $(strip $(shell rocm_agent_enumerator 2>/dev/null | grep -v "cpu" | head -n 1))
+    endif
+    
+    # 2. Manual Override & Failsafe
+    # Allows cross-compiling on CPU nodes via: make PLATFORM=HIP TARGET_GFX=gfx950
+    ifdef TARGET_GFX
+        DETECTED_GFX := $(TARGET_GFX)
+    endif
+
+    # Tool output can include feature suffixes such as gfx90a:sramecc+:xnack-.
+    # hipcc's --offload-arch expects the architecture name only.
+    DETECTED_GFX := $(firstword $(subst :, ,$(DETECTED_GFX)))
+    
+    ifeq ($(DETECTED_GFX),)
+        $(info [DEBUG] Hardware detection failed. Falling back to MI300X default...)
+        DETECTED_GFX := gfx942
+    endif
+    
+    $(info [DEBUG] The AMD target architecture is: $(DETECTED_GFX))
+
+    # RDNA-family parts (gfx11xx, gfx12xx) provide WMMA; CDNA Instinct parts
+    # provide MFMA. Match on the two-digit family prefix rather than on
+    # individual model numbers, so any member of a family -- including ones
+    # that do not exist yet -- selects the right path with no source change.
+    GFX_FAMILY := $(shell printf '%s' '$(DETECTED_GFX)' | sed -E 's/^(gfx[0-9]{2}).*/\1/')
+    ifneq ($(filter gfx11 gfx12,$(GFX_FAMILY)),)
+        CXXFLAGS += -DPANTHEON_AMD_WMMA_TARGET=1
+        $(info [DEBUG] WMMA-capable family detected ($(GFX_FAMILY)); enabling WMMA path.)
+    endif
+  
+    # 3. FIX: Force Native Compilation for Pantheon Kernels
+    # If a specific cross-compile target wasn't requested, trust the compiler to 
+    # generate machine code for the exact silicon it is currently sitting on.
+    # This completely prevents Code 98 on bleeding-edge chips like the MI350X.
+    ifdef TARGET_GFX
+	CXXFLAGS += -std=c++17 --offload-arch=$(DETECTED_GFX)
+    else
+        CXXFLAGS += -std=c++17 --offload-arch=native
+    endif
+    
+    # --- Optional Local HIP-RT Vendoring ---
+    # The RT kernel builds a dummy fallback when HIP-RT headers are absent, so
+    # do not make every AMD build clone and compile HIP-RT by default.
+    VENDOR_DIR := vendor
+    HIPRT_DIR := $(VENDOR_DIR)/HIPRT
+    HIPRT_INSTALL := $(VENDOR_DIR)/install
+    HIPRT_TARGET := $(HIPRT_INSTALL)/lib/libhiprt64.so
+
+    ifeq ($(ENABLE_HIPRT),1)
+        CXXFLAGS += -I$(abspath $(HIPRT_INSTALL)/include)
+        LDFLAGS += -L$(abspath $(HIPRT_INSTALL)/lib) -lhiprt64 -Wl,-rpath=$(abspath $(HIPRT_INSTALL)/lib)
+        PLATFORM_DEPS := $(HIPRT_TARGET)
+    endif
+endif
+
+# OPTIX_PATH is part of the signature: setting it after a build without it
+# must invalidate the cached binaries, or rt_virus silently keeps running the
+# dummy kernel it compiled when the headers were absent.
+BUILD_SIGNATURE := PLATFORM=$(PLATFORM);CUDA_ARCH=$(DETECTED_ARCH);AMD_GFX=$(DETECTED_GFX);TARGET_GFX=$(DETECTED_GFX);ENABLE_HIPRT=$(ENABLE_HIPRT);OPTIX=$(OPTIX_PATH)
+
+# --- Auto-Discovery Logic ---
+
+# 1. Find all .cpp files inside kernels/ subdirectories
+ALL_SRCS := $(shell find kernels -name "*.cpp")
+
+# 2. Exclude anything in the 'common' directory or root headers
+SRCS := $(filter-out kernels/common/%, $(ALL_SRCS))
+
+# Empty discovery means find is unavailable (minimal container images such as
+# rockylinux:8 omit findutils) or the kernels tree is missing. Without this
+# guard make exits 0 having built nothing and every workload fails at launch.
+ifeq ($(strip $(SRCS)),)
+    $(error No kernel sources found under kernels/. Verify that 'find' is installed and the kernels/ directory exists)
+endif
+
+# 3. Determine Target Binaries
+BINS := $(foreach src,$(SRCS),$(BUILD_DIR)/$(basename $(notdir $(src))))
+
+# Shared headers included by many kernel translation units.
+COMMON_HEADERS := kernels/common/common.h kernels/common/ai_workload_template.h
+
+# --- Targets ---
+
+.PHONY: all clean directories FORCE
+
+# Add PLATFORM_DEPS here so HIP-RT is built before the binaries
+all: directories $(BUILD_STAMP) $(PLATFORM_DEPS) $(BINS)
+
+$(BUILD_STAMP): FORCE | directories
+	@if [ ! -f "$@" ] || [ "$$(cat "$@")" != "$(BUILD_SIGNATURE)" ]; then \
+		echo "[BUILD] Platform signature changed. Clearing stale binaries..."; \
+		find $(BUILD_DIR) -maxdepth 1 -type f ! -name ".platform" -delete; \
+		printf '%s\n' '$(BUILD_SIGNATURE)' > "$@"; \
+	fi
+
+$(BINS): $(BUILD_STAMP)
+
+# --- Auto-Dependency: HIP-RT ---
+$(HIPRT_TARGET):
+	@echo "============================================================"
+	@echo "[DEPENDENCY] Fetching and building HIP-RT locally..."
+	@echo "============================================================"
+	@mkdir -p $(VENDOR_DIR)
+	@if [ ! -d "$(HIPRT_DIR)" ]; then \
+		git clone https://github.com/GPUOpen-LibrariesAndSDKs/HIPRT.git $(HIPRT_DIR); \
+		cd $(HIPRT_DIR) && git submodule update --init --recursive; \
+	fi
+	@echo "[DEPENDENCY] Injecting $(DETECTED_GFX) architecture into AMD build scripts..."
+	@find $(HIPRT_DIR) -type f \( -name "*.py" -o -name "CMakeLists.txt" \) -exec sed -i "s/'gfx942'/'gfx942', '$(DETECTED_GFX)'/g" {} +
+	@find $(HIPRT_DIR) -type f \( -name "*.py" -o -name "CMakeLists.txt" \) -exec sed -i 's/"gfx942"/"gfx942", "$(DETECTED_GFX)"/g' {} +
+	@find $(HIPRT_DIR) -type f -name "CMakeLists.txt" -exec sed -i 's/gfx942/gfx942;$(DETECTED_GFX)/g' {} +
+	@mkdir -p $(HIPRT_DIR)/build
+	@cd $(HIPRT_DIR)/build && cmake -DCMAKE_BUILD_TYPE=Release -DBITCODE=ON -DPRECOMPILE=ON -DFORCE_DISABLE_CUDA=ON -DHIP_PATH=$(ROCM_PATH) -DPYTHON_EXECUTABLE=/usr/bin/python3 -DAMDGPU_TARGETS=$(DETECTED_GFX) -DGPU_TARGETS=$(DETECTED_GFX) ..
+	@cd $(HIPRT_DIR)/build && cmake --build . --config Release -j $$(nproc)
+	@echo "[DEPENDENCY] Build finished. Extracting all runtime assets..."
+	@mkdir -p $(HIPRT_INSTALL)/lib
+	@mkdir -p $(HIPRT_INSTALL)/include
+	@find $(HIPRT_DIR) -type f \( -name "*.hipfb" -o -name "*.bc" -o -name "libhiprt*64.so*" \) -exec cp -L {} $(HIPRT_INSTALL)/lib/ \;
+	@REAL_SO=$$(find $(HIPRT_INSTALL)/lib -name "libhiprt*64.so*" -type f | head -n 1); \
+	 ln -sf $$(basename $$REAL_SO) $(HIPRT_INSTALL)/lib/libhiprt64.so
+	@cp -r $(HIPRT_DIR)/hiprt $(HIPRT_INSTALL)/include/
+	@echo "[DEPENDENCY] HIP-RT and Orochi assets successfully installed to local vendor folder!"
+
+# Create build directory
+directories:
+	@mkdir -p $(BUILD_DIR)
+
+# Generic Rule: Build any binary from its corresponding source found in kernels tree
+vpath %.cpp $(sort $(dir $(SRCS)))
+
+$(BUILD_DIR)/%: %.cpp $(COMMON_HEADERS) $(BUILD_STAMP)
+	@echo "[BUILD] Compiling $< -> $@"
+	@$(COMPILER) $(CXXFLAGS) $< -o $@ $(LDFLAGS)
+
+clean:
+	@echo "[CLEAN] Removing build artifacts..."
+	@rm -rf $(BUILD_DIR)
+	@rm -rf $(VENDOR_DIR)

pantheon_gpu/VERSION

@@ -0,0 +1 @@
+1.2.0

pantheon_gpu/__init__.py

@@ -0,0 +1 @@
+"""Pantheon GPU stress and diagnostics suite."""

pantheon_gpu/_entry.py

@@ -0,0 +1,17 @@
+"""Console entry point for an installed Pantheon."""
+
+import os
+
+
+def main():
+    # site-packages is not writable, and the runner defaults to building
+    # beside its own source unless told otherwise. Respect an explicit
+    # override so a user can still choose the location.
+    if not os.environ.get("PANTHEON_BUILD_CACHE_DIR", "").strip():
+        cache = os.environ.get("XDG_CACHE_HOME", "").strip() or os.path.join(
+            os.path.expanduser("~"), ".cache")
+        os.environ["PANTHEON_BUILD_CACHE_DIR"] = os.path.join(
+            cache, "pantheongpu", "builds")
+
+    from . import pantheon
+    return pantheon.main()

pantheon_gpu/kernels/all_reduce/README.md

@@ -0,0 +1,17 @@
+# All-Reduce
+
+`all_reduce` is a two-rank GPU collective. It initializes one rank with `1`
+and the other with `2`, sums them on the primary GPU, then broadcasts the
+resulting `3` back to both ranks. `--verify` checks both GPU buffers.
+
+Pantheon uses direct peer DMA when bidirectional peer access is available. If
+the topology does not expose peer access, it reports a host-staged fallback
+instead. The fallback validates correctness but should not be used as a direct
+P2P bandwidth measurement.
+
+```bash
+pantheon --test all_reduce --duration 60 --gpu 0 --verify --profile
+```
+
+Use `--inject_error --verify` to confirm that verification detects a corrupted
+value on the peer rank.

pantheon_gpu/kernels/all_reduce/all_reduce.cpp

@@ -0,0 +1,178 @@
+#include "../common/common.h"
+
+#include <chrono>
+#include <iostream>
+#include <string>
+
+// A real two-rank all-reduce. GPU 1's input is copied to GPU 0, reduced
+// there, then the completed value is broadcast back to GPU 1.
+__global__ void init_collective(unsigned int* data, size_t count, unsigned int value) {
+    size_t idx = blockIdx.x * blockDim.x + threadIdx.x;
+    size_t stride = blockDim.x * gridDim.x;
+    for (; idx < count; idx += stride) data[idx] = value;
+}
+
+__global__ void add_collective(unsigned int* dst, const unsigned int* src, size_t count) {
+    size_t idx = blockIdx.x * blockDim.x + threadIdx.x;
+    size_t stride = blockDim.x * gridDim.x;
+    for (; idx < count; idx += stride) dst[idx] += src[idx];
+}
+
+__global__ void corrupt_collective(unsigned int* data) {
+    if (blockIdx.x == 0 && threadIdx.x == 0) data[0] ^= 1u;
+}
+
+__global__ void verify_collective(const unsigned int* data, size_t count, unsigned int expected, unsigned int* errors) {
+    size_t idx = blockIdx.x * blockDim.x + threadIdx.x;
+    size_t stride = blockDim.x * gridDim.x;
+    for (; idx < count; idx += stride) if (data[idx] != expected) atomicAdd(errors, 1);
+}
+
+int main(int argc, char** argv) {
+    if (argc < 4) return 1;
+    int primary = atoi(argv[1]);
+    int duration = atoi(argv[2]);
+    bool verify = false;
+    bool inject_error = false;
+    for (int i = 1; i < argc; ++i) {
+        std::string arg(argv[i]);
+        if (arg == "--verify") verify = true;
+        else if (arg == "--inject_error") inject_error = true;
+    }
+
+    int devices = 0;
+    CHECK(hipGetDeviceCount(&devices));
+    if (devices < 2) {
+        std::cout << "[PANTHEON] Skipping ALL_REDUCE: fewer than two GPUs." << std::endl;
+        std::cout << "Throughput: 0.0 GB/s" << std::endl;
+        return 0;
+    }
+
+    int peer = (primary + 1) % devices;
+    int forward_peer_access = 0;
+    int reverse_peer_access = 0;
+    CHECK(hipDeviceCanAccessPeer(&forward_peer_access, primary, peer));
+    CHECK(hipDeviceCanAccessPeer(&reverse_peer_access, peer, primary));
+    bool p2p = forward_peer_access && reverse_peer_access;
+    if (p2p) {
+        CHECK(hipSetDevice(primary));
+        CHECK(hipDeviceEnablePeerAccess(peer, 0));
+        CHECK(hipSetDevice(peer));
+        CHECK(hipDeviceEnablePeerAccess(primary, 0));
+    }
+
+    const size_t bytes = 64ull * 1024 * 1024;
+    const size_t count = bytes / sizeof(unsigned int);
+    const int blocks = 4096;
+    unsigned int* primary_data = nullptr;
+    unsigned int* peer_data = nullptr;
+    unsigned int* scratch = nullptr;
+    void* host_stage = nullptr;
+
+    CHECK(hipSetDevice(primary));
+    CHECK(hipMalloc(&primary_data, bytes));
+    CHECK(hipMalloc(&scratch, bytes));
+    if (!p2p) CHECK(hipHostMalloc(&host_stage, bytes));
+    CHECK(hipSetDevice(peer));
+    CHECK(hipMalloc(&peer_data, bytes));
+    CHECK(hipSetDevice(primary));
+    hipStream_t stream;
+    CHECK(hipStreamCreate(&stream));
+
+    LAUNCH_KERNEL(init_collective, blocks, 256, primary_data, count, 1u);
+    CHECK(hipSetDevice(peer));
+    LAUNCH_KERNEL(init_collective, blocks, 256, peer_data, count, 2u);
+    CHECK(hipDeviceSynchronize());
+    CHECK(hipSetDevice(primary));
+
+    auto reduce_and_broadcast = [&]() {
+        if (p2p) {
+            CHECK(hipMemcpyPeerAsync(scratch, primary, peer_data, peer, bytes, stream));
+        } else {
+            CHECK(hipSetDevice(peer));
+            CHECK(hipMemcpy(host_stage, peer_data, bytes, hipMemcpyDeviceToHost));
+            CHECK(hipSetDevice(primary));
+            CHECK(hipMemcpy(scratch, host_stage, bytes, hipMemcpyHostToDevice));
+        }
+        LAUNCH_KERNEL(add_collective, blocks, 256, primary_data, scratch, count);
+        if (p2p) {
+            CHECK(hipMemcpyPeerAsync(peer_data, peer, primary_data, primary, bytes, stream));
+            CHECK(hipStreamSynchronize(stream));
+        } else {
+            CHECK(hipMemcpy(host_stage, primary_data, bytes, hipMemcpyDeviceToHost));
+            CHECK(hipSetDevice(peer));
+            CHECK(hipMemcpy(peer_data, host_stage, bytes, hipMemcpyHostToDevice));
+            CHECK(hipDeviceSynchronize());
+            CHECK(hipSetDevice(primary));
+        }
+    };
+
+    std::cout << "[PANTHEON] GPU " << primary << ": ALL_REDUCE with GPU " << peer
+              << (p2p ? " (peer DMA)" : " (host-staged fallback)") << std::endl;
+    auto start = std::chrono::high_resolution_clock::now();
+    // Time only the collective itself: the between-iteration re-initialization
+    // is bookkeeping, not interconnect traffic, and must not dilute GB/s.
+    double reduce_seconds = 0.0;
+    size_t transferred = 0;
+    do {
+        auto step_start = std::chrono::high_resolution_clock::now();
+        reduce_and_broadcast();
+        reduce_seconds += std::chrono::duration<double>(
+            std::chrono::high_resolution_clock::now() - step_start).count();
+        LAUNCH_KERNEL(init_collective, blocks, 256, primary_data, count, 1u);
+        CHECK(hipSetDevice(peer));
+        LAUNCH_KERNEL(init_collective, blocks, 256, peer_data, count, 2u);
+        CHECK(hipDeviceSynchronize());
+        CHECK(hipSetDevice(primary));
+        // One gather crossing plus one broadcast crossing per collective.
+        transferred += bytes * 2;
+    } while (std::chrono::duration_cast<std::chrono::seconds>(std::chrono::high_resolution_clock::now() - start).count() < duration);
+    double seconds = reduce_seconds > 0.0 ? reduce_seconds : 1e-6;
+    std::cout << "Throughput: " << (transferred / 1e9) / seconds << " GB/s" << std::endl;
+
+    int exit_code = 0;
+    if (verify) {
+        reduce_and_broadcast();
+        if (inject_error) {
+            CHECK(hipSetDevice(peer));
+            LAUNCH_KERNEL(corrupt_collective, 1, 1, peer_data);
+            CHECK(hipDeviceSynchronize());
+        }
+
+        unsigned int* primary_errors = nullptr;
+        unsigned int* peer_errors = nullptr;
+        unsigned int primary_error_count = 0;
+        unsigned int peer_error_count = 0;
+        CHECK(hipSetDevice(primary));
+        CHECK(hipMalloc(&primary_errors, sizeof(unsigned int)));
+        CHECK(hipMemset(primary_errors, 0, sizeof(unsigned int)));
+        LAUNCH_KERNEL(verify_collective, blocks, 256, primary_data, count, 3u, primary_errors);
+        CHECK(hipDeviceSynchronize());
+        CHECK(hipMemcpy(&primary_error_count, primary_errors, sizeof(primary_error_count), hipMemcpyDeviceToHost));
+        CHECK(hipFree(primary_errors));
+
+        CHECK(hipSetDevice(peer));
+        CHECK(hipMalloc(&peer_errors, sizeof(unsigned int)));
+        CHECK(hipMemset(peer_errors, 0, sizeof(unsigned int)));
+        LAUNCH_KERNEL(verify_collective, blocks, 256, peer_data, count, 3u, peer_errors);
+        CHECK(hipDeviceSynchronize());
+        CHECK(hipMemcpy(&peer_error_count, peer_errors, sizeof(peer_error_count), hipMemcpyDeviceToHost));
+        CHECK(hipFree(peer_errors));
+        unsigned int errors = primary_error_count + peer_error_count;
+        if (errors) {
+            std::cout << "Verification: FAIL (" << errors << " errors)" << std::endl;
+            exit_code = 1;
+        } else {
+            std::cout << "Verification: PASS" << std::endl;
+        }
+    }
+
+    CHECK(hipSetDevice(primary));
+    CHECK(hipStreamDestroy(stream));
+    CHECK(hipFree(primary_data));
+    CHECK(hipFree(scratch));
+    if (host_stage) CHECK(hipHostFree(host_stage));
+    CHECK(hipSetDevice(peer));
+    CHECK(hipFree(peer_data));
+    return exit_code;
+}

pantheon_gpu/kernels/allocation_fragmentation/README.md

@@ -0,0 +1,3 @@
+# Allocation Fragmentation
+
+Synthetic runtime working-set diagnostic. Run `pantheon --test allocation_fragmentation --duration 60 --gpu 0 --mem 25 --verify --profile`.

pantheon_gpu/kernels/allocation_fragmentation/allocation_fragmentation.cpp

@@ -0,0 +1,177 @@
+// Allocation Fragmentation
+//
+// This workload previously included the shared AI kernel template and performed
+// no allocation at all -- it ran the same FMA loop as five other workloads while
+// its name promised allocator behaviour. It now does what it says: repeatedly
+// allocates and frees device memory in a pattern that fragments the heap, and
+// measures how the allocator copes.
+//
+// The stress is on the driver's allocator, not on the GPU pipelines, so the
+// device stays almost idle here by design.
+
+#include "../common/common.h"
+
+#include <algorithm>
+#include <chrono>
+#include <cstdio>
+#include <cstdlib>
+#include <iostream>
+#include <string>
+#include <vector>
+
+// Touch each block so the allocation is actually backed, and so a handed-back
+// pointer that overlaps a live one shows up as corruption rather than passing
+// silently.
+__global__ void stamp_block(unsigned int* block, size_t words, unsigned int tag) {
+    size_t stride = blockDim.x * gridDim.x;
+    for (size_t i = blockIdx.x * blockDim.x + threadIdx.x; i < words; i += stride) {
+        block[i] = tag ^ static_cast<unsigned int>(i);
+    }
+}
+
+__global__ void check_block(const unsigned int* block, size_t words, unsigned int tag,
+                            unsigned int* errors) {
+    size_t stride = blockDim.x * gridDim.x;
+    for (size_t i = blockIdx.x * blockDim.x + threadIdx.x; i < words; i += stride) {
+        if (block[i] != (tag ^ static_cast<unsigned int>(i))) atomicAdd(errors, 1u);
+    }
+}
+
+int main(int argc, char* argv[]) {
+    if (argc < 4) return 1;
+    int gpu_id = atoi(argv[1]);
+    int duration = atoi(argv[2]);
+    int mem_pct = atoi(argv[3]);
+
+    bool verify_mode = false;
+    int inject_error = 0;
+    int block_size = 256, grid_size = 0, kernel_loops = 1, warmup_iters = 0, sync_mode = 2;
+    for (int i = 4; i < argc; ++i) {
+        std::string a(argv[i]);
+        if (a == "--verify") verify_mode = true;
+        else if (a == "--inject_error") inject_error = 1;
+        else if (a == "--block_size" && i + 1 < argc) block_size = atoi(argv[++i]);
+        else if (a == "--grid_size" && i + 1 < argc) grid_size = atoi(argv[++i]);
+        else if (a == "--warmup_iters" && i + 1 < argc) warmup_iters = atoi(argv[++i]);
+        else if (a == "--sync_mode" && i + 1 < argc) sync_mode = atoi(argv[++i]);
+    }
+    normalize_kernel_launch_config(block_size, grid_size, kernel_loops, warmup_iters, sync_mode);
+
+    CHECK(hipSetDevice(gpu_id));
+    hipDeviceProp_t prop;
+    CHECK(hipGetDeviceProperties(&prop, gpu_id));
+
+    size_t free_bytes = 0, total_bytes = 0;
+    CHECK(hipMemGetInfo(&free_bytes, &total_bytes));
+    mem_pct = mem_pct < 1 ? 1 : (mem_pct > 99 ? 99 : mem_pct);
+    size_t budget = (free_bytes * static_cast<size_t>(mem_pct)) / 100;
+
+    // A spread of sizes is what fragments a heap. Equal-sized blocks are always
+    // reusable and would never expose the behaviour this test is named for.
+    const size_t kSmall = 64u * 1024u;
+    const size_t kLarge = 8u * 1024u * 1024u;
+    const size_t kSlots = 512;
+
+    std::cout << "[PANTHEON] GPU " << gpu_id << ": Allocation Fragmentation" << std::endl;
+    std::cout << "  -> Budget:        " << (budget / (1024 * 1024)) << " MiB" << std::endl;
+    std::cout << "  -> Block sizes:   " << (kSmall / 1024) << " KiB to "
+              << (kLarge / (1024 * 1024)) << " MiB" << std::endl;
+    std::cout << "  -> Slots:         " << kSlots << std::endl;
+    std::cout << "  -> Verify Mode:   " << (verify_mode ? "ON" : "OFF") << std::endl;
+    if (inject_error) std::cout << "[PANTHEON] Warning: SDC Fault Injection is ACTIVE!" << std::endl;
+
+    std::vector<unsigned int*> slots(kSlots, nullptr);
+    std::vector<size_t> sizes(kSlots, 0);
+    std::vector<unsigned int> tags(kSlots, 0);
+    size_t live_bytes = 0;
+    unsigned long long allocations = 0, frees = 0, failures = 0;
+    unsigned int* d_errors = nullptr;
+    if (verify_mode) {
+        CHECK(hipMalloc(&d_errors, sizeof(unsigned int)));
+        CHECK(hipMemset(d_errors, 0, sizeof(unsigned int)));
+    }
+
+    unsigned int rng = 0x9E3779B9u;
+    auto next = [&rng]() { rng ^= rng << 13; rng ^= rng >> 17; rng ^= rng << 5; return rng; };
+
+    std::cout << "[PANTHEON] Starting active telemetry phase..." << std::endl;
+    auto start = std::chrono::high_resolution_clock::now();
+    do {
+        for (size_t round = 0; round < kSlots; ++round) {
+            size_t slot = next() % kSlots;
+
+            if (slots[slot]) {
+                // Free roughly half the live blocks each pass, leaving holes of
+                // mixed sizes behind rather than a clean sweep.
+                CHECK(hipFree(slots[slot]));
+                live_bytes -= sizes[slot];
+                slots[slot] = nullptr;
+                sizes[slot] = 0;
+                ++frees;
+                continue;
+            }
+
+            size_t want = kSmall + (static_cast<size_t>(next()) % (kLarge - kSmall));
+            want &= ~static_cast<size_t>(255);
+            if (live_bytes + want > budget) continue;
+
+            unsigned int* block = nullptr;
+            if (hipMalloc(&block, want) != hipSuccess || block == nullptr) {
+                // A refusal while well under budget is the fragmentation signal.
+                ++failures;
+                continue;
+            }
+            size_t words = want / sizeof(unsigned int);
+            unsigned int tag = next();
+            int grid = init_launch_grid_size(prop, words, block_size);
+            LAUNCH_KERNEL(stamp_block, grid, block_size, block, words, tag);
+            slots[slot] = block;
+            sizes[slot] = want;
+            tags[slot] = tag;
+            live_bytes += want;
+            ++allocations;
+        }
+        CHECK(hipDeviceSynchronize());
+    } while (std::chrono::duration_cast<std::chrono::seconds>(
+                 std::chrono::high_resolution_clock::now() - start).count() < duration);
+
+    double seconds = std::chrono::duration<double>(
+        std::chrono::high_resolution_clock::now() - start).count();
+
+    std::cout << "Throughput: " << (static_cast<double>(allocations + frees) / seconds)
+              << " alloc-events/s" << std::endl;
+    std::cout << "  -> Allocations:   " << allocations << std::endl;
+    std::cout << "  -> Frees:         " << frees << std::endl;
+    std::cout << "  -> Refusals:      " << failures << " (allocator declined while under budget)" << std::endl;
+    std::cout << "  -> Live at end:   " << (live_bytes / (1024 * 1024)) << " MiB" << std::endl;
+
+    int exit_code = 0;
+    if (verify_mode) {
+        std::cout << "[PANTHEON] Verifying every live block still holds its tag..." << std::endl;
+        if (inject_error) {
+            for (size_t s = 0; s < kSlots; ++s) {
+                if (slots[s]) {
+                    unsigned int bad = 0x0BADBEEFu;
+                    CHECK(hipMemcpy(slots[s], &bad, sizeof(bad), hipMemcpyHostToDevice));
+                    break;
+                }
+            }
+        }
+        for (size_t s = 0; s < kSlots; ++s) {
+            if (!slots[s]) continue;
+            size_t words = sizes[s] / sizeof(unsigned int);
+            int grid = init_launch_grid_size(prop, words, block_size);
+            LAUNCH_KERNEL(check_block, grid, block_size, slots[s], words, tags[s], d_errors);
+        }
+        CHECK(hipDeviceSynchronize());
+        unsigned int host_errors = 0;
+        CHECK(hipMemcpy(&host_errors, d_errors, sizeof(unsigned int), hipMemcpyDeviceToHost));
+        std::cout << "Verification: " << (host_errors ? "FAIL" : "PASS")
+                  << " (" << host_errors << " corrupted words across live blocks)" << std::endl;
+        exit_code = host_errors ? 1 : 0;
+    }
+
+    for (size_t s = 0; s < kSlots; ++s) if (slots[s]) CHECK(hipFree(slots[s]));
+    if (d_errors) CHECK(hipFree(d_errors));
+    return exit_code;
+}

pantheon_gpu/kernels/atomic_virus/README.md

@@ -0,0 +1,57 @@
+# Atomic Virus
+
+`atomic_virus` is an L2/cache-arbiter stress test. It repeatedly performs atomic adds across a large VRAM allocation, forcing read-modify-write traffic that is harder on cache arbitration and memory ordering than plain reads or writes.
+
+![Atomic Virus execution flow](./atomic_virus_flow.svg)
+
+## What It Stresses
+
+| Area | Stress mechanism |
+| :--- | :--- |
+| L2 cache arbiters | Many threads contend on atomic read-modify-write paths. |
+| Memory fabric | Wide grid-stride access walks a large allocation. |
+| Ordering hardware | Atomic operations force serialization at cache-line granularity. |
+| Retention/correctness | Optional verification checks the expected atomic count. |
+
+## How It Works
+
+1. The test uses `--mem` percent of available VRAM to size the buffer, but **the implementation clamps the allocation** (currently 128 MiB max) so each kernel launch finishes in bounded time. On very large GPUs, `--mem 99` would otherwise imply trillions of atomics **per launch** before the next `hipDeviceSynchronize`, and the host watchdog (`duration + 30` s in `pantheon.py`) would SIGKILL the process with exit code **-9** while telemetry still shows `0.0 ERR`. For a true “almost full VRAM” allocation check, use a different test; keep `atomic_virus` for L2/atomic stress at the clamped footprint.
+2. Memory is initialized to zeroes or ones.
+3. Each thread walks the allocation as `uint4` storage reinterpreted as `unsigned int`.
+4. For each visited integer, the kernel performs `kernel_loops` atomic adds.
+5. Warmup updates are reset before the measured phase so verification has a clean baseline.
+6. With `--verify`, the final values are compared against the expected launch count.
+
+## Command Examples
+
+```bash
+pantheon --test atomic_virus --gpu 0 --duration 30 --mem 60
+pantheon --test atomic_virus --gpu 0 --duration 30 --mem 60 --verify
+```
+
+## Runtime Parameters
+
+| Parameter | Default | Effect |
+| :--- | ---: | :--- |
+| `block_size` | `256` | Threads per block. |
+| `grid_size` | `0` | Number of blocks. `0` means auto-calculate from occupancy. |
+| `kernel_loops` | `100` | Atomic adds per visited integer. |
+| `warmup_iters` | `5` | Warmup launches before telemetry, followed by a memory reset. |
+| `sync_mode` | `2` | `0=Spin`, `1=Yield`, `2=Block`. |
+| `init_pattern` | `0` | `0=Zeroes`, `1=Ones`. |
+
+## Output And Interpretation
+
+`Throughput` is reported in `MAPS`: million atomic operations per second. Higher MAPS means more atomic traffic, but the most stressful configuration may be the one that produces higher power, temperature, or throttle signals rather than raw throughput.
+
+## Failure Signals
+
+| Symptom | Possible meaning |
+| :--- | :--- |
+| Verification mismatch | Atomic count mismatch, cache/L2 fault, or injected corruption. |
+| Very low MAPS | Severe atomic serialization, throttling, or too much memory footprint. |
+| High power with modest MAPS | Useful stress point: the fabric is busy even when progress is slow. |
+
+## Source
+
+The implementation lives in [`atomic_virus.cpp`](./atomic_virus.cpp).