margin-arena 0.2.0b1__tar.gz

margin_arena-0.2.0b1/.gitattributes

@@ -0,0 +1,4 @@
+* text=auto eol=lf
+*.whl binary
+*.gz binary
+*.safetensors binary

margin_arena-0.2.0b1/.github/workflows/ci.yml

@@ -0,0 +1,31 @@
+name: CPU validation
+on: [push, pull_request]
+permissions:
+  contents: read
+jobs:
+  runtime:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
+        with:
+          python-version: '3.12'
+      - run: python -m pip install torch==2.14.0+cpu --index-url https://download.pytorch.org/whl/cpu
+      - run: python -m pip install -r requirements/runtime.txt
+      - run: python -m pip check
+      - run: python -m margin_arena verify-release
+      - run: python -m margin_arena build
+      - run: MARGIN_DOCKER_TESTS=1 python -m unittest discover -s tests -v
+      - run: python -m build
+      - run: python -m twine check dist/*
+  analysis:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
+        with:
+          python-version: '3.12'
+      - run: python -m pip install torch==2.14.0+cpu --index-url https://download.pytorch.org/whl/cpu
+      - run: python -m pip install -r requirements/analysis.txt
+      - run: python -m pip check
+      - run: python -m unittest discover -s tests -p test_interpret.py -v

margin_arena-0.2.0b1/.github/workflows/publish.yml

@@ -0,0 +1,33 @@
+name: Publish package
+on:
+  workflow_dispatch:
+permissions:
+  contents: read
+jobs:
+  build:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
+        with:
+          python-version: '3.12'
+      - run: python -m pip install build twine
+      - run: python -m margin_arena verify-release
+      - run: python -m build
+      - run: python -m twine check dist/*
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+        with:
+          name: distributions
+          path: dist/*
+  publish:
+    needs: build
+    runs-on: ubuntu-24.04
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093
+        with:
+          name: distributions
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33

margin_arena-0.2.0b1/.gitignore

@@ -0,0 +1,8 @@
+.venv/
+__pycache__/
+*.pyc
+*.egg-info/
+build/
+dist/
+runs/
+.env

margin_arena-0.2.0b1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 MARGIN Arena contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

margin_arena-0.2.0b1/MANIFEST.in

@@ -0,0 +1,8 @@
+include release.json
+recursive-include docs *.md
+recursive-include requirements *.txt
+recursive-include scripts *.py
+recursive-include tests *.py
+recursive-include .github *.yml
+include .gitattributes
+include .gitignore

margin_arena-0.2.0b1/PKG-INFO

@@ -0,0 +1,174 @@
+Metadata-Version: 2.4
+Name: margin-arena
+Version: 0.2.0b1
+Summary: Counted resource limits and observation records for external agent tasks
+License-Expression: MIT
+Project-URL: Repository, https://github.com/CarterYoo/margin-arena
+Project-URL: Issues, https://github.com/CarterYoo/margin-arena/issues
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: inspect-ai<0.4,>=0.3.263
+Provides-Extra: inspect
+Requires-Dist: inspect-ai<0.4,>=0.3.263; extra == "inspect"
+Provides-Extra: hf
+Requires-Dist: torch<3,>=2.6; extra == "hf"
+Requires-Dist: transformers<6,>=4.57; extra == "hf"
+Requires-Dist: safetensors<1,>=0.6; extra == "hf"
+Provides-Extra: data
+Requires-Dist: datasets<6,>=4; extra == "data"
+Requires-Dist: inspect-ai<0.4,>=0.3.263; extra == "data"
+Provides-Extra: control
+Requires-Dist: control-arena<20,>=19; extra == "control"
+Provides-Extra: nnsight
+Requires-Dist: nnsight<0.8,>=0.7; extra == "nnsight"
+Provides-Extra: lens
+Requires-Dist: transformer-lens<4,>=3.9; extra == "lens"
+Provides-Extra: sae
+Requires-Dist: sae-lens<7,>=6.51; extra == "sae"
+Provides-Extra: captum
+Requires-Dist: captum<1,>=0.8; extra == "captum"
+Provides-Extra: analysis
+Requires-Dist: pandas<4,>=2; extra == "analysis"
+Requires-Dist: safetensors<1,>=0.6; extra == "analysis"
+Dynamic: license-file
+
+# MARGIN Arena
+
+MARGIN provides counted resource limits and inspectable records for agents working on external tasks. Its scope is **detect, analyze, and understand**. It does not supply a new research dataset, train agents, implement interventions, or claim that a particular limit induces reward hacking.
+
+Version **0.2.0b1** is a beta. Choose a local Hugging Face causal language model, attach an external Inspect task or ControlArena setting, and specify each trial's resources. The source setting keeps its dataset, tools, sandbox, and scorer.
+
+## Install
+
+Python 3.11 or later is required; the tested environments use Python 3.12. Use Linux or WSL for Docker settings.
+
+```bash
+pip install "margin-arena[hf,control]==0.2.0b1"
+```
+
+Until the version appears on PyPI, install the public repository:
+
+```bash
+pip install "margin-arena[hf,control] @ git+https://github.com/CarterYoo/margin-arena.git"
+```
+
+For immutable experiments, replace the branch with a recorded commit. Tested CPU dependency snapshots are in `requirements/runtime.txt` and `requirements/analysis.txt`. Install CPU Torch from the official PyTorch index before applying those files. GPU users must select the appropriate official PyTorch build and record their hardware and driver versions.
+
+## Five explicit limits
+
+| Parameter | Counted quantity | Behavior at the boundary |
+| :--- | :--- | :--- |
+| `tokens` | All generated token IDs, including reasoning, JSON, code, arguments, and final text | Each generation has a reserved cap. A report reserve is included in the total. |
+| `seconds` | Monotonic time from the first actor turn, including observation and tools | Local model worker is terminated; no further actions are admitted. External tool cancellation has a separate caveat below. |
+| `verification_runs` | Admitted calls to explicitly named verification tools | Further verification calls are denied before execution; other tools remain available. |
+| `tool_calls` | Admitted individual calls, including submission and restart requests | Further work tools are unavailable; the existing report reserve remains available. |
+| `sessions` | Initial session plus admitted `restart_session` requests | Further restarts are denied; all other resource counters persist across restarts. |
+
+`None` disables time, tool, or verification limits. Sessions default to one. Each generation sees its remaining counters. Require `0 < report_tokens < tokens`; `per_call_tokens` caps one complete generation. Input tokens are recorded separately. Truncated output is terminal and never executes a partial request. An early final answer or stop is terminal even when resources remain.
+
+Model loading and initial sandbox preparation precede the trial clock. Source scoring and cleanup follow actor execution and are not charged to it. If generation is killed before returning, actual usage is unknown and its outstanding reservation stays in the ledger. Polling and process termination add a small measured delay; an exact millisecond stop is not promised.
+
+## Use a ControlArena setting
+
+```python
+from margin_arena import Arena, HFModel, Limits
+from control_arena.settings.apps.apps_setting import AppsSetting
+
+def main():
+    arena = Arena(
+        HFModel(name="YOUR_MODEL_REPOSITORY", revision="YOUR_MODEL_COMMIT",
+                device="cuda", dtype="bfloat16", context_window=8192),
+        Limits(tokens=16000, seconds=600, tool_calls=40, sessions=1,
+               per_call_tokens=1024),
+        seed=0,
+    )
+    task = arena.from_control_arena(AppsSetting())
+    arena.evaluate(task, out="runs/development_001", limit=1)
+
+if __name__ == "__main__":
+    main()
+```
+
+The main guard is required for the separate model process. Weights must fit the selected device. This beta adds no quantization, multi GPU distribution, or automatic backend switching.
+
+APPS has a submission tool and official scorer, but no interactive verification tool. The example therefore does not invent a verification allowance. For that axis, choose a source with an actual verification service and pass `verification_tools=("its_tool_name",)`. Multiple sessions in a mutable environment require an asynchronous `reset(state)` callback restoring that source's initial state. Missing bindings fail before generation.
+
+ControlArena is used in honest task mode. The actor receives the setting description and main task; hidden solutions and side task metadata stay outside its prompt. This adapter uses the setting lifecycle and official scorer, not ControlArena attack policies or control protocols. Validate each setting, especially tools that call other models or return nontext observations.
+
+```bash
+margin-eval --setting control_arena.settings.apps.apps_setting:AppsSetting \
+  --model YOUR_MODEL_REPOSITORY --revision YOUR_MODEL_COMMIT \
+  --device cuda --dtype bfloat16 --context-window 8192 \
+  --tokens 16000 --seconds 600 --tool-calls 40 --sessions 1 --per-call-tokens 1024 \
+  --limit 1 --out runs/development_001
+```
+
+## Other external data and environments
+
+`Arena.task(dataset, tools=..., scorer=..., sandbox=..., setup=..., post_submission=..., reset=...)` accepts Inspect samples and the source's lifecycle components. `--binding module:function` loads a trusted installed factory receiving the arena and returning this task.
+
+`margin_arena.datasets.from_jsonl` and `from_huggingface` map explicitly selected input, target, and ID columns. Only the input becomes visible. Hub revisions resolve to a commit; row hashes retain provenance. These loaders do **not** create a repository environment or official benchmark scorer from a text row. Loading SWE bench issue text, for example, is not a SWE bench evaluation without its official checkout, execution, and test bindings. Text CLI modes offer only explicit `match` or `includes` scoring.
+
+Each evaluation saves the complete sample snapshot and hash, setting configuration, package versions, limits, seed, and model identity. Preserve external image digests and assets not contained in the samples. Reuse saved samples or pin upstream dataset revisions; moving source references alone are insufficient for reproduction.
+
+## Model interface and evidence
+
+The local backend retains ordinary Hugging Face model and tokenizer objects. It applies the model's chat template and requests one strict JSON action or final report. Exact rendered text, input IDs, mask, output IDs, original response, sampling parameters, and tool schema are recorded. It does not silently trim context, repair JSON, remove reasoning, or retry failed responses. Models without a chat template and encoder decoder models are unsupported. Model selection does not establish competence: check the exact output contract on development cases before a larger run.
+
+Every sample has a hash chained `events.jsonl`, `result.json`, and optional activation artifacts. Inspect logs retain source scores. Their top level model name is the explicitly unused `mockllm/margin_transport_unused` transport; real inference runs through the MARGIN backend recorded in `configuration.json` and each trial. Do not attribute research results to the placeholder.
+
+```bash
+margin verify-trial runs/development_001/sample_SAMPLE_ID
+```
+
+Verification replays resource admissions and settlements, checks raw token counts, and compares the summary with its journal. It does not rerun external scorers, prove human review, or authenticate against a host that can rewrite all files and hashes. Infrastructure errors stop later sample execution. Malformed output, truncation, task failure, and refusal remain separate outcomes.
+
+## Read only interpretation
+
+Set `HFModel(capture_modules=("MODEL_SPECIFIC_MODULE",))` or repeat `--capture-module`. Native PyTorch hooks capture the **first prefill forward of the original generation**, with exact positions, module names, shapes, dtype, model identity, unique request ID, and safetensors hash. This beta does not save decode step activations or provide activation patching.
+
+| Library | Connection | CPU check |
+| :--- | :--- | :--- |
+| PyTorch and safetensors | Original prefill capture and verified tensor loading | Later one token cached forwards execute; output IDs and logits match the unobserved tiny model. |
+| NNsight 0.7 | Existing HF model and recorded input IDs | Layer output saved through an actual trace. |
+| TransformerLens 3.9 | TransformerBridge over existing HF weights | Activation cache obtained; logits match the raw tiny model without legacy weight folding. |
+| SAE Lens 6.51 | Encoding after declared revision, module, and feature width checks | A tiny untrained SAE encodes matching input and rejects a revision mismatch. |
+| Captum 0.9 | PyTorch LayerActivation | Actual layer observation checked. |
+| pandas and HF Datasets | Output token rows joined to prompt position and request ID | Exports agree with raw records. |
+
+These are software checks with an **untrained tiny CPU model**, not trained model findings or universal architecture support. SAE training preprocessing and intended layer must also match; no general pretrained SAE is supplied. Offline replay can differ due to dtype, kernels, cache, or library versions. Preserve that distinction.
+
+ControlArena 19 requires Hugging Face Hub below 1, whereas TransformerLens 3.9 requires Transformers 5 and Hub 1 or later. Use separate environments. There is intentionally no combined `all` extra.
+
+```bash
+python -m venv analysis_env
+analysis_env/bin/pip install "margin-arena[hf,analysis,nnsight,lens,sae,captum]==0.2.0b1"
+```
+
+```python
+from margin_arena.interpret import load_activations, to_pandas
+tensors, metadata = load_activations("runs/development_001/sample_ID/activations/REQUEST_ID.safetensors")
+table = to_pandas("runs/development_001/sample_ID/events.jsonl")
+```
+
+## Enforcement and interpretation limits
+
+Verification limits count named calls, not commands inside an unrestricted shell. True service execution quotas require an exclusive source service binding. MARGIN rejects unbound quotas but cannot discover every alternative path in third party environments.
+
+Local deadlines terminate the model worker. External tool deadlines cancel the awaiting coroutine and record remote effects as **unconfirmed**. Remote job cleanup depends on the source sandbox. Tools that secretly invoke other models are unsupported because those tokens are outside this ledger. Validate source properties before claiming exact end to end enforcement.
+
+Resource exhaustion does not prove a model felt pressure. A source score does not automatically label deception or reward hacking. Compare controlled conditions on the same external tasks and inspect actual actions, effects, and independent source scores. This release reports no new model experiment or causal finding.
+
+## Development checks
+
+```bash
+python -m unittest discover -s tests -v
+python -m margin_arena build
+MARGIN_DOCKER_TESTS=1 python -m unittest discover -s tests -p test_docker.py -v
+python scripts/check_external_apps.py --out runs/apps_binding_check
+```
+
+The final command fetches one public APPS row and submits a fixed diagnostic program through the original Docker setting and scorer, without model calls. The three old Artifact Depot tasks remain legacy regression fixtures, described in [the archived guide](docs/legacy_v0_1.md); they are not the beta's research dataset. First party code is MIT licensed; external artifacts retain their own licenses.
+
+See [the beta report](docs/beta_report.md) for actual validation and release status. References: [ControlArena](https://control-arena.aisi.org.uk/), [Inspect limits](https://inspect.aisi.org.uk/setting-limits.html), [NNsight](https://nnsight.net/documentation/modeling/language/), [TransformerLens](https://transformerlensorg.github.io/TransformerLens/generated/code/transformer_lens.model_bridge.bridge.html), [Captum](https://captum.ai/api/layer.html), [SAE Lens](https://jbloomaus.github.io/SAELens/).

margin_arena-0.2.0b1/README.md

@@ -0,0 +1,139 @@
+# MARGIN Arena
+
+MARGIN provides counted resource limits and inspectable records for agents working on external tasks. Its scope is **detect, analyze, and understand**. It does not supply a new research dataset, train agents, implement interventions, or claim that a particular limit induces reward hacking.
+
+Version **0.2.0b1** is a beta. Choose a local Hugging Face causal language model, attach an external Inspect task or ControlArena setting, and specify each trial's resources. The source setting keeps its dataset, tools, sandbox, and scorer.
+
+## Install
+
+Python 3.11 or later is required; the tested environments use Python 3.12. Use Linux or WSL for Docker settings.
+
+```bash
+pip install "margin-arena[hf,control]==0.2.0b1"
+```
+
+Until the version appears on PyPI, install the public repository:
+
+```bash
+pip install "margin-arena[hf,control] @ git+https://github.com/CarterYoo/margin-arena.git"
+```
+
+For immutable experiments, replace the branch with a recorded commit. Tested CPU dependency snapshots are in `requirements/runtime.txt` and `requirements/analysis.txt`. Install CPU Torch from the official PyTorch index before applying those files. GPU users must select the appropriate official PyTorch build and record their hardware and driver versions.
+
+## Five explicit limits
+
+| Parameter | Counted quantity | Behavior at the boundary |
+| :--- | :--- | :--- |
+| `tokens` | All generated token IDs, including reasoning, JSON, code, arguments, and final text | Each generation has a reserved cap. A report reserve is included in the total. |
+| `seconds` | Monotonic time from the first actor turn, including observation and tools | Local model worker is terminated; no further actions are admitted. External tool cancellation has a separate caveat below. |
+| `verification_runs` | Admitted calls to explicitly named verification tools | Further verification calls are denied before execution; other tools remain available. |
+| `tool_calls` | Admitted individual calls, including submission and restart requests | Further work tools are unavailable; the existing report reserve remains available. |
+| `sessions` | Initial session plus admitted `restart_session` requests | Further restarts are denied; all other resource counters persist across restarts. |
+
+`None` disables time, tool, or verification limits. Sessions default to one. Each generation sees its remaining counters. Require `0 < report_tokens < tokens`; `per_call_tokens` caps one complete generation. Input tokens are recorded separately. Truncated output is terminal and never executes a partial request. An early final answer or stop is terminal even when resources remain.
+
+Model loading and initial sandbox preparation precede the trial clock. Source scoring and cleanup follow actor execution and are not charged to it. If generation is killed before returning, actual usage is unknown and its outstanding reservation stays in the ledger. Polling and process termination add a small measured delay; an exact millisecond stop is not promised.
+
+## Use a ControlArena setting
+
+```python
+from margin_arena import Arena, HFModel, Limits
+from control_arena.settings.apps.apps_setting import AppsSetting
+
+def main():
+    arena = Arena(
+        HFModel(name="YOUR_MODEL_REPOSITORY", revision="YOUR_MODEL_COMMIT",
+                device="cuda", dtype="bfloat16", context_window=8192),
+        Limits(tokens=16000, seconds=600, tool_calls=40, sessions=1,
+               per_call_tokens=1024),
+        seed=0,
+    )
+    task = arena.from_control_arena(AppsSetting())
+    arena.evaluate(task, out="runs/development_001", limit=1)
+
+if __name__ == "__main__":
+    main()
+```
+
+The main guard is required for the separate model process. Weights must fit the selected device. This beta adds no quantization, multi GPU distribution, or automatic backend switching.
+
+APPS has a submission tool and official scorer, but no interactive verification tool. The example therefore does not invent a verification allowance. For that axis, choose a source with an actual verification service and pass `verification_tools=("its_tool_name",)`. Multiple sessions in a mutable environment require an asynchronous `reset(state)` callback restoring that source's initial state. Missing bindings fail before generation.
+
+ControlArena is used in honest task mode. The actor receives the setting description and main task; hidden solutions and side task metadata stay outside its prompt. This adapter uses the setting lifecycle and official scorer, not ControlArena attack policies or control protocols. Validate each setting, especially tools that call other models or return nontext observations.
+
+```bash
+margin-eval --setting control_arena.settings.apps.apps_setting:AppsSetting \
+  --model YOUR_MODEL_REPOSITORY --revision YOUR_MODEL_COMMIT \
+  --device cuda --dtype bfloat16 --context-window 8192 \
+  --tokens 16000 --seconds 600 --tool-calls 40 --sessions 1 --per-call-tokens 1024 \
+  --limit 1 --out runs/development_001
+```
+
+## Other external data and environments
+
+`Arena.task(dataset, tools=..., scorer=..., sandbox=..., setup=..., post_submission=..., reset=...)` accepts Inspect samples and the source's lifecycle components. `--binding module:function` loads a trusted installed factory receiving the arena and returning this task.
+
+`margin_arena.datasets.from_jsonl` and `from_huggingface` map explicitly selected input, target, and ID columns. Only the input becomes visible. Hub revisions resolve to a commit; row hashes retain provenance. These loaders do **not** create a repository environment or official benchmark scorer from a text row. Loading SWE bench issue text, for example, is not a SWE bench evaluation without its official checkout, execution, and test bindings. Text CLI modes offer only explicit `match` or `includes` scoring.
+
+Each evaluation saves the complete sample snapshot and hash, setting configuration, package versions, limits, seed, and model identity. Preserve external image digests and assets not contained in the samples. Reuse saved samples or pin upstream dataset revisions; moving source references alone are insufficient for reproduction.
+
+## Model interface and evidence
+
+The local backend retains ordinary Hugging Face model and tokenizer objects. It applies the model's chat template and requests one strict JSON action or final report. Exact rendered text, input IDs, mask, output IDs, original response, sampling parameters, and tool schema are recorded. It does not silently trim context, repair JSON, remove reasoning, or retry failed responses. Models without a chat template and encoder decoder models are unsupported. Model selection does not establish competence: check the exact output contract on development cases before a larger run.
+
+Every sample has a hash chained `events.jsonl`, `result.json`, and optional activation artifacts. Inspect logs retain source scores. Their top level model name is the explicitly unused `mockllm/margin_transport_unused` transport; real inference runs through the MARGIN backend recorded in `configuration.json` and each trial. Do not attribute research results to the placeholder.
+
+```bash
+margin verify-trial runs/development_001/sample_SAMPLE_ID
+```
+
+Verification replays resource admissions and settlements, checks raw token counts, and compares the summary with its journal. It does not rerun external scorers, prove human review, or authenticate against a host that can rewrite all files and hashes. Infrastructure errors stop later sample execution. Malformed output, truncation, task failure, and refusal remain separate outcomes.
+
+## Read only interpretation
+
+Set `HFModel(capture_modules=("MODEL_SPECIFIC_MODULE",))` or repeat `--capture-module`. Native PyTorch hooks capture the **first prefill forward of the original generation**, with exact positions, module names, shapes, dtype, model identity, unique request ID, and safetensors hash. This beta does not save decode step activations or provide activation patching.
+
+| Library | Connection | CPU check |
+| :--- | :--- | :--- |
+| PyTorch and safetensors | Original prefill capture and verified tensor loading | Later one token cached forwards execute; output IDs and logits match the unobserved tiny model. |
+| NNsight 0.7 | Existing HF model and recorded input IDs | Layer output saved through an actual trace. |
+| TransformerLens 3.9 | TransformerBridge over existing HF weights | Activation cache obtained; logits match the raw tiny model without legacy weight folding. |
+| SAE Lens 6.51 | Encoding after declared revision, module, and feature width checks | A tiny untrained SAE encodes matching input and rejects a revision mismatch. |
+| Captum 0.9 | PyTorch LayerActivation | Actual layer observation checked. |
+| pandas and HF Datasets | Output token rows joined to prompt position and request ID | Exports agree with raw records. |
+
+These are software checks with an **untrained tiny CPU model**, not trained model findings or universal architecture support. SAE training preprocessing and intended layer must also match; no general pretrained SAE is supplied. Offline replay can differ due to dtype, kernels, cache, or library versions. Preserve that distinction.
+
+ControlArena 19 requires Hugging Face Hub below 1, whereas TransformerLens 3.9 requires Transformers 5 and Hub 1 or later. Use separate environments. There is intentionally no combined `all` extra.
+
+```bash
+python -m venv analysis_env
+analysis_env/bin/pip install "margin-arena[hf,analysis,nnsight,lens,sae,captum]==0.2.0b1"
+```
+
+```python
+from margin_arena.interpret import load_activations, to_pandas
+tensors, metadata = load_activations("runs/development_001/sample_ID/activations/REQUEST_ID.safetensors")
+table = to_pandas("runs/development_001/sample_ID/events.jsonl")
+```
+
+## Enforcement and interpretation limits
+
+Verification limits count named calls, not commands inside an unrestricted shell. True service execution quotas require an exclusive source service binding. MARGIN rejects unbound quotas but cannot discover every alternative path in third party environments.
+
+Local deadlines terminate the model worker. External tool deadlines cancel the awaiting coroutine and record remote effects as **unconfirmed**. Remote job cleanup depends on the source sandbox. Tools that secretly invoke other models are unsupported because those tokens are outside this ledger. Validate source properties before claiming exact end to end enforcement.
+
+Resource exhaustion does not prove a model felt pressure. A source score does not automatically label deception or reward hacking. Compare controlled conditions on the same external tasks and inspect actual actions, effects, and independent source scores. This release reports no new model experiment or causal finding.
+
+## Development checks
+
+```bash
+python -m unittest discover -s tests -v
+python -m margin_arena build
+MARGIN_DOCKER_TESTS=1 python -m unittest discover -s tests -p test_docker.py -v
+python scripts/check_external_apps.py --out runs/apps_binding_check
+```
+
+The final command fetches one public APPS row and submits a fixed diagnostic program through the original Docker setting and scorer, without model calls. The three old Artifact Depot tasks remain legacy regression fixtures, described in [the archived guide](docs/legacy_v0_1.md); they are not the beta's research dataset. First party code is MIT licensed; external artifacts retain their own licenses.
+
+See [the beta report](docs/beta_report.md) for actual validation and release status. References: [ControlArena](https://control-arena.aisi.org.uk/), [Inspect limits](https://inspect.aisi.org.uk/setting-limits.html), [NNsight](https://nnsight.net/documentation/modeling/language/), [TransformerLens](https://transformerlensorg.github.io/TransformerLens/generated/code/transformer_lens.model_bridge.bridge.html), [Captum](https://captum.ai/api/layer.html), [SAE Lens](https://jbloomaus.github.io/SAELens/).

margin_arena-0.2.0b1/docs/beta_report.md

@@ -0,0 +1,137 @@
+# MARGIN Arena beta implementation report
+
+Date: September 14, 2026. Package version: 0.2.0b1.
+
+## Purpose
+
+The user requested an installable environment where researchers select an open weight model, connect existing datasets, and vary five countable resources. Its scope is detection, analysis, and understanding of agent behavior. Interventions are excluded and belong to later work. MARGIN is an experimental environment, not a new benchmark dataset, model ranking, or established finding about reward hacking.
+
+This beta connects external Inspect samples and ControlArena settings instead of expanding the three original Artifact Depot tasks. Those remain legacy regression fixtures. Prior HTR study records were not edited or relabeled. No GPU was allocated and no trained model research experiment was run during this release's validation.
+
+## Implementation inventory
+
+| File | Responsibility |
+| :--- | :--- |
+| `limits.py` | Immutable resource settings and trial counters shared across sessions. |
+| `arena.py` | Admission rules, source lifecycle, model execution, tool receipts, terminal stopping, and failure handling. |
+| `datasets.py` | External JSONL and Hugging Face rows with explicit input, target, and ID mapping. |
+| `beta_cli.py` | Installed `margin-eval` interface for model, source, resource, and capture settings. |
+| `huggingface.py` | Ordinary HF model and tokenizer, exact prompt and token records, read only prefill hooks. |
+| `model_worker.py` | Separate local model process, with confirmed process termination at a deadline. |
+| `interpret.py` | Verified tensor loading, token tables, NNsight, TransformerLens, SAE Lens, and Captum connections. |
+| `validation.py` | Independent replay of discrete resource accounting and raw record integrity. |
+| `tests/` | Counter boundaries, external task integration, worker cancellation, actual CPU generation, interpretation, and legacy regression checks. |
+| `requirements/` | Separate tested CPU runtime and analysis dependency snapshots. |
+| `.github/workflows/` | CPU CI and a separately triggered PyPI publishing workflow. |
+
+The existing native vLLM adapter, passive monitor, recorded action replay, and Artifact Depot environment remain legacy components. They are not claimed to be fully integrated alternatives to every new HF feature. A remote vLLM endpoint does not automatically expose internal activations. This release adds no monitor calibration result or intervention runner.
+
+## Resource rules
+
+| Resource | Count and boundary |
+| :--- | :--- |
+| Generated tokens | Every output ID, including reasoning, JSON, code, arguments, and report text. Before each call, reserve the smaller of its cap and remaining allowance. Settle returned usage; keep an outstanding reservation if usage is unknown. Report capacity is inside the total. |
+| Time | Monotonic elapsed time from before the first actor generation, including records and tool execution. Terminate the local worker and admit no later actions at expiry. |
+| Verification executions | Admitted calls to explicitly bound verification tools. An exhausted verification call never invokes the tool or consumes a normal tool slot. |
+| Tool calls | Each admitted action, including submission and explicit restart. Once exhausted, only the existing report allowance is available. |
+| Sessions | Initial attempt plus explicitly admitted restarts. Reset mutable source state with the source callback, while preserving every resource counter and prior record. |
+
+Model loading and initial sandbox preparation precede the actor clock; source scoring and cleanup follow it. These phases still have operational cost. Polling and process termination add a measured delay, so the implementation does not promise an exact millisecond stop. Input tokens are recorded separately and do not consume output allowance.
+
+Final submission or refusal is terminal even when resources remain. No additional response is requested after a final answer. Per call truncation is terminal and cannot execute a partial action. No context trimming, automatic JSON repair, hidden retry, or backend switching is performed. Model errors stop later actor execution. Incorrect task answers, malformed outputs, truncation, and infrastructure failures remain distinct.
+
+Missing source bindings fail before generation. An active verification limit requires actual named tools; mutable multi session tasks require an explicit asynchronous reset callback. Named tool invocation counting is a true service execution limit only when the source makes that service exclusive. Alternative access through an unrestricted shell is not automatically prevented. Tools that secretly invoke other models are unsupported because their tokens are outside this ledger.
+
+External tool deadlines cancel the awaiting coroutine and mark remote effects unconfirmed. Reset timeout effects are similarly unconfirmed. The source sandbox must terminate remote processes. Only the local model worker's termination is confirmed by MARGIN. These distinctions must remain visible in any experimental claim.
+
+## External datasets and scoring
+
+`Arena.task` accepts Inspect samples, source tools, scorer, sandbox, setup, post submission hook, and optional reset. `Arena.from_control_arena` uses the setting's public lifecycle in honest task mode. The actor receives the setting description and main task; hidden solutions and side task metadata stay outside the prompt. This is setting integration, not execution of ControlArena attack or control protocols.
+
+The JSONL and Hub loaders expose only the selected input field. The target remains available to the scorer. Hub revisions resolve to a commit and row hashes preserve provenance. Each evaluation saves a full source sample snapshot, sample hash, setting configuration, package versions, limits, seed, and package source hash. External images and source assets must also be preserved if they are not in those samples. Merely loading a SWE bench issue is not a SWE bench evaluation without its repository environment and official tests.
+
+One actual upstream APPS row was exercised through the original setting, Docker sandbox, submit tool, and scorer:
+
+| Item | Observed value |
+| :--- | :--- |
+| Dataset | `RoganInglis/apps-control-arena` |
+| Revision | `29db28f76042952d56add8671f798280b200744d` |
+| Sample | `0` |
+| Setting | ControlArena 19.0.0 `AppsSetting` |
+| Submission | Fixed diagnostic program `print(0)` |
+| Model inference | Zero calls |
+| Execution | One original submit invocation, then terminal stop |
+| Scorer | Original `apps_scorer`, one source test case |
+| Task result | `main_task_success = I`; expected `4`, observed `0` |
+| Infrastructure | Diagnostic execution succeeded and the sandbox was cleaned up |
+| Journal | Nine events; `a0fa7681c69928ecbdac59198eb7d8fb16986aa3e758781511e783df31bf607d` |
+
+The deliberately wrong program verifies that task failure does not become infrastructure failure. This is a software diagnostic, not a measurement of model APPS ability. APPS has no interactive verification tool, so the usage example does not invent one. Its Python 3.4 image is an upstream compatibility requirement.
+
+## Exact model and activation records
+
+The HF backend retains ordinary model and tokenizer objects, applies the model's chat template, and requests one strict JSON action or final report. It saves rendered text, input IDs, attention mask, generated IDs, original response, tool schema, sampling settings, and model identity. Models must have a chat template and fit the chosen device; encoder decoder models, quantization, and automatic multi GPU placement are outside this beta. Open weight model selection is not a guarantee of structured output competence. Validate each intended model contract on development cases.
+
+Read only PyTorch hooks save selected module outputs from the first prefill forward of the original generation. Every artifact has a unique request ID, input and output IDs, token positions, module names, shapes, dtypes, phase, model identity, forward call count, and safetensors hash. The loader verifies bytes and tensor metadata. Decode step activations are not stored. Offline replay is explicitly different from observation during original generation and may differ numerically across kernels, dtype, cache, and package versions.
+
+Inspect logs retain the external scores. Their top level model name is the explicitly unused `mockllm/margin_transport_unused` transport. Actual inference occurs through MARGIN's backend, identified in the configuration and each trial. Do not attribute a model result to the placeholder.
+
+The `margin verify-trial PATH` command replays resource admissions and settlement, checks raw token counts, verifies the journal chain, and compares the final summary with the journal. It does not rerun the external scorer, prove human review, or authenticate a host capable of rewriting all files and hashes.
+
+## Interpretation compatibility actually checked
+
+| Library | Tested connection | Scope |
+| :--- | :--- | :--- |
+| PyTorch and safetensors 0.8.0 | Original prefill capture, tensor round trip, hash and shape verification | Three generated tokens included two later one token cached forwards. Hooked and unhooked runs matched output IDs and followup logits on a small untrained CPU GPT2. |
+| NNsight 0.7.0 | Existing HF model wrapped in `LanguageModel`; actual trace saves MLP output | CPU observation integration, not all architectures or remote services. |
+| TransformerLens 3.9.0 | Existing HF weights through `TransformerBridge`; cache and logits compared | No legacy weight folding. Other architectures require separate equivalence checks. |
+| SAE Lens 6.51.0 | `StandardSAE.encode` after declared revision, module, and width checks | Untrained test SAE only; pretrained SAE choice and training preprocessing must match separately. |
+| Captum 0.9.0 | Actual `LayerActivation` output | Observation compatibility; no attribution or intervention conclusion. |
+| pandas and HF Datasets | One row per generated token joined to request and prompt position | Exports checked against raw records. |
+
+An additional tiny local saved model was loaded by the actual worker and run through an external Inspect task. Its three output tokens, terminal truncation, activation artifact, ledger verification, and process cleanup were checked. These are CPU software diagnostics with untrained models, not new research experiments or evidence that the environment elicits reward hacking.
+
+Runtime and analysis must currently be separate environments. ControlArena 19 requires Hugging Face Hub below 1; TransformerLens 3.9 requires Transformers 5 and Hub 1 or later. Runtime used Transformers 4.57.6; analysis used 5.17.0. Both used Python 3.12 and Torch 2.14.0+cpu. Both passed `pip check`. Saved tensors and exact input records connect the environments; no incompatible combined `all` extra is advertised.
+
+## Intermediate review and fixes
+
+Checks were inspected during development rather than only at the end. The following issues were corrected:
+
+1. Inspect dataset names are read only after construction; the adapter now sets the name in its initializer.
+2. The tool executor requires a message list and returns messages plus optional nested generation. The adapter now uses that contract and rejects unmetered nested generation.
+3. A source submission can have a custom name. Termination now uses the actual configured name.
+4. Inspect could schedule another sample despite its fail on error setting. A failure latch now prevents later actor execution; a test checks that the simulated first kernel failure creates only one trial.
+5. TransformerBridge was not exported at the installed package's top level. The documented module import now passes a real cache and logits test.
+6. ControlArena metadata contains enum values. Pydantic JSON conversion now preserves them in the journal. The failed first APPS diagnostic remains separate from its corrected run.
+7. Legacy source identification could scan an entire site packages directory after installation. Installed wheels now check their own RECORD hashes; source checkouts retain manifest verification.
+8. Reset timeout and nontext tool observations now preserve explicit failure or unconfirmed effect records instead of silently claiming compatibility.
+
+The regression suite checks counter boundaries, denial before service execution, persistent resources across resets, terminal submission, no context trimming, truncated action rejection, batch stopping, summary tamper rejection, actual cached generation, process termination, and analysis connections. Legacy Docker tests cover reference repairs, unauthorized exposure followed by restoration, protected paths, and editable tests that cannot forge the independent score.
+
+Upstream NNsight and SAE Lens emit deprecation warnings. Repeated Inspect evaluations emit an AnyIO unclosed stream warning; the analysis stack also emits an unclosed `/dev/null` warning. These are retained in logs. MARGIN tests check model process and task container termination, but the upstream warnings are not claimed to be fixed.
+
+## Distribution and reproduction
+
+Local validation completed with 50 tests discovered in the runtime environment: 46 passed and four optional interpretation tests were skipped there. The separate analysis environment ran all seven interpretation tests successfully, including those four. Thus all 50 distinct tests were exercised successfully across the two environments. The final runtime suite took 155.847 seconds; the seven analysis tests took 32.371 seconds. All four actual Docker tests passed. Both dependency environments passed `pip check`.
+
+An earlier full suite run encountered a source manifest mismatch while distribution assembly was running concurrently in the same tree. The builder temporarily creates a source archive staging directory. The failure was preserved; the final suite ran with the source unchanged and passed. Build and source verification now run sequentially. Source archive verification also recognizes the two exact setuptools metadata files added during packaging, while still checking every frozen source file.
+
+The wheel was installed into a fresh environment outside the checkout. The installed commands, packaged Docker assets, wheel RECORD checks, an external Inspect task, its original match scorer, and journal replay all passed without using the source directory. Wheel and source archive metadata passed `twine check`. Final public installation and publication details are recorded in the delivery report.
+
+The public repository is [CarterYoo/margin-arena](https://github.com/CarterYoo/margin-arena). Wheel and source archive builds expose `margin` and `margin-eval`. Source, tests, documentation, package assets, and dependency snapshots are covered by `release.json`. Runs, build output, caches, and credentials are excluded. Old 0.1 records and archives remain historical.
+
+CI defines separate runtime and analysis CPU jobs. PyPI publishing is a separately triggered workflow using Trusted Publishing. Ordinary pushes do not automatically publish. GitHub creation is complete. PyPI creation needs the user's authenticated account and a pending publisher bound to owner `CarterYoo`, repository `margin-arena`, workflow `publish.yml`, environment `pypi`. A configured pending publisher alone neither reserves a name nor establishes that upload succeeded. The final delivery report records actual publication and installation results.
+
+Use the README's commands with fixed source and model revisions. Begin with a few development examples checking exact context, tokenization, output format, actual effects, scores, and stop behavior. Review records before scaling. An invalid interface or excessive refusal is a measurement issue to diagnose, not a scientific null result. Preserve failed and negative outcomes and version any protocol revision.
+
+## Current claim
+
+MARGIN is a beta environment with explicit resource admission rules, external task integration, original generation records, and tested analysis interfaces. These particular CPU paths have been exercised. This does not show that resource scarcity causes reward hacking, that a model feels pressure, that every source has exclusive verification access, or that observational activations identify a causal neural mechanism. Those remain questions for calibrated model studies.
+
+## Primary references
+
+Architecture and data: [ControlArena](https://control-arena.aisi.org.uk/), [its Setting interface](https://github.com/UKGovernmentBEIS/control-arena/blob/main/control_arena/settings/_setting.py), [Inspect limits](https://inspect.aisi.org.uk/setting-limits.html), [upstream APPS data](https://huggingface.co/datasets/RoganInglis/apps-control-arena).
+
+Analysis interfaces: [NNsight](https://nnsight.net/documentation/modeling/language/), [TransformerLens TransformerBridge](https://transformerlensorg.github.io/TransformerLens/generated/code/transformer_lens.model_bridge.bridge.html), [SAE Lens](https://jbloomaus.github.io/SAELens/), [Captum](https://captum.ai/api/layer.html). Interfaces were also exercised in the exact installed versions described above.
+
+Distribution: [PyPI project creation through Trusted Publishing](https://docs.pypi.org/trusted-publishers/creating-a-project-through-oidc/), [the publishing workflow](https://docs.pypi.org/trusted-publishers/using-a-publisher/).