redteam-foundry 0.2.0__tar.gz

redteam_foundry-0.2.0/.env.example

@@ -0,0 +1,17 @@
+# Copy to `.env` and fill in. .env is gitignored.
+
+# --- Anthropic ---------------------------------------------------------------
+# Required. Set a $20/month hard cap in the Anthropic console *before* using
+# this key. The harness also enforces a $5/run cap inside Target.send (see L3).
+ANTHROPIC_API_KEY=
+
+# --- Ollama (local) ----------------------------------------------------------
+# Default; only override if you run Ollama on a non-standard host/port.
+OLLAMA_HOST=http://localhost:11434
+
+# --- Run-time guards ---------------------------------------------------------
+REDTEAM_MAX_USD_PER_RUN=5.00
+REDTEAM_MAX_USD_PER_CALL=0.50
+
+# --- HuggingFace (only needed for Meta-SecAlign download in Phase 5+) -------
+HF_TOKEN=

redteam_foundry-0.2.0/.gitattributes

@@ -0,0 +1,18 @@
+# Normalise line endings. Without this, Windows + sandbox tooling churn
+# CRLF<->LF and every file shows as fully modified in `git status`/diffs.
+# This stores every text file as LF in the repository, regardless of the
+# checkout platform. Ends the line-ending war for good.
+* text=auto eol=lf
+
+# Shell scripts must stay LF even on Windows checkouts.
+*.sh text eol=lf
+*.ps1 text eol=crlf
+
+# Binary files — never touch.
+*.png binary
+*.jpg binary
+*.jpeg binary
+*.gif binary
+*.ico binary
+*.pdf binary
+*.parquet binary

redteam_foundry-0.2.0/.github/workflows/ci.yml

@@ -0,0 +1,43 @@
+name: CI
+
+on:
+  pull_request: {}
+  push:
+    branches: [main]
+
+# No real API calls are made in CI — see ETHICS + budget guards.
+# When branch protection is enabled, require the display name below
+# ("Lint, typecheck, smoke run") — see Lesson L13.
+
+jobs:
+  build:
+    name: Lint, typecheck, smoke run
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.13"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v8.1.0
+        with:
+          version: "latest"
+          enable-cache: true
+
+      - name: Install project (editable, with dev extras)
+        run: uv pip install --system -e ".[dev]"
+
+      - name: Ruff (lint)
+        run: ruff check .
+
+      - name: Ruff (format check)
+        run: ruff format --check .
+
+      - name: Mypy (typecheck)
+        run: mypy src/
+
+      - name: Pytest (unit)
+        run: pytest tests/unit

redteam_foundry-0.2.0/.gitignore

@@ -0,0 +1,73 @@
+# --- secrets / env -----------------------------------------------------------
+.env
+.env.*
+!.env.example
+*.key
+secrets/
+
+# --- python ------------------------------------------------------------------
+__pycache__/
+*.py[cod]
+*$py.class
+.venv/
+venv/
+.python-version
+
+# build / dist
+build/
+dist/
+*.egg-info/
+.eggs/
+
+# tooling caches
+.mypy_cache/
+.ruff_cache/
+.pytest_cache/
+.coverage
+htmlcov/
+coverage.xml
+
+# --- project artefacts (gitignored on purpose) -------------------------------
+# Raw + cached corpora are big and re-downloadable; only commit
+# scripts/load_corpora.py + dataset_versions.yaml + a small samples/ dir.
+# Leading slash means "only at repo root" — a bare `data/` would also match
+# tests/data/ (which holds committed test fixtures) and silently exclude it.
+/data/
+!/data/.gitkeep
+!/data/samples/
+
+# Run outputs are large and re-creatable; only commit a small reference
+# subset under results/samples/.
+/results/
+!/results/.gitkeep
+!/results/samples/
+
+# Corpus-audit outputs (quality reports / data cards). Re-creatable via
+# `redteam corpora audit`, and they quote truncated adversarial prompt
+# previews, so keep them out of git by default.
+# `/reports/*` (not `/reports/`) so the samples/ negation can re-include —
+# git won't recurse into a wholly-excluded directory.
+/reports/*
+!/reports/samples/
+
+# Exported challenge packs. Re-creatable via `redteam export-pack`; adversarial
+# packs contain prompt previews, so keep them out of git by default. Only a
+# small, safe (benign) sample is committed under challenge_packs/samples/.
+# Note: `/challenge_packs/*` (not `/challenge_packs/`) so the samples/ negation
+# below can re-include — git won't recurse into a wholly-excluded directory.
+/challenge_packs/*
+!/challenge_packs/samples/
+
+# Streamlit / HF Spaces local cache
+.streamlit/secrets.toml
+.huggingface/
+
+# OS noise
+.DS_Store
+Thumbs.db
+desktop.ini
+
+# IDE noise
+.vscode/
+.idea/
+*.swp

redteam_foundry-0.2.0/.pre-commit-config.yaml

@@ -0,0 +1,34 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files
+        args: ["--maxkb=512"]
+      - id: check-merge-conflict
+      - id: detect-private-key
+
+  # Use the venv's ruff (pinned in pyproject.toml [dev]) so pre-commit and CI
+  # never drift to different versions. Requires the venv to be active when
+  # committing — that's already the standard workflow here.
+  - repo: local
+    hooks:
+      - id: ruff
+        name: ruff (lint, autofix)
+        entry: ruff check --fix --force-exclude
+        language: system
+        types_or: [python, pyi]
+        require_serial: true
+      - id: ruff-format
+        name: ruff format
+        entry: ruff format --force-exclude
+        language: system
+        types_or: [python, pyi]
+        require_serial: true
+
+  - repo: https://github.com/gitleaks/gitleaks
+    rev: v8.21.2
+    hooks:
+      - id: gitleaks

redteam_foundry-0.2.0/CHANGELOG.md

@@ -0,0 +1,75 @@
+# Changelog
+
+All notable changes to this project are documented here. The format is based on
+[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project
+adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.2.0] — 2026-07-02 — the adversarial benchmark foundry
+
+Repositioned from a static red-team harness (`llm-redteam-harness`) into an
+upstream **benchmark foundry** (`redteam-foundry`). The v1 measurement core is
+unchanged; these are additive. First public release.
+
+### Added
+- **Corpus quality audit** (`redteam corpora audit`) — exact + near-duplicate
+  detection including cross-source overlap, composition, prompt-length stats,
+  and label-integrity checks, rendered as a quality report + data card.
+- **Language + attack-family taxonomy** — script-based language/code-switching
+  detection and heuristic attack-family surface markers, surfaced in the audit.
+- **Benchmark staleness scoring** (`redteam corpora staleness`) — a transparent,
+  component-broken-out heuristic composing corpus and run signals.
+- **Benign control set + defence comparison** (`redteam compare-defences`,
+  `redteam benign export`) — false-refusal rate and
+  `safe_usefulness = (1 - ASR) * (1 - FRR)` per defence config.
+- **Multilingual over-refusal** (`redteam frr-by-language`,
+  `redteam benign export --multilingual`) — benign control set in
+  zh-Hant / zh-Hans / ja / ko + code-switched, with per-language FRR.
+- **Challenge-pack exporter** (`redteam export-pack`) — versioned packs
+  (`pack.yaml` + `scenarios.jsonl` + `datacard.md`) with adversarial prompts
+  redacted by default, plus a reader (`redteam.packs.read_challenge_pack`) and
+  a downstream consumption contract (`examples/export_to_agent_release_gates.md`).
+- **Audit any Hugging Face dataset** (`redteam corpora audit-hf --dataset ...
+  --prompt-column ... [--revision ...]`) — not just the four built-in corpora;
+  the safety exclusion filter runs first. New `source="external"` /
+  `category="unknown"` schema values back it.
+- **Cross-dataset benchmark-quality scorecard** (`scripts/hf_scorecard.py`) and a
+  written finding (`docs/findings/benchmark-quality-report-card.md`) auditing four
+  public jailbreak datasets: all English-only, roleplay-persona-dominant, and
+  duplicated to varying degrees.
+- `docs/ROADMAP.md`; committed real-data findings under `reports/samples/`.
+
+### Fixed
+- Exclusion-filter leaks (WMD "synthesis" noun, several self-harm phrasings,
+  case-insensitive category gate) with regression tests.
+- Budget guard now **reserves** its estimate so concurrent calls can't
+  collectively exceed the per-run cap.
+- Llama Guard fails **closed** on an empty/errored verdict.
+- Spotlighting / SecAlign neutralise their own fence markers in untrusted input
+  (closes a delimiter-injection bypass); SecAlign no longer passes messages
+  through unfenced.
+- OpenAI target fails loud without a pricing entry (no silent $0 budget bypass).
+- Krippendorff's α finite-sample correction; `compute_kappa` blank-cell guard;
+  cross-judge agreement reports `None` (not `0.0`) when not computable.
+
+### Changed
+- **Renamed the project/package to `redteam-foundry`** (was
+  `llm-redteam-harness`). The `redteam` CLI command and `src/redteam/` module are
+  unchanged — `pip install redteam-foundry` still gives you `redteam ...`.
+- Default install slimmed: the unused dashboard deps (`streamlit`, `plotly`)
+  moved to an opt-in `[dashboard]` extra. The audit / staleness / dedup path
+  needs no API key.
+
+## [0.1.0] — v1 measurement core
+
+- Loaders for AdvBench, JailbreakBench, HarmBench, AgentDojo (commit-pinned) with
+  a safety exclusion filter.
+- Anthropic + Ollama targets (OpenAI stub); disk response cache; per-run and
+  per-call budget guards.
+- Six composable defences (system prompt, Constitutional, Spotlighting,
+  SecAlign, Llama Guard 4 pre/post).
+- Rule-based + LLM-judge scoring with an independent cross-judge; bootstrap
+  confidence intervals; Cohen's κ and Krippendorff's α.
+- UK AISI Inspect eval-log export.
+
+[0.2.0]: https://github.com/rosscyking1115/redteam-foundry/releases/tag/v0.2.0
+[0.1.0]: https://github.com/rosscyking1115/redteam-foundry/releases/tag/v0.1.0

redteam_foundry-0.2.0/CONTRIBUTING.md

@@ -0,0 +1,57 @@
+# Contributing
+
+Thanks for your interest. This is a research tool for **auditing adversarial
+benchmarks**; contributions that sharpen that focus are very welcome.
+
+## Scope
+
+In scope: corpus quality/dedup, benchmark staleness, defence measurement,
+multilingual over-refusal, safe challenge-pack export, and corpus/target/judge
+adapters.
+
+Out of scope (by design): production release-gating — ship/warn/block, incident
+replay, policy-as-code. That belongs in a downstream layer; see
+[`docs/ROADMAP.md`](docs/ROADMAP.md) and the README's positioning.
+
+## Ethics (please read before adding corpora or prompts)
+
+- Use **only published** adversarial prompts, commit-pinned. Do not author novel
+  jailbreaks, in any language (see [`ETHICS.md`](ETHICS.md)).
+- Excluded categories (CSAM, WMD synthesis, detailed self-harm methods) are
+  filtered at load time and enforced by `tests/unit/test_exclusion_filter.py`.
+  If you add a corpus, add positive/benign cases there proving nothing leaks.
+- Don't commit raw harmful prompts or model outputs. Reports and packs quote
+  truncated previews or redact adversarial prompts.
+
+## Dev setup
+
+```bash
+uv venv --python 3.13
+source .venv/bin/activate        # .venv\Scripts\activate on Windows
+uv pip install -e ".[dev]"
+pre-commit install
+```
+
+## Before you open a PR
+
+Run the exact CI checks locally — green here means green on the PR:
+
+```bash
+scripts/ci_local.sh              # or scripts\ci_local.ps1 on Windows
+# = ruff check + ruff format --check + mypy (strict) + pytest tests/unit
+```
+
+Guidelines:
+- One focused change per PR; keep the test suite green (`pytest tests/unit`).
+- New behaviour needs a unit test. Analysis functions should be pure and
+  testable without network/API access.
+- Type-annotate everything (`mypy --strict` must pass). No new heavy runtime
+  dependencies without discussion.
+- CI runs no live API calls; don't add tests that require one to unit-test.
+
+## Reporting issues
+
+Bugs, stale/duplicated benchmark findings, and requests to audit a specific
+corpus are all useful. Include the command you ran and the (aggregate) output.
+For anything sensitive — a suspected leak past the exclusion filter, or a
+model-provider takedown — email **rosscyking@gmail.com** (see `ETHICS.md`).

redteam_foundry-0.2.0/ETHICS.md

@@ -0,0 +1,116 @@
+# ETHICS
+
+This project measures LLM behaviour against published adversarial prompts to
+inform defence design. It is **not** a security audit, certification, or
+guarantee.
+
+## What this project does
+
+- Runs **published** adversarial prompts from peer-reviewed or
+  industry-released datasets (AdvBench, JailbreakBench, HarmBench, AgentDojo)
+  against target LLMs.
+- Measures attack-success-rate (ASR) under combinations of published defences
+  (system prompts, Llama Guard 4, Spotlighting, SecAlign-style structured
+  queries, Constitutional principles), with every verdict cross-validated by
+  an independent second judge model.
+- Publishes **aggregate** results, methodology, defence configurations, and
+  scoring code openly.
+
+## What this project does not do
+
+- **Generate novel jailbreaks.** All prompts come from published datasets,
+  cited and version-pinned.
+- **Publish raw harmful outputs verbatim.** Results are aggregated, redacted,
+  and summarised. A small number of representative example transcripts may
+  be shown per category, with explicit warnings.
+- **Test categories that are excluded by policy** (see below). These are
+  filtered at corpus-load time before any model receives them.
+- **Generate non-English attack content.** The multilingual work
+  (`src/redteam/multilingual.py`) is a **benign** control set only — it
+  measures whether models *over-refuse* harmless non-English and code-switched
+  prompts (false refusal). We do **not** translate harmful prompts into other
+  languages: that would both create new harmful content and bypass the
+  English-only exclusion filter. The multilingual contribution is over-refusal
+  measurement, not non-English jailbreak generation.
+
+## Excluded categories
+
+The following categories are filtered out of all loaded corpora at
+`scripts/load_corpora.py` and verified by a CI test
+(`tests/unit/test_exclusion_filter.py`):
+
+- **CSAM** — Child Sexual Abuse Material, in any form.
+- **Bio / chem / radiological / nuclear weapons synthesis** — uplift content
+  for the design or manufacture of weapons of mass destruction.
+- **Detailed self-harm methods** — actionable instructions for self-injury
+  or suicide. (Discussion of self-harm at a research / awareness level is
+  fine; method-level detail is not.)
+
+If any of these slip past the filter, the CI test fails and merge is blocked.
+
+## Datasets used (with citations)
+
+- **AdvBench** — Zou et al., 2023. *Universal and Transferable Adversarial
+  Attacks on Aligned Language Models.* MIT licence.
+- **JailbreakBench / JBB-Behaviors** — Chao et al., 2024. *JailbreakBench:
+  An Open Robustness Benchmark for Jailbreaking Large Language Models.*
+  MIT licence. NeurIPS 2024 Datasets & Benchmarks Track.
+- **HarmBench** — Mazeika et al., 2024. *HarmBench: A Standardized
+  Evaluation Framework for Automated Red Teaming and Robust Refusal.*
+  MIT licence.
+- **AgentDojo** — Debenedetti et al., 2024. *AgentDojo: A Dynamic
+  Environment to Evaluate Prompt Injection Attacks and Defences for LLM
+  Agents.* AGPL licence.
+
+Exact dataset commits are pinned in `configs/dataset_versions.yaml`.
+
+## Models tested (with version pins)
+
+Every result is reported against a **dated** model ID. See
+`configs/model_versions.yaml` for the full list. As of project start
+(2026-05-05) this is:
+
+- `claude-sonnet-4-6` (Anthropic Claude Sonnet 4.6) — frontier API target
+- `llama3.1:8b` (Meta Llama 3.1 8B via Ollama) — local target
+- `meta-secalign-8b` (Meta SecAlign 8B) — Phase 5+ stretch target
+- Judge: `claude-haiku-4-5-20251001` (Claude Haiku 4.5) — distinct from any
+  evaluated target
+- Cross-judge: `claude-sonnet-4-6` — independent second judge used to
+  validate every verdict
+
+## Responsible disclosure
+
+If you are a model provider whose model is included and you would like
+findings or example transcripts removed, email
+**rosscyking@gmail.com** with the subject line
+`[redteam-foundry] takedown request`.
+
+**Removal commitment: 24 hours from receipt.**
+
+## Scope and disclaimer
+
+This work is for research and educational purposes. Results are reported as:
+
+> "On benchmark X, with defence stack Y, model Z showed ASR = N% [95% CI ...],
+> judged by Claude Haiku 4.5 and cross-validated by an independent second
+> judge (Claude Sonnet 4.6)."
+
+Results **do not** imply that any model is "safe" or "unsafe" in general.
+See `METHODOLOGY.md` for how each metric is validated, and which metrics are
+reported as descriptive only.
+
+Do not apply these prompts against systems you do not own or have explicit
+permission to test.
+
+## Provider terms of service
+
+Both Anthropic's and OpenAI's usage policies explicitly permit safety
+evaluation and red-teaming research. We comply with:
+
+- Anthropic Usage Policy (verified 2026-05-05)
+- API budget caps set in the provider console (hard limit) **and** in the
+  harness itself (per-run, per-call) — see `src/redteam/budget.py`.
+
+## Maintainer
+
+Ross — rosscyking@gmail.com

redteam_foundry-0.2.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Ross (rosscyking@gmail.com)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.