simpleaudit-studio 0.3.0__tar.gz → 0.4.0__tar.gz

{simpleaudit_studio-0.3.0 → simpleaudit_studio-0.4.0}/.env.example

@@ -76,8 +76,9 @@ WORKER_POOL=cpu
 # against an unmerged revision, create a gitignored simpleaudit-dependency.yaml
 # at the project root with a single `git_ref:` key — do NOT edit pyproject.toml.
 
-# --- Model API credentials (resolved at execution time, never stored) ----------
-# Add one variable per model endpoint's `secret_reference`, e.g.:
+# --- Model API keys (optional) ---------------------------------------------------
+# Keys can be stored on a connection in the Models page. For connections set to
+# "Environment variable", add that variable here (read when a run starts), e.g.:
 # SIMULACHAT_API_KEY=sk-...
 
 # --- Demo audit seeding ---------------------------------------------------------

{simpleaudit_studio-0.3.0 → simpleaudit_studio-0.4.0}/.env.local.example

@@ -1,17 +1,14 @@
 # =============================================================================
-# SimpleAudit Platform — LOCAL (non-Docker) development environment
+# SimpleAudit Studio — local development
 #
-# Copy to `.env` and edit. `manage.py` loads `.env` automatically.
+# Copy to `.env` (manage.py loads it automatically), then:
 #
-# This setup runs the web server and worker as plain processes on your machine
-# (SQLite for the domain DB, Hatchet + Postgres via Docker for the queue).
-# It is for development only — the canonical deployment is Docker Compose
-# (see `.env.example`).
+#   uv sync --extra dev
+#   uv run manage.py setup_local              # first time: + seed data
+#   uv run manage.py dev_server --embedded    # daily
 #
-# Prerequisites:
-#   1. A Python 3.12+ virtualenv with `uv sync --extra dev --extra postgres`
-#   2. Docker, for Postgres + Hatchet (the "partial Docker" pattern below)
-#   3. A SimpleAudit engine checkout (for the worker to execute audits)
+# SQLite holds the app data and `--embedded` starts its own Hatchet queue, so
+# no Docker is needed. For production use Docker Compose with `.env.example`.
 # =============================================================================
 
 # --- Django -----------------------------------------------------------------
@@ -40,23 +37,10 @@ POSTGRES_HOST=localhost
 POSTGRES_PORT=5432
 
 # --- Hatchet (durable job queue) ----------------------------------------------
-# Run Postgres + Hatchet via Docker ("partial Docker" pattern):
-#
-#   docker compose up -d postgres hatchet-server
-#
-# That gives you:
-#   - Postgres on localhost:5432 (Hatchet's own queue DB, created by
-#     deploy/postgres-init.sql on first init)
-#   - Hatchet HTTP API on localhost:8888 (dashboard at http://localhost:8888)
-#   - Hatchet gRPC on localhost:7077
-#
-# The auth-disabled hatchet-lite-dev image writes its worker JWT to its config
-# volume. To use it from a local (non-container) worker, copy the token out:
-#
+# Not used with `dev_server --embedded`, which starts its own engine.
+# Only for plain `dev_server` against an external Hatchet, e.g.
+# `docker compose up -d postgres hatchet-server`, then copy its worker token:
 #   docker compose cp hatchet-server:/config/authdisabled-token ./hatchet-token
-#
-# and point HATCHET_TOKEN_FILE at it (below). Alternatively set HATCHET_API_KEY
-# directly to the token's contents.
 HATCHET_SERVER_URL=http://localhost:8888
 HATCHET_GRPC_URL=localhost:7077
 HATCHET_API_KEY=
@@ -75,8 +59,9 @@ WORKER_POOL=cpu
 # That value is recorded as the run's git_commit provenance. Do NOT edit
 # pyproject.toml for a dev override.
 
-# --- Model API credentials (resolved at execution time, never stored) ----------
-# Add one variable per model endpoint's `secret_reference`, e.g.:
+# --- Model API keys (optional) ---------------------------------------------------
+# Keys can be stored on a connection in the Models page. For connections set to
+# "Environment variable", add that variable here (read when a run starts), e.g.:
 # SIMULACHAT_API_KEY=sk-...
 
 # --- Optional (leave empty to disable) ------------------------------------------

{simpleaudit_studio-0.3.0 → simpleaudit_studio-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: simpleaudit-studio
-Version: 0.3.0
+Version: 0.4.0
 Summary: Production AI model audit platform — one-liner demo mode via uvx
 License: MIT
 License-File: LICENSE
@@ -15,7 +15,7 @@ Requires-Dist: pydantic<3,>=2.7
 Requires-Dist: python-dotenv==1.1.1
 Requires-Dist: requests==2.32.5
 Requires-Dist: sentry-sdk[django]>=2.0
-Requires-Dist: simpleaudit==0.2.1
+Requires-Dist: simpleaudit==0.2.3
 Requires-Dist: workos==10.4.0
 Provides-Extra: dev
 Requires-Dist: factory-boy==3.3.2; extra == 'dev'
@@ -82,37 +82,45 @@ Set the API key in the connection form (leave blank for local servers that don't
 
 ## 🛠️ Local Development
 
-For contributors working on the codebase, use [uv](https://docs.astral.sh/uv/) — no manual venv steps, no Makefile. Two verbs cover everything:
+Needs Python 3.11+ and [uv](https://docs.astral.sh/uv/). No Docker, no Makefile.
 
-- **`uv sync`** — create/update `.venv` from `pyproject.toml` + `uv.lock`
-- **`uv run <cmd>`** — run any command inside `.venv` (auto-syncs first if the lockfile changed)
-
-### Setup (one time)
+### Setup (once)
 
 ```bash
 git clone https://github.com/SushantGautam/SimpleAuditStudio
 cd SimpleAuditStudio
-cp .env.local.example .env          # minimal local config (SQLite + mock models)
+cp .env.local.example .env        # SQLite + embedded queue; admin is studio / BOOTSTRAP_PASSWORD
+uv sync --extra dev               # create .venv with app + dev tools
+uv run manage.py setup_local      # migrate, create admin + workspace, seed scenario packs & models
+```
 
-uv sync --extra dev --extra postgres   # create .venv + install all deps
-uv run manage.py setup_local           # migrate + create admin (studio / BOOTSTRAP_PASSWORD from .env) + seed data
+### Daily work
+
+```bash
+uv run manage.py dev_server --embedded    # web UI + API + worker + embedded Hatchet → http://localhost:8000
 ```
 
-`setup_local` chains the three idempotent first-run steps (migrate → bootstrap admin → seed scenario packs & model connections) into one command. Re-run it any time — it skips what already exists.
+Every start applies migrations and makes sure the admin (a superuser) and default workspace exist, so pulling new code needs no extra steps. Keep in mind:
 
-### Daily work
+- Only one `dev_server --embedded` (or `uvx simpleaudit-studio`) can run at a time: they share the embedded queue database.
+- The web server reloads on code changes; the worker does not. Restart `dev_server` after changing `infra/worker.py` or `infra/engine.py`.
+
+### Tests and lint
 
 ```bash
-uv run manage.py dev_server --embedded   # easiest: web UI + API + audit worker, zero-Docker → http://localhost:8000
+SIMPLEAUDIT_LOCAL_SQLITE=1 uv run manage.py test infra --exclude-tag embedded_hatchet
+uv run ruff check .
 ```
 
-Other options:
+### Other setups
+
 ```bash
-uv run manage.py runserver               # web UI + API only (no worker)
-uv run manage.py dev_server              # web UI + worker, uses .env (edit POSTGRES_*/HATCHET_* to point at your infra)
-SIMPLEAUDIT_LOCAL_SQLITE=1 uv run manage.py test infra   # tests
+uv run manage.py dev_server --no-worker   # web UI + API only
+uv run manage.py dev_server               # use the Postgres + Hatchet configured in .env (see .env.example)
 ```
 
+See [docs/deployment.md](docs/deployment.md) for every environment variable and management command.
+
 ## 🐳 Self-Hosting
 
 For teams or multi-user setups, use Docker Compose:
@@ -132,6 +140,8 @@ See [docs/deployment.md](docs/deployment.md) for production hardening, backups,
 ## ✨ What You Can Do
 
 - Build versioned scenario sets and register OpenAI-compatible models
+- Define versioned judges: start from a SimpleAudit judge (safety, harm, helpfulness, factuality, abstention, checklist, …) and edit its criteria, or write your own criteria as a severity, 1–10 score (with your own dimensions) or yes/no judge; run with any judge model
+- Test your deployment's target system prompt, and compare prompts or judges side by side
 - Run experiments across models, scenario versions and settings, with a review step before launch
 - Watch runs live, including per-repetition results
 - Monitor models for drift on a schedule (interval or cron)

{simpleaudit_studio-0.3.0 → simpleaudit_studio-0.4.0}/README.md

@@ -52,37 +52,45 @@ Set the API key in the connection form (leave blank for local servers that don't
 
 ## 🛠️ Local Development
 
-For contributors working on the codebase, use [uv](https://docs.astral.sh/uv/) — no manual venv steps, no Makefile. Two verbs cover everything:
+Needs Python 3.11+ and [uv](https://docs.astral.sh/uv/). No Docker, no Makefile.
 
-- **`uv sync`** — create/update `.venv` from `pyproject.toml` + `uv.lock`
-- **`uv run <cmd>`** — run any command inside `.venv` (auto-syncs first if the lockfile changed)
-
-### Setup (one time)
+### Setup (once)
 
 ```bash
 git clone https://github.com/SushantGautam/SimpleAuditStudio
 cd SimpleAuditStudio
-cp .env.local.example .env          # minimal local config (SQLite + mock models)
+cp .env.local.example .env        # SQLite + embedded queue; admin is studio / BOOTSTRAP_PASSWORD
+uv sync --extra dev               # create .venv with app + dev tools
+uv run manage.py setup_local      # migrate, create admin + workspace, seed scenario packs & models
+```
 
-uv sync --extra dev --extra postgres   # create .venv + install all deps
-uv run manage.py setup_local           # migrate + create admin (studio / BOOTSTRAP_PASSWORD from .env) + seed data
+### Daily work
+
+```bash
+uv run manage.py dev_server --embedded    # web UI + API + worker + embedded Hatchet → http://localhost:8000
 ```
 
-`setup_local` chains the three idempotent first-run steps (migrate → bootstrap admin → seed scenario packs & model connections) into one command. Re-run it any time — it skips what already exists.
+Every start applies migrations and makes sure the admin (a superuser) and default workspace exist, so pulling new code needs no extra steps. Keep in mind:
 
-### Daily work
+- Only one `dev_server --embedded` (or `uvx simpleaudit-studio`) can run at a time: they share the embedded queue database.
+- The web server reloads on code changes; the worker does not. Restart `dev_server` after changing `infra/worker.py` or `infra/engine.py`.
+
+### Tests and lint
 
 ```bash
-uv run manage.py dev_server --embedded   # easiest: web UI + API + audit worker, zero-Docker → http://localhost:8000
+SIMPLEAUDIT_LOCAL_SQLITE=1 uv run manage.py test infra --exclude-tag embedded_hatchet
+uv run ruff check .
 ```
 
-Other options:
+### Other setups
+
 ```bash
-uv run manage.py runserver               # web UI + API only (no worker)
-uv run manage.py dev_server              # web UI + worker, uses .env (edit POSTGRES_*/HATCHET_* to point at your infra)
-SIMPLEAUDIT_LOCAL_SQLITE=1 uv run manage.py test infra   # tests
+uv run manage.py dev_server --no-worker   # web UI + API only
+uv run manage.py dev_server               # use the Postgres + Hatchet configured in .env (see .env.example)
 ```
 
+See [docs/deployment.md](docs/deployment.md) for every environment variable and management command.
+
 ## 🐳 Self-Hosting
 
 For teams or multi-user setups, use Docker Compose:
@@ -102,6 +110,8 @@ See [docs/deployment.md](docs/deployment.md) for production hardening, backups,
 ## ✨ What You Can Do
 
 - Build versioned scenario sets and register OpenAI-compatible models
+- Define versioned judges: start from a SimpleAudit judge (safety, harm, helpfulness, factuality, abstention, checklist, …) and edit its criteria, or write your own criteria as a severity, 1–10 score (with your own dimensions) or yes/no judge; run with any judge model
+- Test your deployment's target system prompt, and compare prompts or judges side by side
 - Run experiments across models, scenario versions and settings, with a review step before launch
 - Watch runs live, including per-repetition results
 - Monitor models for drift on a schedule (interval or cron)

{simpleaudit_studio-0.3.0 → simpleaudit_studio-0.4.0}/accounts/services.py

@@ -68,6 +68,10 @@ def bootstrap_admin_and_default_project(
         user=user,
         defaults={"role": ProjectMembership.Role.ADMIN},
     )
+    if created:
+        from infra.seed import seed_workspace
+
+        seed_workspace(project, user)   # standard scenario sets and SimpleAudit's judges
     return user, project
 
 
@@ -140,11 +144,14 @@ def create_workspace(*, user, name: str, description: str = "") -> Project:
             with transaction.atomic():
                 project = Project.objects.create(name=clean_name, slug=candidate, description=(description or "").strip())
                 ProjectMembership.objects.create(project=project, user=user, role=ProjectMembership.Role.ADMIN)
-            return project
         except IntegrityError:
             if attempt == 4:
                 raise StableAPIError(detail="A workspace with this name already exists.", code="workspace_name_conflict", http_status=409)
             continue
+        from infra.seed import seed_workspace
+
+        seed_workspace(project, user)   # standard scenario sets and SimpleAudit's judges
+        return project
     raise StableAPIError(detail="A workspace with this name already exists.", code="workspace_name_conflict", http_status=409)
 
 

{simpleaudit_studio-0.3.0 → simpleaudit_studio-0.4.0}/audits/comparison.py

@@ -10,6 +10,7 @@ from __future__ import annotations
 
 from audits.events import ScenarioResult
 from audits.models import AuditRun
+from audits.services import frozen_judge, frozen_name
 
 
 class ComparisonIncompatible(Exception):
@@ -46,15 +47,15 @@ def compare_runs(project, run_ids: list[int]) -> dict:
         warnings.append(f"Scenario set versions differ: {', '.join('v'+str(v) for v in vnums)}. Results may not be directly comparable.")
 
     # Check: same judge?
-    judge_ids = {r.judge_model_id for r in runs}
+    judge_ids = {r.judge_version_id for r in runs}
     if len(judge_ids) > 1:
-        judge_names = sorted({r.judge_model.display_name for r in runs})
-        warnings.append(f"Different judge models used: {', '.join(judge_names)}. Judge effects can dominate model effects — interpret with caution.")
+        judge_names = sorted({f"{frozen_judge(r)['label']} ({frozen_name(r, 'judge')})" for r in runs})
+        warnings.append(f"Different judges used: {', '.join(judge_names)}. Judge effects can dominate model effects — interpret with caution.")
 
     # Check: same auditor?
     auditor_ids = {r.auditor_model_id for r in runs}
     if len(auditor_ids) > 1:
-        auditor_names = sorted({r.auditor_model.display_name for r in runs})
+        auditor_names = sorted({frozen_name(r, "auditor", with_id=True) for r in runs})
         warnings.append(f"Different auditor models used: {', '.join(auditor_names)}.")
 
     # Check: same SimpleAudit version?
@@ -119,7 +120,7 @@ def compare_runs(project, run_ids: list[int]) -> dict:
             r = run_results.get(run.id, {}).get(key)
             entry["runs"][str(run.id)] = {
                 "name": run.name,
-                "target": run.target_model.display_name if run.target_model else None,
+                "target": frozen_name(run, "target"),
                 **(r or {"status": "missing", "severity": None}),
             }
         results.append(entry)
@@ -127,10 +128,13 @@ def compare_runs(project, run_ids: list[int]) -> dict:
     # Build inputs comparison: key parameters that differ between runs
     input_rows = []
     param_defs = [
-        ("Target model", lambda r: r.target_model.display_name if r.target_model else "—"),
-        ("Auditor model", lambda r: r.auditor_model.display_name if r.auditor_model else "—"),
-        ("Judge model", lambda r: r.judge_model.display_name if r.judge_model else "—"),
-        ("Scenario set version", lambda r: f"{r.scenario_set_version.scenario_set.name} (v{r.scenario_set_version.version})" if r.scenario_set_version else "—"),
+        ("Target model", lambda r: frozen_name(r, "target", with_id=True)),
+        ("Auditor model", lambda r: frozen_name(r, "auditor", with_id=True)),
+        ("Judge", lambda r: frozen_judge(r)["label"]),
+        ("Judge format", lambda r: frozen_judge(r)["output_label"]),
+        ("Judge model", lambda r: frozen_name(r, "judge", with_id=True)),
+        ("System prompt", lambda r: (r.generation_parameters_snapshot or {}).get("system_prompt") or "none"),
+        ("Scenario set version", lambda r: r.scenario_set_version.label if r.scenario_set_version else "—"),
         ("SimpleAudit version", lambda r: f"{r.simpleaudit_version} ({r.git_commit[:8]}…)" if r.simpleaudit_version and r.git_commit else (r.simpleaudit_version or "—")),
         ("Temperature (target)", lambda r: (r.generation_parameters_snapshot or {}).get("temperature_target", "—")),
         ("Temperature (auditor)", lambda r: (r.generation_parameters_snapshot or {}).get("temperature_auditor", "—")),
@@ -155,6 +159,8 @@ def compare_runs(project, run_ids: list[int]) -> dict:
                 url = f"/models/#conn-{r.target_model.connection_id}"
             elif label == "Auditor model" and r.auditor_model_id:
                 url = f"/models/#conn-{r.auditor_model.connection_id}"
+            elif label == "Judge" and frozen_judge(r)["judge_id"]:
+                url = f"/judges/{frozen_judge(r)['judge_id']}/?v={frozen_judge(r)['version']}"
             elif label == "Judge model" and r.judge_model_id:
                 url = f"/models/#conn-{r.judge_model.connection_id}"
             elif label == "Scenario set version" and r.scenario_set_version:
@@ -171,8 +177,8 @@ def compare_runs(project, run_ids: list[int]) -> dict:
                 "id": r.id,
                 "name": r.name,
                 "status": r.status,
-                "target": r.target_model.display_name if r.target_model else None,
-                "judge": r.judge_model.display_name if r.judge_model else None,
+                "target": frozen_name(r, "target"),
+                "judge": frozen_judge(r)["label"],
                 "scenario_set_version": f"v{r.scenario_set_version.version}" if r.scenario_set_version else None,
                 "total_scenarios": r.total_scenarios,
                 "successful_scenarios": r.successful_scenarios,

{simpleaudit_studio-0.3.0 → simpleaudit_studio-0.4.0}/audits/experiments.py

@@ -15,6 +15,7 @@ from django.db import transaction
 
 from audits.models import AuditRun, Experiment
 from audits.monitors import pass_counts, wilson
+from audits.services import frozen_judge, frozen_name
 
 MAX_RUNS_PER_EXPERIMENT = 50
 
@@ -23,15 +24,22 @@ DESIGN_AXES = {
     "scenario_set": "Scenario set",
     "target": "Target",
     "auditor": "Auditor",
+    "judge_model": "Judge model",
     "judge": "Judge",
     "max_turns": "Max turns",
     "language": "Language",
 }
 # Everything an experiment can compare: the design axes plus per-run edits
-# made on the review screen (repetitions, generation config).
-FACTORS = {**DESIGN_AXES, "n_repetitions": "Repetitions", "params": "Parameters"}
-_MODEL_ROLES = ("target", "auditor", "judge")
-_FORM_KEYS = ("max_turns", "language", "n_repetitions")
+# made on the review screen (repetitions, system prompt, generation config).
+FACTORS = {**DESIGN_AXES, "n_repetitions": "Repetitions", "system_prompt": "System prompt", "params": "Parameters"}
+# Axes whose values are registered models (axis -> form field and label);
+# "judge" values are JudgeVersions.
+_MODEL_AXES = {"target": ("target_model", "target"), "auditor": ("auditor_model", "auditor"),
+               "judge_model": ("judge_model", "judge")}
+_MODEL_ROLES = tuple(_MODEL_AXES)
+_ROLE_AXES = ("target", "auditor", "judge_model", "judge")
+# Generation keys with their own form fields (not "parameters").
+_FORM_KEYS = ("max_turns", "language", "n_repetitions", "system_prompt")
 
 
 class DesignError(ValueError):
@@ -66,6 +74,7 @@ def parse_design(post, project) -> dict:
 
     Raises DesignError with a user-facing message.
     """
+    from judges.models import Judge, JudgeVersion
     from model_registry.models import RegisteredModel
     from scenarios.models import ScenarioSet, ScenarioSetVersion
 
@@ -106,15 +115,45 @@ def parse_design(post, project) -> dict:
     )
 
     models = {}
-    for role in _MODEL_ROLES:
-        ids = [m for m in post.getlist(f"{role}_model") if m]
+    for axis, (field, label) in _MODEL_AXES.items():
+        ids = [m for m in post.getlist(field) if m]
         if not ids:
-            raise DesignError(f"Pick at least one {role} model.")
-        found = {str(m.pk): m for m in RegisteredModel.objects.filter(project=project, pk__in=ids).select_related("connection")}
+            raise DesignError(f"Pick at least one {label} model.")
+        # A model is usable when it belongs to this workspace OR its connection
+        # is shared into this workspace (public / admin-shared / explicit).
+        from model_registry.services import visible_connection_ids_for
+
+        allowed_conn_ids = set(visible_connection_ids_for(project))
+        found = {
+            str(m.pk): m
+            for m in RegisteredModel.objects.filter(pk__in=ids).select_related("connection")
+            if m.project_id == project.id or m.connection_id in allowed_conn_ids
+        }
         missing = [i for i in ids if i not in found]
         if missing:
-            raise DesignError(f"A selected {role} model is not in this workspace.")
-        models[role] = [found[i] for i in dict.fromkeys(ids)]
+            raise DesignError(f"A selected {label} model isn't available in this workspace.")
+        models[axis] = [found[i] for i in dict.fromkeys(ids)]
+
+    # Judges: version ids, and/or "latest:<judge id>" (always latest: unpinned,
+    # so repeating runs follow new versions).
+    raw = [v for v in dict.fromkeys(post.getlist("judge")) if v]
+    if not raw:
+        raise DesignError("Pick at least one judge.")
+    follow_ids = [v.split(":", 1)[1] for v in raw if v.startswith("latest:")]
+    pinned_ids = [v for v in raw if not v.startswith("latest:")]
+    judges = list(
+        JudgeVersion.objects.filter(pk__in=pinned_ids, judge__project=project).select_related("judge")
+    )
+    follow = list(Judge.objects.filter(pk__in=follow_ids, project=project))
+    if len(judges) != len(pinned_ids) or len(follow) != len(follow_ids):
+        raise DesignError("A selected judge is not in this workspace.")
+    for judge in follow:
+        latest = judge.versions.select_related("judge").order_by("-version").first()
+        if latest is None:
+            raise DesignError(f"Judge “{judge.name}” has no version.")
+        latest.follow_latest = True
+        judges.append(latest)
+    judges.sort(key=lambda v: (v.judge.name, is_follow(v), v.version))
 
     n_reps_raw = (post.get("n_repetitions") or "").strip()
     try:
@@ -140,11 +179,18 @@ def parse_design(post, project) -> dict:
         raise DesignError(f"Max turns: {exc} Use whole numbers from 1 to 50.") from exc
     languages = _split_values(post.get("language", ""), str)
 
+    # The target's system prompt lives in the generation config (the engine
+    # reads it there); the field wins over a key typed into the JSON.
+    system_prompt = (post.get("system_prompt") or "").strip()
+    if system_prompt:
+        gen_config = {**(gen_config or {}), "system_prompt": system_prompt}
+
     return {
         "scenario_set": versions,
         "target": models["target"],
         "auditor": models["auditor"],
-        "judge": models["judge"],
+        "judge_model": models["judge_model"],
+        "judge": judges,
         "max_turns": max_turns,
         "language": languages,
         "n_repetitions": n_reps if n_reps and n_reps > 1 else None,
@@ -153,7 +199,7 @@ def parse_design(post, project) -> dict:
 
 
 def is_follow(version) -> bool:
-    """Whether this scenario set version was picked as "Always latest"."""
+    """Whether this scenario set / judge version was picked as "Always latest"."""
     return bool(getattr(version, "follow_latest", False))
 
 
@@ -184,13 +230,34 @@ def expand(design: dict) -> list[dict]:
     return specs
 
 
+def spec_to_run(spec: dict, name: str) -> dict:
+    """A design spec (from ``expand``) as a planned run: the ``launch_experiment``
+    run item shape, also used by script generation (infra.codegen)."""
+    return {
+        "name": name,
+        "version": spec["scenario_set"],
+        "target": spec["target"],
+        "auditor": spec["auditor"],
+        "judge_model": spec["judge_model"],
+        "judge": spec["judge"],
+        "max_turns": spec["max_turns"],
+        "language": spec["language"],
+        "n_repetitions": spec["n_repetitions"],
+        "gen_config": spec["gen_config"],
+    }
+
+
 def factor_value_label(key: str, value) -> str:
     if value is None:
         return "default"
     if key == "scenario_set":
         if is_follow(value):
             return f"{value.scenario_set.name} (always latest, now v{value.version})"
-        return f"{value.scenario_set.name} v{value.version}"
+        return value.label
+    if key == "judge":
+        if is_follow(value):
+            return f"{value.judge.name} (always latest, now v{value.version})"
+        return value.label
     if key in _MODEL_ROLES:
         return value.display_name
     return str(value)
@@ -203,17 +270,19 @@ def spec_label(spec: dict, factors: list[str]) -> str:
 
 def spec_warnings(spec: dict) -> list[str]:
     warnings = []
-    if spec["judge"].pk == spec["target"].pk:
-        warnings.append("Judge is the target model: it grades itself.")
-    for role in _MODEL_ROLES:
-        if not spec[role].has_key:
-            warnings.append(f"{role.title()} model has no API key set.")
+    if spec["judge_model"].pk == spec["target"].pk:
+        warnings.append("Judge model is the target model: it grades itself.")
+    for role, model in (("Target", spec["target"]), ("Auditor", spec["auditor"]), ("Judge", spec["judge_model"])):
+        if not model.has_key:
+            warnings.append(f"{role} model has no API key set.")
+        elif not (model.enabled and model.connection.enabled):
+            warnings.append(f"{role} model is disabled.")
     return warnings
 
 
 def design_warnings(factors: list[str]) -> list[str]:
     """Experiment-level cautions about how results can be read."""
-    roles = [f for f in factors if f in _MODEL_ROLES]
+    roles = [f for f in factors if f in _ROLE_AXES]
     warnings = []
     if "target" in roles and len(roles) > 1:
         others = " and ".join(FACTORS[r].lower() for r in roles if r != "target")
@@ -238,6 +307,14 @@ def params_label(params: dict | None) -> str:
     return json.dumps(rest, sort_keys=True, separators=(", ", ": ")) if rest else "default"
 
 
+def prompt_label(prompt: str | None) -> str:
+    """Short text for a system prompt in tables: its first line, clipped."""
+    first = (prompt or "").strip().splitlines()[0] if (prompt or "").strip() else ""
+    if not first:
+        return "none"
+    return first if len(first) <= 48 else first[:47] + "…"
+
+
 def factors_from_runs(runs: list[dict]) -> list[str]:
     """Which inputs actually differ across the runs about to be launched.
 
@@ -247,8 +324,12 @@ def factors_from_runs(runs: list[dict]) -> list[str]:
     def value(run, key):
         if key == "scenario_set":
             return (run["version"].pk, is_follow(run["version"]))
+        if key == "judge":
+            return (run["judge"].pk, is_follow(run["judge"]))
         if key in _MODEL_ROLES:
             return run[key].pk
+        if key == "system_prompt":
+            return (run["gen_config"] or {}).get("system_prompt") or ""
         if key == "params":
             return params_label(run["gen_config"])
         return run[key]
@@ -263,14 +344,16 @@ def spec_to_row(spec: dict) -> str:
         "f": 1 if is_follow(spec["scenario_set"]) else 0,
         "t": spec["target"].pk,
         "a": spec["auditor"].pk,
+        "jm": spec["judge_model"].pk,
         "j": spec["judge"].pk,
+        "jf": 1 if is_follow(spec["judge"]) else 0,
     })
 
 
 def launch_experiment(*, project, user, name: str, runs: list[dict], repeat: dict | None = None) -> Experiment:
     """Create the Experiment and its frozen runs, then submit them.
 
-    ``runs`` items: {"name", "version", "target", "auditor", "judge",
+    ``runs`` items: {"name", "version", "target", "auditor", "judge_model", "judge",
     "max_turns", "language", "n_repetitions", "gen_config"}. With ``repeat``
     (see audits.monitors.parse_repeat) each run setup also gets a Monitor
     linked to the experiment; runs start now unless repeat["start"] is "at".
@@ -302,7 +385,8 @@ def launch_experiment(*, project, user, name: str, runs: list[dict], repeat: dic
                 scenario_set_version=r["version"],
                 target_model=r["target"],
                 auditor_model=r["auditor"],
-                judge_model=r["judge"],
+                judge_model=r["judge_model"],
+                judge=r["judge"],
                 max_turns_override=r["max_turns"],
                 language_override=r["language"],
                 n_repetitions_override=r["n_repetitions"],
@@ -334,13 +418,15 @@ def launch_experiment(*, project, user, name: str, runs: list[dict], repeat: dic
 def run_factor_values(run: AuditRun) -> dict:
     params = run.generation_parameters_snapshot or {}
     return {
-        "scenario_set": f"{run.scenario_set_version.scenario_set.name} v{run.scenario_set_version.version}",
-        "target": run.target_model.display_name,
-        "auditor": run.auditor_model.display_name,
-        "judge": run.judge_model.display_name,
+        "scenario_set": run.scenario_set_version.label,
+        "target": frozen_name(run, "target"),
+        "auditor": frozen_name(run, "auditor"),
+        "judge_model": frozen_name(run, "judge"),
+        "judge": frozen_judge(run)["label"],
         "max_turns": str(params.get("max_turns") or "default"),
         "language": params.get("language") or "default",
         "n_repetitions": f"{params.get('n_repetitions') or 1}×",
+        "system_prompt": prompt_label(params.get("system_prompt")),
         "params": params_label(params),
     }