argus-proof 0.2.2__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- argus_proof-0.3.0/.cursor/rules/agent-coding-packs.mdc +33 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/.env.example +15 -2
- {argus_proof-0.2.2 → argus_proof-0.3.0}/.github/workflows/ci.yml +2 -2
- {argus_proof-0.2.2 → argus_proof-0.3.0}/.github/workflows/release.yml +52 -3
- argus_proof-0.3.0/.github/workflows/suite.yml +11 -0
- argus_proof-0.3.0/AGENTS.md +28 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/CHANGELOG.md +68 -1
- {argus_proof-0.2.2 → argus_proof-0.3.0}/CLAUDE.md +15 -1
- {argus_proof-0.2.2 → argus_proof-0.3.0}/PKG-INFO +107 -15
- {argus_proof-0.2.2 → argus_proof-0.3.0}/README.md +105 -13
- {argus_proof-0.2.2 → argus_proof-0.3.0}/pyproject.toml +2 -1
- {argus_proof-0.2.2 → argus_proof-0.3.0}/schema/proof-wire.schema.json +231 -16
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/_version.py +2 -2
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/backends/a1111.py +4 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/backends/comfyui.py +17 -3
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/backends/diffusers.py +3 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/backends/workflow.py +83 -7
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/cli.py +28 -5
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/evaluate.py +5 -1
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/models.py +115 -2
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/moderation.py +175 -16
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/reports.py +23 -22
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/server/app.py +237 -17
- argus_proof-0.3.0/src/argus_proof/templates/comfyui_flux_lora.json +125 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/tests/test_a1111_backend.py +9 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/tests/test_backends.py +30 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/tests/test_diffusers_backend.py +13 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/tests/test_moderation.py +116 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/tests/test_reports.py +16 -0
- argus_proof-0.3.0/tests/test_schema.py +160 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/tests/test_server.py +218 -0
- argus_proof-0.3.0/tests/test_workflow.py +193 -0
- argus_proof-0.2.2/tests/test_workflow.py +0 -100
- {argus_proof-0.2.2 → argus_proof-0.3.0}/.copier-answers.yml +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/.gitignore +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/Dockerfile +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/LICENSE +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/Makefile +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/__init__.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/acceptance.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/archive.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/backends/__init__.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/backends/base.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/backends/http.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/backends/pnginfo.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/backends/remote.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/crossrun.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/experiment.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/explore.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/grid.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/hashing.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/py.typed +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/recommend.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/refinement.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/scoring/__init__.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/scoring/base.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/scoring/gate.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/scoring/orchestrator.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/scoring/scorers/__init__.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/scoring/scorers/_util.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/scoring/scorers/identity.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/scoring/scorers/phash.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/scoring/scorers/quality.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/scoring/scorers/safety.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/scoring/summary.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/server/__init__.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/stats.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/src/argus_proof/templates/comfyui_sdxl_lora.json +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/tests/fakebackend.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/tests/pnghelpers.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/tests/test_acceptance.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/tests/test_archive.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/tests/test_cli.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/tests/test_crossrun.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/tests/test_evaluate.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/tests/test_experiment.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/tests/test_explore.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/tests/test_grid.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/tests/test_models.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/tests/test_pnginfo.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/tests/test_recommend.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/tests/test_refinement.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/tests/test_remote_backend.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/tests/test_scorer_util.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/tests/test_scorers_identity.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/tests/test_scorers_phash.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/tests/test_scorers_quality.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/tests/test_scorers_safety.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/tests/test_scoring.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/tests/test_smoke.py +0 -0
- {argus_proof-0.2.2 → argus_proof-0.3.0}/tests/test_stats.py +0 -0
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
---
|
|
2
|
+
description: Fetch ai-code-auditor coding packs for argus-proof before changing code
|
|
3
|
+
alwaysApply: true
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
<!-- ai-audit:agent-packs:start -->
|
|
7
|
+
# Agent coding packs
|
|
8
|
+
|
|
9
|
+
Do not wait for a pasted brief — fetch the audit packs for `argus-proof` yourself.
|
|
10
|
+
|
|
11
|
+
The audit API base URL is `$AI_AUDIT_PUBLIC_BASE_URL`. The real host/IP is in the local `.env`
|
|
12
|
+
(see `.env.example`, placeholder `http://<AUDIT_HOST>:8765`); read it there, don't guess.
|
|
13
|
+
|
|
14
|
+
1. **Bundle (one call):** `GET $AI_AUDIT_PUBLIC_BASE_URL/audit/packs/bundle?repo=argus-proof`
|
|
15
|
+
(world model + ranked specialist packs + token-capped `briefs_markdown`;
|
|
16
|
+
optional `&token_budget=2500&q=<keyword>&limit_specialists=5`).
|
|
17
|
+
CLI equivalent: `ai-audit packs --bundle --repo argus-proof`.
|
|
18
|
+
2. **Claim** the specialist pack you pick:
|
|
19
|
+
`POST $AI_AUDIT_PUBLIC_BASE_URL/audit/packs/<id>/status` `{"status":"claimed"}`
|
|
20
|
+
(send `X-API-Token: $AI_AUDIT_API_TOKEN`; the token is in your environment, set for Claude Code
|
|
21
|
+
in `~/.claude/settings.json`. If it is empty or you get a 401, ask the user; don't search env files).
|
|
22
|
+
3. **Fix:** `ai-audit repair --pack-id <id>` (agent-composer repair loop; fetches the pack for you;
|
|
23
|
+
HTTP: `POST http://127.0.0.1:9150/api/repair/run` `{"pack_id":"<id>","mode":"review"}`),
|
|
24
|
+
or read `GET $AI_AUDIT_PUBLIC_BASE_URL/audit/packs/<id>` / `.../brief` and implement it yourself.
|
|
25
|
+
4. **Close:** `POST $AI_AUDIT_PUBLIC_BASE_URL/audit/validate_patch` `{"repo":"argus-proof","patch":"<diff>","pack_id":"<id>"}`
|
|
26
|
+
— a passing run marks the pack `resolved`.
|
|
27
|
+
|
|
28
|
+
**Search** before crawling: `GET $AI_AUDIT_PUBLIC_BASE_URL/audit/search?q=<question>&repo=argus-proof&kind=wiki`
|
|
29
|
+
(wiki sections and packs with links; CLI `ai-audit search "<question>" --repo argus-proof`).
|
|
30
|
+
|
|
31
|
+
World-model constraints (ownership, boundaries, scan branch `dev`) win over guesswork.
|
|
32
|
+
Full flow: ai-code-auditor `AGENTS.md`.
|
|
33
|
+
<!-- ai-audit:agent-packs:end -->
|
|
@@ -18,8 +18,11 @@ PROOF_BACKEND=comfyui
|
|
|
18
18
|
# given PATH-style (dir1:dir2), mirroring ComfyUI's extra_model_paths trees.
|
|
19
19
|
PROOF_MODELS_DIR=~/models
|
|
20
20
|
|
|
21
|
-
# ComfyUI workflow template (API-format graph with $placeholders)
|
|
22
|
-
# packaged
|
|
21
|
+
# ComfyUI workflow template (API-format graph with $placeholders), used for every
|
|
22
|
+
# checkpoint. Unset = the packaged templates, picked by the checkpoint's family
|
|
23
|
+
# folder: checkpoints/flux/... -> comfyui_flux_lora.json, anything else ->
|
|
24
|
+
# comfyui_sdxl_lora.json. The Flux template loads t5xxl_fp8_e4m3fn, clip_l and ae
|
|
25
|
+
# by name from ComfyUI's clip/ and vae/ folders.
|
|
23
26
|
#PROOF_WORKFLOW_TEMPLATE=/path/to/workflow.json
|
|
24
27
|
|
|
25
28
|
# ── Storage ──────────────────────────────────────────────────────────
|
|
@@ -37,6 +40,11 @@ PROOF_MODELS_DIR=~/models
|
|
|
37
40
|
# at the seeded "tape" reports/runs when this is on. GET /health echoes the flag.
|
|
38
41
|
#ARGUS_PROOF_READ_ONLY=1
|
|
39
42
|
|
|
43
|
+
# Dev mode (issue #59): register POST /purge and POST /purge/{run_id}, which
|
|
44
|
+
# delete runs and their reports (body {"confirm": "purge"}). Off by default —
|
|
45
|
+
# reports are the evaluation record. Compose sets it from DEVMODE.
|
|
46
|
+
#ARGUS_PROOF_DEVMODE=1
|
|
47
|
+
|
|
40
48
|
# HTTP backends — the engine's base URL.
|
|
41
49
|
COMFYUI_BASE_URL=http://127.0.0.1:8188
|
|
42
50
|
A1111_BASE_URL=http://127.0.0.1:7860
|
|
@@ -51,3 +59,8 @@ A1111_BASE_URL=http://127.0.0.1:7860
|
|
|
51
59
|
# remote (cloud) backend — the hosted generation endpoint + its bearer token.
|
|
52
60
|
PROOF_REMOTE_URL=https://your-proof-gen.example.com
|
|
53
61
|
PROOF_REMOTE_API_KEY=
|
|
62
|
+
|
|
63
|
+
# ── Agent tooling (not read by this service) ──
|
|
64
|
+
# ai-code-auditor audit API base that AGENTS.md tells coding agents to use.
|
|
65
|
+
# Put the real host/IP in your local .env, never here.
|
|
66
|
+
# AI_AUDIT_PUBLIC_BASE_URL=http://<AUDIT_HOST>:8765
|
|
@@ -6,6 +6,12 @@ name: Release to PyPI
|
|
|
6
6
|
on:
|
|
7
7
|
push:
|
|
8
8
|
tags: ["v*"]
|
|
9
|
+
# Sent by argus-cortex's release workflow. The image installs argus-cortex
|
|
10
|
+
# from the pyproject range at build time, so rebuilding the newest release's
|
|
11
|
+
# image is how a new cortex reaches it. Only the image jobs run: PyPI and the
|
|
12
|
+
# GitHub Release for that tag already exist.
|
|
13
|
+
repository_dispatch:
|
|
14
|
+
types: [cortex-released]
|
|
9
15
|
|
|
10
16
|
permissions:
|
|
11
17
|
contents: read
|
|
@@ -13,6 +19,9 @@ permissions:
|
|
|
13
19
|
|
|
14
20
|
jobs:
|
|
15
21
|
build:
|
|
22
|
+
# Not on a cortex dispatch, so the PyPI and Release jobs that chain off it
|
|
23
|
+
# skip too.
|
|
24
|
+
if: github.event_name == 'push'
|
|
16
25
|
runs-on: ubuntu-latest
|
|
17
26
|
steps:
|
|
18
27
|
- uses: actions/checkout@v4
|
|
@@ -79,15 +88,51 @@ jobs:
|
|
|
79
88
|
path: dist/
|
|
80
89
|
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
81
90
|
|
|
91
|
+
release-tag:
|
|
92
|
+
# The tag the image jobs build: the pushed one, or on a cortex dispatch the
|
|
93
|
+
# newest final release. A pre-release never moved `latest`, so a dispatch
|
|
94
|
+
# does not rebuild one.
|
|
95
|
+
runs-on: ubuntu-latest
|
|
96
|
+
outputs:
|
|
97
|
+
tag: ${{ steps.tag.outputs.tag }}
|
|
98
|
+
sha: ${{ steps.tag.outputs.sha }}
|
|
99
|
+
steps:
|
|
100
|
+
- uses: actions/checkout@v4
|
|
101
|
+
if: github.event_name == 'repository_dispatch'
|
|
102
|
+
- id: tag
|
|
103
|
+
env:
|
|
104
|
+
CORTEX_VERSION: ${{ github.event.client_payload.version }}
|
|
105
|
+
run: |
|
|
106
|
+
if [ "$GITHUB_EVENT_NAME" != repository_dispatch ]; then
|
|
107
|
+
echo "tag=$GITHUB_REF_NAME" >> "$GITHUB_OUTPUT"
|
|
108
|
+
echo "sha=$GITHUB_SHA" >> "$GITHUB_OUTPUT"
|
|
109
|
+
exit 0
|
|
110
|
+
fi
|
|
111
|
+
refs=$(git ls-remote --tags origin 'refs/tags/v*')
|
|
112
|
+
tag=$(echo "$refs" | sed -n 's|.*refs/tags/\(v[0-9]*\.[0-9]*\.[0-9]*\)\(\^{}\)\{0,1\}$|\1|p' | sort -uV | tail -n1)
|
|
113
|
+
if [ -z "$tag" ]; then
|
|
114
|
+
echo "::notice::No release tag yet, so no image to rebuild for cortex $CORTEX_VERSION."
|
|
115
|
+
exit 0
|
|
116
|
+
fi
|
|
117
|
+
# An annotated tag lists its commit on the peeled ^{} line.
|
|
118
|
+
sha=$(echo "$refs" | awk -v t="refs/tags/$tag" '$2 == t {l = $1} $2 == t "^{}" {p = $1} END {print p ? p : l}')
|
|
119
|
+
echo "Rebuilding $tag ($sha) for cortex $CORTEX_VERSION."
|
|
120
|
+
echo "tag=$tag" >> "$GITHUB_OUTPUT"
|
|
121
|
+
echo "sha=$sha" >> "$GITHUB_OUTPUT"
|
|
122
|
+
|
|
82
123
|
publish-image:
|
|
83
124
|
runs-on: ubuntu-latest
|
|
84
|
-
needs: test-install
|
|
125
|
+
needs: [release-tag, test-install]
|
|
126
|
+
# On a cortex dispatch test-install is skipped (nothing was built for PyPI),
|
|
127
|
+
# so the job runs on the tag alone.
|
|
128
|
+
if: "!cancelled() && needs.release-tag.outputs.tag != '' && (needs.test-install.result == 'success' || github.event_name == 'repository_dispatch')"
|
|
85
129
|
permissions:
|
|
86
130
|
contents: read
|
|
87
131
|
packages: write
|
|
88
132
|
steps:
|
|
89
133
|
- uses: actions/checkout@v4
|
|
90
134
|
with:
|
|
135
|
+
ref: ${{ needs.release-tag.outputs.tag }}
|
|
91
136
|
fetch-depth: 0 # hatch-vcs reads the tag from git history
|
|
92
137
|
- uses: docker/setup-buildx-action@v3
|
|
93
138
|
- uses: docker/login-action@v3
|
|
@@ -99,9 +144,10 @@ jobs:
|
|
|
99
144
|
uses: docker/metadata-action@v5
|
|
100
145
|
with:
|
|
101
146
|
images: ghcr.io/${{ github.repository_owner }}/argus-proof
|
|
147
|
+
labels: org.opencontainers.image.revision=${{ needs.release-tag.outputs.sha }}
|
|
102
148
|
tags: |
|
|
103
|
-
type=semver,pattern={{version}}
|
|
104
|
-
type=semver,pattern={{major}}.{{minor}}
|
|
149
|
+
type=semver,pattern={{version}},value=${{ needs.release-tag.outputs.tag }}
|
|
150
|
+
type=semver,pattern={{major}}.{{minor}},value=${{ needs.release-tag.outputs.tag }}
|
|
105
151
|
type=raw,value=latest
|
|
106
152
|
- uses: docker/build-push-action@v6
|
|
107
153
|
with:
|
|
@@ -110,6 +156,9 @@ jobs:
|
|
|
110
156
|
push: true
|
|
111
157
|
tags: ${{ steps.meta.outputs.tags }}
|
|
112
158
|
labels: ${{ steps.meta.outputs.labels }}
|
|
159
|
+
# A dispatch rebuilds an unchanged context, so a cached install layer
|
|
160
|
+
# would keep the old argus-cortex.
|
|
161
|
+
no-cache: ${{ github.event_name == 'repository_dispatch' }}
|
|
113
162
|
cache-from: type=gha
|
|
114
163
|
cache-to: type=gha,mode=max
|
|
115
164
|
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
<!-- ai-audit:agent-packs:start -->
|
|
2
|
+
## Agent coding packs (ai-code-auditor)
|
|
3
|
+
|
|
4
|
+
Do not wait for a pasted brief — fetch the audit packs for `argus-proof` yourself.
|
|
5
|
+
|
|
6
|
+
The audit API base URL is `$AI_AUDIT_PUBLIC_BASE_URL`. The real host/IP is in the local `.env`
|
|
7
|
+
(see `.env.example`, placeholder `http://<AUDIT_HOST>:8765`); read it there, don't guess.
|
|
8
|
+
|
|
9
|
+
1. **Bundle (one call):** `GET $AI_AUDIT_PUBLIC_BASE_URL/audit/packs/bundle?repo=argus-proof`
|
|
10
|
+
(world model + ranked specialist packs + token-capped `briefs_markdown`;
|
|
11
|
+
optional `&token_budget=2500&q=<keyword>&limit_specialists=5`).
|
|
12
|
+
CLI equivalent: `ai-audit packs --bundle --repo argus-proof`.
|
|
13
|
+
2. **Claim** the specialist pack you pick:
|
|
14
|
+
`POST $AI_AUDIT_PUBLIC_BASE_URL/audit/packs/<id>/status` `{"status":"claimed"}`
|
|
15
|
+
(send `X-API-Token: $AI_AUDIT_API_TOKEN`; the token is in your environment, set for Claude Code
|
|
16
|
+
in `~/.claude/settings.json`. If it is empty or you get a 401, ask the user; don't search env files).
|
|
17
|
+
3. **Fix:** `ai-audit repair --pack-id <id>` (agent-composer repair loop; fetches the pack for you;
|
|
18
|
+
HTTP: `POST http://127.0.0.1:9150/api/repair/run` `{"pack_id":"<id>","mode":"review"}`),
|
|
19
|
+
or read `GET $AI_AUDIT_PUBLIC_BASE_URL/audit/packs/<id>` / `.../brief` and implement it yourself.
|
|
20
|
+
4. **Close:** `POST $AI_AUDIT_PUBLIC_BASE_URL/audit/validate_patch` `{"repo":"argus-proof","patch":"<diff>","pack_id":"<id>"}`
|
|
21
|
+
— a passing run marks the pack `resolved`.
|
|
22
|
+
|
|
23
|
+
**Search** before crawling: `GET $AI_AUDIT_PUBLIC_BASE_URL/audit/search?q=<question>&repo=argus-proof&kind=wiki`
|
|
24
|
+
(wiki sections and packs with links; CLI `ai-audit search "<question>" --repo argus-proof`).
|
|
25
|
+
|
|
26
|
+
World-model constraints (ownership, boundaries, scan branch `dev`) win over guesswork.
|
|
27
|
+
Full flow: ai-code-auditor `AGENTS.md`.
|
|
28
|
+
<!-- ai-audit:agent-packs:end -->
|
|
@@ -7,6 +7,72 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.3.0] - 2026-11-15
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
- **A new argus-cortex rebuilds the published image.** `release.yml` also runs on
|
|
14
|
+
the `cortex-released` dispatch argus-cortex's release sends: only the image job runs,
|
|
15
|
+
rebuilding the newest final `v*` tag without cache and republishing its tags, so the
|
|
16
|
+
image picks up the cortex its pyproject range now resolves to.
|
|
17
|
+
- **`GET /report/{run_id}/manifest`** (#57) serves the `RunManifest` behind a
|
|
18
|
+
report (checkpoint, VAE, LoRAs, sampling, prompt, seeds), so studio can prefill
|
|
19
|
+
`/darkroom` from a passed run (argus-studio#125). It is `404` when there is no report,
|
|
20
|
+
`410 Gone` when the report was kept but its run dir was pruned, and `400` for an
|
|
21
|
+
unreadable manifest or one belonging to another run. `GET /health` gains a
|
|
22
|
+
`capabilities` list (`["report_manifest"]`) so a client can detect the route.
|
|
23
|
+
- **Wire schema 1.4** (#58): `ReportSummary` and the `/run/stream` frames join
|
|
24
|
+
`schema/proof-wire.schema.json`. New `ScoringFrame` and `CompleteFrame` models, plus the
|
|
25
|
+
`RunStreamFrame` union discriminated by `type`, describe every NDJSON line, and the server
|
|
26
|
+
builds its frames from them. `ReportSummary` moved to `argus_proof.models` and is still
|
|
27
|
+
importable from `argus_proof.reports`.
|
|
28
|
+
- **Flux LoRAs** (#60): `SamplingParams.guidance` (wire 1.4; same field as darkroom's)
|
|
29
|
+
and a `$guidance` template slot. A packaged `comfyui_flux_lora.json` is picked when
|
|
30
|
+
the base checkpoint sits under `checkpoints/flux/`, with SDXL's template for every
|
|
31
|
+
other family, unless `PROOF_WORKFLOW_TEMPLATE` overrides both. `/run/stream` and
|
|
32
|
+
`argus-proof run` take `guidance`, and leaving out `cfg`/`sampler`/`scheduler`/`guidance`
|
|
33
|
+
gets the family's defaults (Flux: cfg 1, `euler`/`simple`, guidance 3.5).
|
|
34
|
+
- **Devmode purge** (#59): `POST /purge/{run_id}` deletes one run's report and run dir,
|
|
35
|
+
and `POST /purge` deletes all of them. They're registered only with `ARGUS_PROOF_DEVMODE=1`
|
|
36
|
+
(`serve --devmode`) and need a `{"confirm": "purge"}` body. A run still generating gets
|
|
37
|
+
`409`, and read-only mode refuses them. The HITL review and refinement go with the report.
|
|
38
|
+
`/health` gains `devmode`, and lists `purge` in `capabilities` when the routes work.
|
|
39
|
+
`/run/stream` now also refuses (`409`) a `run_id` that is already generating.
|
|
40
|
+
- **Llama Guard 3 moderation adapters** (#67): `LlamaGuardImageDetector`
|
|
41
|
+
(Llama-Guard-3-11B-Vision) and `LlamaGuardTextDetector` (Llama-Guard-3-8B) now run
|
|
42
|
+
the model instead of raising `NotImplementedError`. They lazy-load, place the model
|
|
43
|
+
through cortex's `resolve_device`, and score each flagged category with P(unsafe) at
|
|
44
|
+
the verdict token. `parse_guard_output` reads the verdict. The checkpoints are gated
|
|
45
|
+
(`HF_TOKEN`); the README covers the licenses. `[moderation]` now needs
|
|
46
|
+
`transformers>=4.45`.
|
|
47
|
+
|
|
48
|
+
### Changed
|
|
49
|
+
- **The wire schema lists defaulted fields as required where proof always sends them**
|
|
50
|
+
(#58), like curator's `RESPONSE_ONLY`: the `EvalReport` tree, `RunManifest`, `ReportSummary`,
|
|
51
|
+
the stream frames, `GateResult`, `RejectArchive` and the grid plan. Models a client
|
|
52
|
+
builds (`RunSpec`, the configs, `SamplingParams`, `RejectReason`, `ProgressEvent`) are
|
|
53
|
+
unchanged. Payloads and validation are the same; generated client types get stricter.
|
|
54
|
+
|
|
55
|
+
### Fixed
|
|
56
|
+
- **A run is refused instead of silently dropping a value** (#60) when the template
|
|
57
|
+
has no slot for it: a non-empty negative prompt (`$negative`, which Flux templates
|
|
58
|
+
don't have), `guidance` (`$guidance`), or `clip_skip` above 2 (`$clip_skip`). The
|
|
59
|
+
`diffusers` and `a1111` backends refuse `guidance`. ComfyUI renders validate before
|
|
60
|
+
the first seed is queued.
|
|
61
|
+
- **`clip_skip` 1 no longer selects CLIP's last layer** (#60, as darkroom #19): ComfyUI
|
|
62
|
+
gets `-max(clip_skip, 2)`, so 1 and 2 both mean the penultimate layer SDXL uses.
|
|
63
|
+
`-1` blanked some SDXL checkpoints. A rerun of an older `clip_skip: 1` manifest
|
|
64
|
+
now renders at layer -2.
|
|
65
|
+
- **Moderation no longer reads as safe when a Guard model can't load** (#67).
|
|
66
|
+
`PolicyModerator` re-raises `ModerationError` (a gated checkpoint with no token, an
|
|
67
|
+
unreadable verdict) instead of skipping the detector as flaky. Other per-item errors
|
|
68
|
+
are still skipped.
|
|
69
|
+
- README drift (#65): the policy-moderation section lists the `sexual` category;
|
|
70
|
+
the route list adds `/health`, `/scorers` and `/report/{run_id}/image_at/{index}`;
|
|
71
|
+
the CLI section documents `argus-proof schema [--check]`; the `gate` example is
|
|
72
|
+
copy-pasteable and mentions `--confidence`; the experiment sample cell id is
|
|
73
|
+
corrected; the diffusers limitations and the ImageReward-less default scorer set
|
|
74
|
+
are noted.
|
|
75
|
+
|
|
10
76
|
## [0.2.2] - 2026-10-26
|
|
11
77
|
|
|
12
78
|
### Added
|
|
@@ -87,7 +153,8 @@ First release: the MVP evaluation pipeline for trained LoRAs.
|
|
|
87
153
|
`/run/stream`, report/HITL/refine endpoints, and model and export listings.
|
|
88
154
|
- Container image `ghcr.io/smk762/argus-proof`.
|
|
89
155
|
|
|
90
|
-
[Unreleased]: https://github.com/smk762/argus-proof/compare/v0.
|
|
156
|
+
[Unreleased]: https://github.com/smk762/argus-proof/compare/v0.3.0...HEAD
|
|
157
|
+
[0.3.0]: https://github.com/smk762/argus-proof/compare/v0.2.2...v0.3.0
|
|
91
158
|
[0.2.2]: https://github.com/smk762/argus-proof/compare/v0.2.1...v0.2.2
|
|
92
159
|
[0.2.1]: https://github.com/smk762/argus-proof/compare/v0.2.0...v0.2.1
|
|
93
160
|
[0.2.0]: https://github.com/smk762/argus-proof/compare/v0.1.0...v0.2.0
|
|
@@ -13,6 +13,16 @@ argus-quarry -> argus-curator -> argus-lens -> argus-forge -> your trainer -> ar
|
|
|
13
13
|
|
|
14
14
|
**The forge→proof handoff is a soft join, not an embed.** A `RunManifest` links back to the curator export via `source_manifest` (the `manifest.jsonl`) and, when the LoRA came through the suite, to forge's `training_run_id` (== forge `RunEvent.run_id`). Both are optional — a LoRA can be supplied ad hoc. Proof owns its own `run_id` (the eval run), distinct from the forge training run. README.md is the source of truth for behaviour; it is unusually detailed.
|
|
15
15
|
|
|
16
|
+
## Argus suite
|
|
17
|
+
<!-- argus-suite: keep this block; argus-ci checks for it -->
|
|
18
|
+
This repo is part of the Argus suite. The suite map below lists every sibling and what it owns; check it before
|
|
19
|
+
building something a sibling may already do. **On every release, update this repo's row and capabilities in
|
|
20
|
+
argus-cortex's SUITE.md** (a PR to argus-cortex in the same release).
|
|
21
|
+
|
|
22
|
+
@../argus-core/SUITE.md
|
|
23
|
+
|
|
24
|
+
Not checked out? Read https://github.com/smk762/argus-cortex/blob/main/SUITE.md
|
|
25
|
+
|
|
16
26
|
## Layout
|
|
17
27
|
|
|
18
28
|
`src/argus_proof/`:
|
|
@@ -39,7 +49,7 @@ Run a single test: `uv run --no-sync pytest tests/test_scoring.py::test_name -q`
|
|
|
39
49
|
|
|
40
50
|
## Conventions & gotchas
|
|
41
51
|
|
|
42
|
-
- **The wire schema is checked in CI** (`argus-proof schema --check` runs post-test). If you touch `WIRE_MODELS`/types in `models.py`, regenerate
|
|
52
|
+
- **The wire schema is checked in CI** (`argus-proof schema --check` runs post-test). If you touch `WIRE_MODELS`/types in `models.py`, regenerate with `argus-proof schema` (it writes `schema/proof-wire.schema.json` itself; don't redirect stdout into it, which appends its status line). Bump `PROOF_VERSION` (minor = additive field/metric, major = breaking) and mind `SUPPORTED_PROOF_MAJORS`.
|
|
43
53
|
- **Pass-rate is computed over near-duplicate *groups*, not raw frames** (`n_passed / n_groups`), so a Monte-Carlo cluster counts once. Never revert this to per-image — it's the core anti-inflation invariant. `n_groups == n_images` when no deduper ran.
|
|
44
54
|
- **Scorers return normalised `[0,1]` (higher = better) or `None`** — never a fabricated `0.0` for an axis that didn't run. At most one scorer per metric (`METRIC_FIELDS`); the orchestrator raises if two target the same one. The `[0,1]` quality-scorer normalisation `lo`/`hi` defaults are **placeholders** — calibrate against real generations.
|
|
45
55
|
- **Every model file is pinned by SHA256** in the `RunManifest`, not by filename — a run must reconstruct exactly. Backends resolve a requested name to a file and hash it; don't record a name without its digest. `RejectArchive`/`RejectRecord` are **image-free by construction** (keyed by `(run_id, seed)` — seed + manifest reconstructs the pixels); never add a path/thumbnail field there.
|
|
@@ -51,3 +61,7 @@ Run a single test: `uv run --no-sync pytest tests/test_scoring.py::test_name -q`
|
|
|
51
61
|
- Concurrent report writes are serialised by a per-run `flock` sidecar (`reports.py`), so a streaming run's write and a reviewer's HITL save can't drop updates. On non-POSIX it degrades to a no-op — don't rely on it for multi-host.
|
|
52
62
|
- Shared suite code (taxonomy, wire-schema tooling, the server write-guard/env-flag helpers, `RemoteBackend`) lives in **argus-cortex** (`argus-cortex[server]`) — reuse it, don't re-implement.
|
|
53
63
|
- Versioning is git-tag-derived (`hatch-vcs`); `_version.py` is generated (gitignored). `structlog` logging, Pydantic v2, async server (`asyncio_mode = auto`), ruff line-length 120. Scaffolded from `argus-pkg-template` — `copier update` pulls tooling changes.
|
|
64
|
+
|
|
65
|
+
## Branches
|
|
66
|
+
|
|
67
|
+
`dev` is the default branch and the base for every PR. Cut feature branches from `origin/dev` and bring them up to date by merging `origin/dev`, never `main`. Never open a PR against `main` or push work to it directly: `main` must never be ahead of `dev`. A release fast-forwards `main` to a `dev` commit that passed CI (`git push origin origin/dev:main`), then tags it. If `main` is protected and needs a PR, merge the release PR, then immediately merge `main` back into `dev` so the merge commit is not left on `main` alone.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: argus-proof
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Post-training LoRA evaluation and optimisation: generate samples from a trained LoRA and score them against the curated dataset it was trained from
|
|
5
5
|
Project-URL: Repository, https://github.com/smk762/argus-proof
|
|
6
6
|
Author: smk762
|
|
@@ -46,7 +46,7 @@ Provides-Extra: moderation
|
|
|
46
46
|
Requires-Dist: accelerate>=0.27; extra == 'moderation'
|
|
47
47
|
Requires-Dist: pillow>=10.0; extra == 'moderation'
|
|
48
48
|
Requires-Dist: torch>=2.0; extra == 'moderation'
|
|
49
|
-
Requires-Dist: transformers>=4.
|
|
49
|
+
Requires-Dist: transformers>=4.45; extra == 'moderation'
|
|
50
50
|
Provides-Extra: opt
|
|
51
51
|
Requires-Dist: optuna>=3.0; extra == 'opt'
|
|
52
52
|
Provides-Extra: score
|
|
@@ -68,7 +68,12 @@ Requires-Dist: krippendorff>=0.6; extra == 'stats'
|
|
|
68
68
|
Requires-Dist: polars>=1.0; extra == 'stats'
|
|
69
69
|
Description-Content-Type: text/markdown
|
|
70
70
|
|
|
71
|
-
|
|
71
|
+
<h1>
|
|
72
|
+
<picture>
|
|
73
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/smk762/argus-cortex/dev/brand/packages/proof/argus-proof-lockup-dark.svg">
|
|
74
|
+
<img alt="argus-proof" src="https://raw.githubusercontent.com/smk762/argus-cortex/dev/brand/packages/proof/argus-proof-lockup-light.svg" height="52">
|
|
75
|
+
</picture>
|
|
76
|
+
</h1>
|
|
72
77
|
|
|
73
78
|
Post-training LoRA evaluation and optimisation: generate samples from a trained LoRA and score them against the curated dataset it was trained from.
|
|
74
79
|
|
|
@@ -113,30 +118,56 @@ argus-proof report proof-l00-w00-p000 [--json]
|
|
|
113
118
|
# Summarise an export dir (prompt sources) or a run dir (manifest + images):
|
|
114
119
|
argus-proof inspect ./curated_export
|
|
115
120
|
argus-proof inspect runs/proof-l00-w00-p000
|
|
121
|
+
|
|
122
|
+
# Emit the wire-contract JSON Schema consumers codegen against (default
|
|
123
|
+
# schema/proof-wire.schema.json; --check exits non-zero if the committed copy is stale):
|
|
124
|
+
argus-proof schema [--output PATH] [--check]
|
|
116
125
|
```
|
|
117
126
|
|
|
127
|
+
`gate`, `recommend`, `experiment` and `explore` are covered in their own sections below.
|
|
128
|
+
|
|
118
129
|
## Serve
|
|
119
130
|
|
|
120
131
|
```bash
|
|
121
132
|
argus-proof serve --port 8104 --cors # peer to lens :8100, curator :8101, quarry :8102, forge :8103
|
|
122
133
|
argus-proof serve --read-only # replay/demo mode: serve stored reports, 403 live eval + writes
|
|
123
|
-
curl -s localhost:8104/health # {"status":"ok","service":"argus-proof","version":"...","read_only":false}
|
|
134
|
+
curl -s localhost:8104/health # {"status":"ok","service":"argus-proof","version":"...","read_only":false,"capabilities":["report_manifest"]}
|
|
124
135
|
```
|
|
125
136
|
|
|
126
137
|
Routes (the backend of the argus-studio `/proof` view):
|
|
127
138
|
|
|
128
139
|
```
|
|
140
|
+
GET /health liveness + version + read_only flag + capabilities
|
|
129
141
|
GET /exports curator export dirs available to evaluate against
|
|
130
142
|
GET /models checkpoints + LoRAs under $PROOF_MODELS_DIR
|
|
131
|
-
|
|
143
|
+
GET /scorers scorers a run applies + whether each is installed
|
|
144
|
+
POST /run/stream generate + score + store one run (NDJSON RunStreamFrame lines)
|
|
132
145
|
GET /reports stored report digests (run browser)
|
|
133
146
|
GET /report/{run_id} full EvalReport PUT to store one
|
|
147
|
+
GET /report/{run_id}/manifest the RunManifest behind a report (410 if its run dir is gone)
|
|
134
148
|
GET /report/{run_id}/refined passing subset, refined ranks first
|
|
135
149
|
GET /report/{run_id}/image/{image_id} a generated sample (ids only — no path input)
|
|
150
|
+
GET /report/{run_id}/image_at/{index} sample by position — seed-free URL for blind review
|
|
136
151
|
POST /report/{run_id}/hitl apply a review; recomputes pass-rate + verdict
|
|
137
152
|
POST /report/{run_id}/refine second-pass re-rank (rank: null retracts)
|
|
153
|
+
POST /purge/{run_id} devmode only: delete one run's report + run dir
|
|
154
|
+
POST /purge devmode only: delete every report + run dir
|
|
138
155
|
```
|
|
139
156
|
|
|
157
|
+
`/report/{run_id}/manifest` serves the SHA256-pinned setup (checkpoint, VAE,
|
|
158
|
+
LoRAs, sampling, prompt, seeds) so a client can carry a passed run forward, e.g.
|
|
159
|
+
studio's proof → darkroom handoff. It is `404` when there is no report and `410
|
|
160
|
+
Gone` when the report was kept but the run dir was pruned. `/health` lists
|
|
161
|
+
`report_manifest` in `capabilities`, so a client can tell an older proof (route
|
|
162
|
+
absent) from a run with no manifest.
|
|
163
|
+
|
|
164
|
+
The `/purge` routes are a dev tool, since reports are the evaluation record. They exist
|
|
165
|
+
only with `ARGUS_PROOF_DEVMODE=1` (or `serve --devmode`), need a `{"confirm": "purge"}`
|
|
166
|
+
body, return `409` while the run (or, for `POST /purge`, any run) is still generating,
|
|
167
|
+
and read-only mode refuses them. A run's HITL review and refinement are stored in its
|
|
168
|
+
report, so they go with it. `/health` echoes `devmode` and lists `purge` in
|
|
169
|
+
`capabilities` only when the routes will work.
|
|
170
|
+
|
|
140
171
|
Concurrent reviews of the same run are serialised with a per-run file lock, so
|
|
141
172
|
a streaming run's report write and a reviewer's HITL save can't drop updates.
|
|
142
173
|
|
|
@@ -178,6 +209,25 @@ checkpoint/LoRA by **SHA256** so the run reconstructs exactly. See
|
|
|
178
209
|
[`templates/comfyui_sdxl_lora.json`](src/argus_proof/templates/comfyui_sdxl_lora.json)
|
|
179
210
|
for the shipped example.
|
|
180
211
|
|
|
212
|
+
**SDXL and Flux.** The template is chosen by the base checkpoint's *family*, its
|
|
213
|
+
top-level folder under `checkpoints/`: `flux/…` renders
|
|
214
|
+
[`templates/comfyui_flux_lora.json`](src/argus_proof/templates/comfyui_flux_lora.json)
|
|
215
|
+
and every other family renders the SDXL template, so one server evaluates both.
|
|
216
|
+
`PROOF_WORKFLOW_TEMPLATE` replaces both for every checkpoint. The Flux checkpoints
|
|
217
|
+
are bare transformers, so the Flux template loads `t5xxl_fp8_e4m3fn.safetensors`
|
|
218
|
+
and `clip_l.safetensors` (`DualCLIPLoader`, type `flux`) and `ae.safetensors`
|
|
219
|
+
(`VAELoader`) by those names; they must be in ComfyUI's `clip/` and `vae/`
|
|
220
|
+
folders. Flux takes no negative prompt (the template zeroes the positive) and runs
|
|
221
|
+
at `cfg` 1 with `sampling.guidance` (the `$guidance` slot, a `FluxGuidance` node).
|
|
222
|
+
A request that leaves `cfg`/`sampler`/`scheduler`/`guidance` out gets its family's
|
|
223
|
+
defaults: SDXL cfg 7, `dpmpp_2m`/`karras`; Flux cfg 1, `euler`/`simple`, guidance 3.5.
|
|
224
|
+
|
|
225
|
+
Nothing a run sets is silently dropped. A run is refused when it sets a value
|
|
226
|
+
the template has no slot for: a VAE, a non-empty negative prompt, `guidance`,
|
|
227
|
+
or a `clip_skip` above 2. The `diffusers` and `a1111` backends refuse `guidance`
|
|
228
|
+
outright. `clip_skip` is sent to ComfyUI as `-max(clip_skip, 2)`: SDXL uses the
|
|
229
|
+
penultimate layer, and `-1` blanks some checkpoints.
|
|
230
|
+
|
|
181
231
|
**More backends.** `get_backend(name, ...)` selects the engine by config
|
|
182
232
|
(`PROOF_BACKEND` in [`.env.example`](.env.example)); scoring/report code is
|
|
183
233
|
unchanged regardless of which produced the run, and the `RunManifest` records the
|
|
@@ -185,6 +235,10 @@ engine + version:
|
|
|
185
235
|
|
|
186
236
|
- **`diffusers`** — in-process diffusers SDXL pipeline: deterministic, no external
|
|
187
237
|
service, weights hashed from disk (`pip install "argus-proof[diffusers]"`).
|
|
238
|
+
Limitations: `clip_skip` is not applied, and only a handful of samplers map to a
|
|
239
|
+
diffusers scheduler (`euler`, `euler_a`, `ddim`, `dpmpp_2m`, `unipc`; unknown
|
|
240
|
+
names fall back to the pipeline default). The manifest still records the
|
|
241
|
+
*requested* values, so compare across engines with care.
|
|
188
242
|
- **`a1111`** — an AUTOMATIC1111 / SD.Next `/sdapi` server (checkpoint via
|
|
189
243
|
`override_settings`, LoRAs via `<lora:…>` prompt syntax; models hashed from disk).
|
|
190
244
|
- **`remote`** — a hosted/cloud endpoint that speaks the proof wire (`POST /generate`
|
|
@@ -222,6 +276,12 @@ skips it when the extra is absent). Shipped:
|
|
|
222
276
|
- **quality / adherence** — `clip_score_scorer()` (CLIPScore), `pyiqa_scorer()` (CLIP-IQA), `image_reward_scorer()` (ImageReward), each normalising its raw score to `[0,1]`
|
|
223
277
|
- **safety** — `SafetyScorer` (NudeNet ensemble, `1 - unsafe`); set a `safety` hard gate to auto-fail unsafe images. `safety_tail_aggregate()` surfaces the any-hit/min/percentile tail. NudeNet runs as a **server over HTTP** (`PROOF_NUDENET_URL`, via argus-cortex's `NudeNetClient`), never in-process (it is AGPL-3.0); without a URL the scorer is skipped. "Unsafe" is the suite's exposed-nudity set (exposed buttocks, breasts, genitalia, anus), the same as curator's
|
|
224
278
|
|
|
279
|
+
The default scorer set used by `argus-proof run` / `score` and the server
|
|
280
|
+
(`evaluate.default_scorers()`, plus phash dedup/diversity) is identity, CLIPScore,
|
|
281
|
+
CLIP-IQA and safety — it does **not** include ImageReward, so the `preference`
|
|
282
|
+
metric stays empty unless you pass `image_reward_scorer()` via the library API.
|
|
283
|
+
`GET /scorers` reports which of the default set this install can actually run.
|
|
284
|
+
|
|
225
285
|
```python
|
|
226
286
|
from argus_proof.scoring import score_run
|
|
227
287
|
from argus_proof.scoring.scorers import (
|
|
@@ -270,15 +330,42 @@ best_first = refined_ranking(refined) # passing subset, refined re-ranks on to
|
|
|
270
330
|
## Policy moderation (optional)
|
|
271
331
|
|
|
272
332
|
The Phase-2 `safety` scorer catches **nudity**; `argus_proof.moderation` extends
|
|
273
|
-
it to a **Guard-class policy taxonomy** (violence / hate / self-harm /
|
|
274
|
-
illegal) — over both the **generated images** and the **input prompts /
|
|
275
|
-
captions**, so a toxic prompt is flagged even when its output is clean
|
|
276
|
-
|
|
333
|
+
it to a **Guard-class policy taxonomy** (sexual / violence / hate / self-harm /
|
|
334
|
+
weapons / illegal) — over both the **generated images** and the **input prompts /
|
|
335
|
+
captions**, so a toxic prompt is flagged even when its output is clean.
|
|
336
|
+
|
|
337
|
+
The default detectors are **Llama Guard 3**: `LlamaGuardImageDetector`
|
|
338
|
+
(`meta-llama/Llama-Guard-3-11B-Vision`, about 22 GB in bf16) for outputs and
|
|
339
|
+
`LlamaGuardTextDetector` (`meta-llama/Llama-Guard-3-8B`, about 16 GB) for prompts and
|
|
340
|
+
captions. Install `argus-proof[moderation]`. Both checkpoints are **gated** on Hugging
|
|
341
|
+
Face: accept the license on each model page and set `HF_TOKEN`. The weights download
|
|
342
|
+
at first use; nothing is bundled. Device placement goes through argus-cortex's
|
|
343
|
+
`resolve_device` (`device="auto"` picks cuda, then mps, then cpu).
|
|
344
|
+
|
|
345
|
+
Each verdict (`safe`, or `unsafe` plus hazard codes `S1`–`S14`) maps onto the taxonomy.
|
|
346
|
+
A flagged category scores P(unsafe) at the verdict token rather than a hard 1.0.
|
|
347
|
+
Moderation fails loud rather than reading as safe. A model that can't load (no
|
|
348
|
+
token, license not accepted) or an unreadable verdict raises `ModerationError`, and an
|
|
349
|
+
`unsafe` verdict naming no code flags every category. An image is sent with the
|
|
350
|
+
neutral prompt `"Describe this image."`, so a flag comes from the image.
|
|
351
|
+
|
|
352
|
+
> **Licenses.** Llama-Guard-3-8B is under the Llama 3.1 Community License, and
|
|
353
|
+
> Llama-Guard-3-11B-Vision under the Llama 3.2 Community License. The 3.2 license
|
|
354
|
+
> does **not** grant its rights for multimodal models to individuals domiciled in,
|
|
355
|
+
> or companies with their principal place of business in, the European Union
|
|
356
|
+
> (end users of a product built on them are exempt). Both carry Meta's acceptable
|
|
357
|
+
> use policy. argus-proof only downloads the weights at runtime and doesn't
|
|
358
|
+
> redistribute them. Check the terms against your use, and pass your own detectors
|
|
359
|
+
> if they don't fit.
|
|
360
|
+
|
|
361
|
+
Any object with `name`, `version`, `is_available()` and `moderate_image(path)` /
|
|
362
|
+
`moderate_text(text)` returning per-category `[0,1]` scores can replace them:
|
|
277
363
|
|
|
278
364
|
```python
|
|
279
365
|
from argus_proof.moderation import PolicyModerator, moderate_images, moderate_texts
|
|
280
366
|
|
|
281
|
-
mod = PolicyModerator()
|
|
367
|
+
mod = PolicyModerator() # Llama Guard 3 Vision + text
|
|
368
|
+
# mod = PolicyModerator(image_detectors=[MyImageGuard()], text_detectors=[MyTextGuard()])
|
|
282
369
|
out = moderate_images(image_paths, mod) # per-category tails over the outputs
|
|
283
370
|
inp = moderate_texts(prompt_grid_variants, mod) # ...and over the inputs
|
|
284
371
|
out.flagged() # e.g. ["violence", "hate"], worst first
|
|
@@ -288,8 +375,9 @@ report.scorers.append(mod.provenance("output")) # version-stamp the Guard model
|
|
|
288
375
|
Each category gets a **tail** view (any-hit / max / 95th percentile — the extremes
|
|
289
376
|
that matter, not the mean, same rule as `safety`), combined conservatively across
|
|
290
377
|
an ensemble (most-unsafe detector wins). Detectors are pluggable and injectable, so
|
|
291
|
-
the taxonomy/ensemble/tail logic is dependency-free and unit-tested
|
|
292
|
-
|
|
378
|
+
the taxonomy/ensemble/tail logic is dependency-free and unit-tested. Of the Llama
|
|
379
|
+
Guard adapters, only the model call needs the `[moderation]` extra; verdict parsing
|
|
380
|
+
and mapping are unit-tested. A reviewer's HITL flag attributes to a
|
|
293
381
|
category via `RejectReason.category`. CSAM matching stays a **separate** policy gate
|
|
294
382
|
(Thorn Safer / PhotoDNA), not an ML metric here.
|
|
295
383
|
|
|
@@ -300,11 +388,15 @@ gate` evaluates a scored `EvalReport` against declared thresholds and **exits
|
|
|
300
388
|
non-zero when rejected**, so it drops straight into CI:
|
|
301
389
|
|
|
302
390
|
```bash
|
|
391
|
+
# --min-pass-rate-ci-lower: Wilson lower bound — a lucky 3/3 won't pass
|
|
392
|
+
# --confidence: confidence level for that bound (default 0.95)
|
|
393
|
+
# exit 0 = accepted, 1 = rejected, 2 = unreadable report / invalid threshold
|
|
303
394
|
argus-proof gate eval_report.json \
|
|
304
395
|
--min-pass-rate 0.75 \
|
|
305
|
-
--min-pass-rate-ci-lower 0.7 \
|
|
396
|
+
--min-pass-rate-ci-lower 0.7 \
|
|
397
|
+
--confidence 0.95 \
|
|
306
398
|
--min-identity 0.6 \
|
|
307
|
-
--max-unsafe-rate 0.0
|
|
399
|
+
--max-unsafe-rate 0.0
|
|
308
400
|
```
|
|
309
401
|
|
|
310
402
|
The pass-rate lower bound uses a Wilson score interval (`argus_proof.stats`, no
|
|
@@ -375,7 +467,7 @@ with a `--max-gpu-hours` guardrail that refuses an intractable matrix:
|
|
|
375
467
|
```bash
|
|
376
468
|
argus-proof experiment matrix.json --export ./curated_export --max-gpu-hours 40
|
|
377
469
|
# experiment exp: 4 cells, 32 runs, 96 images
|
|
378
|
-
# [exp-
|
|
470
|
+
# [exp-c00-sdxl-a-fast] 24 images (8 runs)
|
|
379
471
|
# ...
|
|
380
472
|
# est. 0.2 GPU-hours @ 6s/image
|
|
381
473
|
```
|