argus-cortex 0.3.0__tar.gz → 0.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/.github/workflows/release.yml +40 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/CHANGELOG.md +48 -1
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/CLAUDE.md +1 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/PKG-INFO +23 -1
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/README.md +19 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/pyproject.toml +4 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/src/argus_cortex/__init__.py +15 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/src/argus_cortex/_version.py +2 -2
- argus_cortex-0.4.1/src/argus_cortex/moderation/__init__.py +50 -0
- argus_cortex-0.4.1/src/argus_cortex/moderation/_http.py +72 -0
- argus_cortex-0.4.1/src/argus_cortex/moderation/nudenet.py +162 -0
- argus_cortex-0.4.1/src/argus_cortex/moderation/singguard.py +218 -0
- argus_cortex-0.4.1/tests/test_moderation.py +359 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/.copier-answers.yml +0 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/.env.example +0 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/.github/workflows/ci.yml +0 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/.gitignore +0 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/LICENSE +0 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/Makefile +0 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/src/argus_cortex/backends.py +0 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/src/argus_cortex/py.typed +0 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/src/argus_cortex/server.py +0 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/src/argus_cortex/store/__init__.py +0 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/src/argus_cortex/store/blob.py +0 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/src/argus_cortex/store/config.py +0 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/src/argus_cortex/store/errors.py +0 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/src/argus_cortex/store/lineage.py +0 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/src/argus_cortex/store/models.py +0 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/src/argus_cortex/store/schema.py +0 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/src/argus_cortex/store/vector.py +0 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/src/argus_cortex/taxonomy.py +0 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/src/argus_cortex/wire.py +0 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/src/argus_cortex/xmp.py +0 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/tests/test_backends.py +0 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/tests/test_blob.py +0 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/tests/test_server.py +0 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/tests/test_smoke.py +0 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/tests/test_store.py +0 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/tests/test_store_pg_integration.py +0 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/tests/test_taxonomy.py +0 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/tests/test_vector.py +0 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/tests/test_wire.py +0 -0
- {argus_cortex-0.3.0 → argus_cortex-0.4.1}/tests/test_xmp.py +0 -0
|
@@ -78,3 +78,43 @@ jobs:
|
|
|
78
78
|
name: dist
|
|
79
79
|
path: dist/
|
|
80
80
|
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
81
|
+
|
|
82
|
+
github-release:
|
|
83
|
+
# The GitHub Release is the last step, so it only appears once the version is
|
|
84
|
+
# actually on PyPI. Its notes are this version's CHANGELOG.md section.
|
|
85
|
+
# Gated on publish-pypi's own result, not the default success(): publish-pypi
|
|
86
|
+
# runs even when the TestPyPI rehearsal failed, and that must not skip this.
|
|
87
|
+
runs-on: ubuntu-latest
|
|
88
|
+
needs: publish-pypi
|
|
89
|
+
if: "!cancelled() && needs.publish-pypi.result == 'success'"
|
|
90
|
+
permissions:
|
|
91
|
+
contents: write
|
|
92
|
+
steps:
|
|
93
|
+
- uses: actions/checkout@v4
|
|
94
|
+
- uses: actions/download-artifact@v4
|
|
95
|
+
with:
|
|
96
|
+
name: dist
|
|
97
|
+
path: dist/
|
|
98
|
+
- name: Extract release notes from CHANGELOG.md
|
|
99
|
+
run: |
|
|
100
|
+
version="${GITHUB_REF_NAME#v}"
|
|
101
|
+
awk -v head="## [${version}]" 'index($0, head) == 1 {found = 1; next} found && /^## \[/ {exit} found' \
|
|
102
|
+
CHANGELOG.md > notes.md
|
|
103
|
+
if ! grep -q '[^[:space:]]' notes.md; then
|
|
104
|
+
echo "::error::CHANGELOG.md has no section for ${version}"
|
|
105
|
+
exit 1
|
|
106
|
+
fi
|
|
107
|
+
- name: Create or update the GitHub Release
|
|
108
|
+
env:
|
|
109
|
+
GH_TOKEN: ${{ github.token }}
|
|
110
|
+
run: |
|
|
111
|
+
tag="${GITHUB_REF_NAME}"
|
|
112
|
+
prerelease=""
|
|
113
|
+
case "$tag" in *-*|*rc*|*a[0-9]*|*b[0-9]*) prerelease="--prerelease" ;; esac
|
|
114
|
+
if gh release view "$tag" >/dev/null 2>&1; then
|
|
115
|
+
# Re-run of the workflow: refresh notes and assets rather than failing.
|
|
116
|
+
gh release edit "$tag" --notes-file notes.md
|
|
117
|
+
gh release upload "$tag" dist/* --clobber
|
|
118
|
+
else
|
|
119
|
+
gh release create "$tag" dist/* --verify-tag --title "$tag" --notes-file notes.md $prerelease
|
|
120
|
+
fi
|
|
@@ -7,6 +7,51 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.4.1] - 2026-10-27
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
- `SingGuardClient(thinking="slow")` and `thinking="cascade"`: the argus-cull
|
|
14
|
+
`singguard` service's modes. `slow` reasons with no fast first-line verdict,
|
|
15
|
+
so the reasoning is not anchored to a verdict made before it; `cascade` runs
|
|
16
|
+
`fast` on every image and `slow` only on the ones fast flags, and slow's
|
|
17
|
+
verdict stands. `THINKING_MODES` lists them. A SingGuard server without a
|
|
18
|
+
mode answers HTTP 400, raised as `GuardCheckFailed` per check.
|
|
19
|
+
- `GuardResult.thinking`: the mode the server says it judged in (`None` from a
|
|
20
|
+
server that does not say). The argus-cull service can be forced server-side
|
|
21
|
+
to a mode other than the one asked for, and this records which it used.
|
|
22
|
+
|
|
23
|
+
## [0.4.0] - 2026-10-26
|
|
24
|
+
|
|
25
|
+
### Added
|
|
26
|
+
- **`argus_cortex.moderation`** — content-moderation clients shared by the
|
|
27
|
+
stages, hoisted from argus-curator. Both talk HTTP to a server the deployer
|
|
28
|
+
runs; nothing imports a detector.
|
|
29
|
+
- `NudeNetClient(base_url, timeout)` — a NudeNet server's `/infer`, returning
|
|
30
|
+
`NsfwResult` (detections, highest score per class, exposed score) and
|
|
31
|
+
raising `NudeNetCheckFailed` per image. `NudeNetLabel` / `NUDENET_LABELS`
|
|
32
|
+
are NudeNet's 18 classes; `DEFAULT_EXPOSED_LABELS` (the five exposed
|
|
33
|
+
buttocks/breast/genitalia/anus classes) and `exposed_score()` are the
|
|
34
|
+
suite's one definition of nudity.
|
|
35
|
+
- `SingGuardClient(base_url, timeout, thinking, policy, downscale)` — a
|
|
36
|
+
SingGuard server's `/api/guard/check`, returning `GuardResult` (verdict,
|
|
37
|
+
the rules that fired, explanation, model, policy name) and raising
|
|
38
|
+
`GuardCheckFailed`. `rule_matches()` / `rule_letter()` match rules by
|
|
39
|
+
letter or full name; `policy_name()` names a custom policy by a sha256
|
|
40
|
+
prefix.
|
|
41
|
+
- Proxies in the environment are ignored: images go only to the configured
|
|
42
|
+
server. Config errors (`ModerationConfigError`) never echo the URL. A
|
|
43
|
+
`user:password@` in the URL is sent as HTTP Basic auth.
|
|
44
|
+
- `[images]` extra (Pillow), for `SingGuardClient`'s downscaling of images
|
|
45
|
+
over 1536 px / 4 MB; imported only when it downscales. Downscaling is on by
|
|
46
|
+
default, so without the extra `SingGuardClient(...)` raises
|
|
47
|
+
`ModerationConfigError` at construction unless given `downscale=False`.
|
|
48
|
+
|
|
49
|
+
### Changed
|
|
50
|
+
- `release.yml` now creates the **GitHub Release** for each `v*` tag once the
|
|
51
|
+
version is on PyPI, with that version's `CHANGELOG.md` section as notes and
|
|
52
|
+
the built sdist/wheel attached (a missing section fails the job; a re-run
|
|
53
|
+
edits the Release). Tags with `-`, `rc`, `a<n>` or `b<n>` are pre-releases.
|
|
54
|
+
|
|
10
55
|
## [0.3.0] - 2026-10-25
|
|
11
56
|
|
|
12
57
|
### Added
|
|
@@ -71,7 +116,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
71
116
|
(`LocalBackend`, `RemoteBackend`, `RemoteProvider`, `with_retries`
|
|
72
117
|
retrying only transient errors, `resolve_device`).
|
|
73
118
|
|
|
74
|
-
[Unreleased]: https://github.com/smk762/argus-cortex/compare/v0.
|
|
119
|
+
[Unreleased]: https://github.com/smk762/argus-cortex/compare/v0.4.1...HEAD
|
|
120
|
+
[0.4.1]: https://github.com/smk762/argus-cortex/compare/v0.4.0...v0.4.1
|
|
121
|
+
[0.4.0]: https://github.com/smk762/argus-cortex/compare/v0.3.0...v0.4.0
|
|
75
122
|
[0.3.0]: https://github.com/smk762/argus-cortex/compare/v0.2.0...v0.3.0
|
|
76
123
|
[0.2.0]: https://github.com/smk762/argus-cortex/compare/v0.1.0...v0.2.0
|
|
77
124
|
[0.1.0]: https://github.com/smk762/argus-cortex/releases/tag/v0.1.0
|
|
@@ -19,6 +19,7 @@ Because everything here is imported by the downstream stages, **a change ripples
|
|
|
19
19
|
- `backends.py` — the AI-backend contract: `Backend`/`LocalBackend`/`RemoteBackend` (point at a hosted service via `from_host(host, port)` or `from_provider(RemoteProvider.…)`), plus `with_retries()` and `resolve_device()`. httpx is lazy, behind `[remote]`.
|
|
20
20
|
- `server.py` — the shared FastAPI/ASGI scaffolding other repos' micro-servers mount: `WriteGuard` (pure-ASGI method gate), the `constant_refuse()` (read-only/replay) and `cross_site_refuse()` (cross-origin write) predicates, `env_flag()`, and the `UNSAFE_METHODS`/`TRUTHY`/`FALSY` constants. Starlette is behind `[server]`.
|
|
21
21
|
- `xmp.py` — the suite's XMP sidecar read/merge/write (`write_xmp`/`read_xmp`/`merge_xmp`/`parse_xmp`, `XmpData`/`XmpRegions`, `XmpError`), used by lens (keywords/caption) and curator (faces, content rating). Stdlib `xml.etree` only. **Writes merge — latest wins per property, foreign properties preserved; keep that.** `None` = leave untouched, empty = remove. A DTD is refused and an unparseable sidecar raises rather than being overwritten. The `argus` namespace URI is a wire contract — don't change it.
|
|
22
|
+
- `moderation/` — content-moderation clients over HTTP: `nudenet.py` (`NudeNetClient`, `NsfwResult`, `DEFAULT_EXPOSED_LABELS` — **the suite's one definition of nudity; don't widen it casually**, every stage's NSFW numbers move with it), `singguard.py` (`SingGuardClient`, `GuardResult`, `rule_matches`), `_http.py` (shared POST, proxy-free opener, `CheckFailed`/`ModerationConfigError`). Stdlib + pydantic; Pillow lazily, behind `[images]`. Never import NudeNet (AGPL) — HTTP only. URLs are passed in; stages own their env config.
|
|
22
23
|
- `store/` — the optional stateful-services layer, opt-in per `CORTEX_*` env var: `lineage.py` (Phase 1 Postgres DAG, `open_lineage_store()`), `vector.py` (Phase 2 Qdrant, `open_vector_store()`), `blob.py` (Phase 3 MinIO/S3, `open_blob_store()`), over `config.py`/`models.py`/`errors.py`. Each `open_*()` returns a `Null*` no-op when its URL is unset. Drivers are behind `[postgres]`/`[qdrant]`/`[s3]`.
|
|
23
24
|
|
|
24
25
|
There is **no CLI and no committed schema in this repo** — the `schema`/`schema --check` pattern is something `wire.py` *enables* for downstream repos, not something cortex runs on itself.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: argus-cortex
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.1
|
|
4
4
|
Summary: Shared foundation for the Argus suite: taxonomy, versioned wire-schema tooling, and the local/remote AI backend contract
|
|
5
5
|
Project-URL: Repository, https://github.com/smk762/argus-cortex
|
|
6
6
|
Author: smk762
|
|
@@ -20,12 +20,15 @@ Requires-Dist: certifi>=2026.10.19; extra == 'dev'
|
|
|
20
20
|
Requires-Dist: httpx2>=2.7; extra == 'dev'
|
|
21
21
|
Requires-Dist: httpx>=0.27; extra == 'dev'
|
|
22
22
|
Requires-Dist: minio>=7.2; extra == 'dev'
|
|
23
|
+
Requires-Dist: pillow>=10.0; extra == 'dev'
|
|
23
24
|
Requires-Dist: psycopg[binary,pool]>=3.1; extra == 'dev'
|
|
24
25
|
Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
|
|
25
26
|
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
26
27
|
Requires-Dist: qdrant-client>=1.10; extra == 'dev'
|
|
27
28
|
Requires-Dist: ruff==0.15.16; extra == 'dev'
|
|
28
29
|
Requires-Dist: starlette>=0.37; extra == 'dev'
|
|
30
|
+
Provides-Extra: images
|
|
31
|
+
Requires-Dist: pillow>=10.0; extra == 'images'
|
|
29
32
|
Provides-Extra: postgres
|
|
30
33
|
Requires-Dist: psycopg[binary,pool]>=3.1; extra == 'postgres'
|
|
31
34
|
Provides-Extra: qdrant
|
|
@@ -54,6 +57,7 @@ Part of the [Argus suite](https://github.com/smk762?tab=repositories&q=argus) (q
|
|
|
54
57
|
- **`argus_cortex.backends`** — the AI-backend contract generalised from argus-lens: `Backend` / `LocalBackend` / `RemoteBackend` (point at a hosted service by `base_url`, `host`/`port`, or a known `RemoteProvider`), plus `with_retries()` and `resolve_device()`. httpx lives behind the `[remote]` extra.
|
|
55
58
|
- **`argus_cortex.server`** — shared write-guard + env-flag scaffolding for the suite's FastAPI micro-servers: `WriteGuard` (a pure-ASGI method-gate that never buffers streaming responses), the `constant_refuse()` / `cross_site_refuse()` predicates (read-only/replay mode and cross-site-write protection are two configs of the one guard), the `UNSAFE_METHODS` / `TRUTHY` / `FALSY` constants, and `env_flag()` (warns on a mistyped protection flag rather than silently disabling it). Starlette lives behind the `[server]` extra.
|
|
56
59
|
- **`argus_cortex.xmp`** — the suite's one XMP sidecar implementation (hoisted from argus-lens): `write_xmp()` / `read_xmp()` / `merge_xmp()` / `parse_xmp()` over an `XmpData` model covering `dc` (keywords, description), `xmp` (Rating, Label), `lr` (hierarchical keywords), `Iptc4xmpExt` (PersonInImage), `mwg-rs` face/pet regions, and `argus:ContentRating`. Writes **merge**: each property you set replaces that property (latest write wins), everything else in the sidecar — Lightroom develop settings, GPS, another stage's fields — is preserved. `None` leaves a property alone, an empty value removes it. Writes are atomic; an unparseable existing sidecar raises `XmpError` instead of being clobbered. Stdlib only.
|
|
60
|
+
- **`argus_cortex.moderation`** — the suite's content-moderation clients, over HTTP to servers the deployer runs (nothing here imports a detector): `NudeNetClient` (a NudeNet server's `/infer` → `NsfwResult`: detections, per-class maxima, and an **exposed score**) and `SingGuardClient` (a SingGuard server's `/api/guard/check` → `GuardResult`: verdict, the policy rules that fired, explanation; optional custom policy text). `DEFAULT_EXPOSED_LABELS` is the suite's **one definition of nudity**: the five exposed buttocks/breast/genitalia/anus classes — faces, covered parts, and bare feet, belly, armpits or male chest never count. Stdlib + pydantic; Pillow (for SingGuard's downscaling of large images, on by default) is behind `[images]`: without it `SingGuardClient(...)` raises `ModerationConfigError` at construction unless given `downscale=False`. A `user:password@` in a URL is sent as HTTP Basic auth. URLs are passed in, never read from the environment here: each stage owns its config. NudeNet itself is AGPL-3.0, which is why it stays behind an HTTP boundary.
|
|
57
61
|
- **`argus_cortex.store`** — the optional **stateful services layer** (the suite's persistent "memory"). External and opt-in — every store is configured by `CORTEX_*` env vars and no-ops when its URL is unset.
|
|
58
62
|
- **Phase 1 — Postgres lineage store.** Persists the `source_asset → caption → human_edit → dataset_membership → training_run` DAG for LoRA reproducibility, edit capture, and the feedback loop. Backed by a psycopg connection **pool** (concurrent `asyncio.to_thread` calls are safe; a dropped connection is replaced), with a `transaction()` boundary for atomic multi-step writes. `open_lineage_store()` returns a no-op store when `CORTEX_PG_URL` is unset; psycopg lives behind the `[postgres]` extra.
|
|
59
63
|
- **Phase 2 — Qdrant vector store.** Stores image + tag-set embeddings for retrieval-augmented few-shot and near-duplicate curation; put a `caption_id`/`asset_id` in a point's payload and a search hit joins straight back to the lineage. `open_vector_store()` no-ops when `CORTEX_QDRANT_URL` is unset; qdrant-client lives behind the `[qdrant]` extra. cortex stores vectors it's *handed* — computing embeddings is the caller's job.
|
|
@@ -87,6 +91,24 @@ write_xmp(
|
|
|
87
91
|
)
|
|
88
92
|
```
|
|
89
93
|
|
|
94
|
+
```python
|
|
95
|
+
from argus_cortex.moderation import NudeNetClient, SingGuardClient, rule_matches
|
|
96
|
+
|
|
97
|
+
nudenet = NudeNetClient("http://nudenet:8080") # its origin; /infer is appended
|
|
98
|
+
result = nudenet.check(image_bytes, "beach.jpg")
|
|
99
|
+
result.exposed_score # 0.0-1.0 over DEFAULT_EXPOSED_LABELS; result.labels = per-class maxima
|
|
100
|
+
|
|
101
|
+
guard = SingGuardClient("http://guard:9150", policy=None) # None = SingGuard's default rules
|
|
102
|
+
# thinking="fast" (default) is a verdict and rule; "fast-slow" also writes why; "slow" writes why
|
|
103
|
+
# with no fast verdict ahead of the reasoning; "cascade" is fast, then slow only for what fast
|
|
104
|
+
# flagged (the last two: argus-cull's singguard service only).
|
|
105
|
+
verdict = guard.check(image_bytes, text="the prompt, if any")
|
|
106
|
+
verdict.safe, verdict.rules # False, ["B. Real-World Crimes & Public Safety"]
|
|
107
|
+
any(rule_matches(r, "B") for r in verdict.rules) # match a rule by letter or full name
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
Each `check()` raises `CheckFailed` (`NudeNetCheckFailed` / `GuardCheckFailed`) for a network error or an unusable answer, so a caller checking many images can record "unchecked" against the one image — and should treat unchecked as *not* safe. Bad configuration raises `ModerationConfigError`, whose message never echoes the URL.
|
|
111
|
+
|
|
90
112
|
```python
|
|
91
113
|
from argus_cortex.store import open_lineage_store, SourceAsset, Caption, HumanEdit
|
|
92
114
|
|
|
@@ -11,6 +11,7 @@ Part of the [Argus suite](https://github.com/smk762?tab=repositories&q=argus) (q
|
|
|
11
11
|
- **`argus_cortex.backends`** — the AI-backend contract generalised from argus-lens: `Backend` / `LocalBackend` / `RemoteBackend` (point at a hosted service by `base_url`, `host`/`port`, or a known `RemoteProvider`), plus `with_retries()` and `resolve_device()`. httpx lives behind the `[remote]` extra.
|
|
12
12
|
- **`argus_cortex.server`** — shared write-guard + env-flag scaffolding for the suite's FastAPI micro-servers: `WriteGuard` (a pure-ASGI method-gate that never buffers streaming responses), the `constant_refuse()` / `cross_site_refuse()` predicates (read-only/replay mode and cross-site-write protection are two configs of the one guard), the `UNSAFE_METHODS` / `TRUTHY` / `FALSY` constants, and `env_flag()` (warns on a mistyped protection flag rather than silently disabling it). Starlette lives behind the `[server]` extra.
|
|
13
13
|
- **`argus_cortex.xmp`** — the suite's one XMP sidecar implementation (hoisted from argus-lens): `write_xmp()` / `read_xmp()` / `merge_xmp()` / `parse_xmp()` over an `XmpData` model covering `dc` (keywords, description), `xmp` (Rating, Label), `lr` (hierarchical keywords), `Iptc4xmpExt` (PersonInImage), `mwg-rs` face/pet regions, and `argus:ContentRating`. Writes **merge**: each property you set replaces that property (latest write wins), everything else in the sidecar — Lightroom develop settings, GPS, another stage's fields — is preserved. `None` leaves a property alone, an empty value removes it. Writes are atomic; an unparseable existing sidecar raises `XmpError` instead of being clobbered. Stdlib only.
|
|
14
|
+
- **`argus_cortex.moderation`** — the suite's content-moderation clients, over HTTP to servers the deployer runs (nothing here imports a detector): `NudeNetClient` (a NudeNet server's `/infer` → `NsfwResult`: detections, per-class maxima, and an **exposed score**) and `SingGuardClient` (a SingGuard server's `/api/guard/check` → `GuardResult`: verdict, the policy rules that fired, explanation; optional custom policy text). `DEFAULT_EXPOSED_LABELS` is the suite's **one definition of nudity**: the five exposed buttocks/breast/genitalia/anus classes — faces, covered parts, and bare feet, belly, armpits or male chest never count. Stdlib + pydantic; Pillow (for SingGuard's downscaling of large images, on by default) is behind `[images]`: without it `SingGuardClient(...)` raises `ModerationConfigError` at construction unless given `downscale=False`. A `user:password@` in a URL is sent as HTTP Basic auth. URLs are passed in, never read from the environment here: each stage owns its config. NudeNet itself is AGPL-3.0, which is why it stays behind an HTTP boundary.
|
|
14
15
|
- **`argus_cortex.store`** — the optional **stateful services layer** (the suite's persistent "memory"). External and opt-in — every store is configured by `CORTEX_*` env vars and no-ops when its URL is unset.
|
|
15
16
|
- **Phase 1 — Postgres lineage store.** Persists the `source_asset → caption → human_edit → dataset_membership → training_run` DAG for LoRA reproducibility, edit capture, and the feedback loop. Backed by a psycopg connection **pool** (concurrent `asyncio.to_thread` calls are safe; a dropped connection is replaced), with a `transaction()` boundary for atomic multi-step writes. `open_lineage_store()` returns a no-op store when `CORTEX_PG_URL` is unset; psycopg lives behind the `[postgres]` extra.
|
|
16
17
|
- **Phase 2 — Qdrant vector store.** Stores image + tag-set embeddings for retrieval-augmented few-shot and near-duplicate curation; put a `caption_id`/`asset_id` in a point's payload and a search hit joins straight back to the lineage. `open_vector_store()` no-ops when `CORTEX_QDRANT_URL` is unset; qdrant-client lives behind the `[qdrant]` extra. cortex stores vectors it's *handed* — computing embeddings is the caller's job.
|
|
@@ -44,6 +45,24 @@ write_xmp(
|
|
|
44
45
|
)
|
|
45
46
|
```
|
|
46
47
|
|
|
48
|
+
```python
|
|
49
|
+
from argus_cortex.moderation import NudeNetClient, SingGuardClient, rule_matches
|
|
50
|
+
|
|
51
|
+
nudenet = NudeNetClient("http://nudenet:8080") # its origin; /infer is appended
|
|
52
|
+
result = nudenet.check(image_bytes, "beach.jpg")
|
|
53
|
+
result.exposed_score # 0.0-1.0 over DEFAULT_EXPOSED_LABELS; result.labels = per-class maxima
|
|
54
|
+
|
|
55
|
+
guard = SingGuardClient("http://guard:9150", policy=None) # None = SingGuard's default rules
|
|
56
|
+
# thinking="fast" (default) is a verdict and rule; "fast-slow" also writes why; "slow" writes why
|
|
57
|
+
# with no fast verdict ahead of the reasoning; "cascade" is fast, then slow only for what fast
|
|
58
|
+
# flagged (the last two: argus-cull's singguard service only).
|
|
59
|
+
verdict = guard.check(image_bytes, text="the prompt, if any")
|
|
60
|
+
verdict.safe, verdict.rules # False, ["B. Real-World Crimes & Public Safety"]
|
|
61
|
+
any(rule_matches(r, "B") for r in verdict.rules) # match a rule by letter or full name
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Each `check()` raises `CheckFailed` (`NudeNetCheckFailed` / `GuardCheckFailed`) for a network error or an unusable answer, so a caller checking many images can record "unchecked" against the one image — and should treat unchecked as *not* safe. Bad configuration raises `ModerationConfigError`, whose message never echoes the URL.
|
|
65
|
+
|
|
47
66
|
```python
|
|
48
67
|
from argus_cortex.store import open_lineage_store, SourceAsset, Caption, HumanEdit
|
|
49
68
|
|
|
@@ -32,6 +32,9 @@ s3 = ["minio>=7.2", "certifi>=2026.10.19"]
|
|
|
32
32
|
# (argus_cortex.server). Starlette ships with FastAPI, so consumers already have
|
|
33
33
|
# it via their own [server] extra.
|
|
34
34
|
server = ["starlette>=0.37"]
|
|
35
|
+
# Pillow, for SingGuardClient's downscaling of large images
|
|
36
|
+
# (argus_cortex.moderation.singguard), imported only when it downscales.
|
|
37
|
+
images = ["pillow>=10.0"]
|
|
35
38
|
# The store drivers are in dev so each store's real client path (pool/enum/client
|
|
36
39
|
# construction, error wrapping) is exercised in CI, not just via fakes.
|
|
37
40
|
# starlette + httpx power the pure-ASGI guard tests (Starlette TestClient).
|
|
@@ -45,6 +48,7 @@ dev = [
|
|
|
45
48
|
"starlette>=0.37",
|
|
46
49
|
"httpx>=0.27",
|
|
47
50
|
"httpx2>=2.7",
|
|
51
|
+
"pillow>=10.0",
|
|
48
52
|
"certifi>=2026.10.19",
|
|
49
53
|
]
|
|
50
54
|
|
|
@@ -23,6 +23,14 @@ from argus_cortex.backends import (
|
|
|
23
23
|
resolve_device,
|
|
24
24
|
with_retries,
|
|
25
25
|
)
|
|
26
|
+
from argus_cortex.moderation import (
|
|
27
|
+
CheckFailed,
|
|
28
|
+
GuardResult,
|
|
29
|
+
ModerationConfigError,
|
|
30
|
+
NsfwResult,
|
|
31
|
+
NudeNetClient,
|
|
32
|
+
SingGuardClient,
|
|
33
|
+
)
|
|
26
34
|
from argus_cortex.store import (
|
|
27
35
|
IMAGE_COLLECTION,
|
|
28
36
|
TAGSET_COLLECTION,
|
|
@@ -102,6 +110,13 @@ __all__ = [
|
|
|
102
110
|
"RemoteProvider",
|
|
103
111
|
"resolve_device",
|
|
104
112
|
"with_retries",
|
|
113
|
+
# moderation clients (NudeNet, SingGuard over HTTP)
|
|
114
|
+
"CheckFailed",
|
|
115
|
+
"GuardResult",
|
|
116
|
+
"ModerationConfigError",
|
|
117
|
+
"NsfwResult",
|
|
118
|
+
"NudeNetClient",
|
|
119
|
+
"SingGuardClient",
|
|
105
120
|
# store (optional stateful services layer)
|
|
106
121
|
"StoreConfig",
|
|
107
122
|
"StoreError",
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '0.
|
|
22
|
-
__version_tuple__ = version_tuple = (0,
|
|
21
|
+
__version__ = version = '0.4.1'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 4, 1)
|
|
23
23
|
|
|
24
24
|
__commit_id__ = commit_id = None
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""Content-moderation clients the stages share: NudeNet and SingGuard over HTTP.
|
|
2
|
+
|
|
3
|
+
Each client talks to a server the deployer runs; nothing here imports a
|
|
4
|
+
detector. Stdlib + pydantic only (Pillow, for SingGuard's downscaling, is
|
|
5
|
+
behind the ``[images]`` extra and imported only when used). Configuration is
|
|
6
|
+
the caller's: each stage reads its own environment and passes a URL in.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from argus_cortex.moderation._http import CheckFailed, ModerationConfigError
|
|
12
|
+
from argus_cortex.moderation.nudenet import (
|
|
13
|
+
DEFAULT_EXPOSED_LABELS,
|
|
14
|
+
NUDENET_LABELS,
|
|
15
|
+
NsfwDetection,
|
|
16
|
+
NsfwResult,
|
|
17
|
+
NudeNetCheckFailed,
|
|
18
|
+
NudeNetClient,
|
|
19
|
+
NudeNetLabel,
|
|
20
|
+
exposed_score,
|
|
21
|
+
)
|
|
22
|
+
from argus_cortex.moderation.singguard import (
|
|
23
|
+
THINKING_MODES,
|
|
24
|
+
GuardCheckFailed,
|
|
25
|
+
GuardResult,
|
|
26
|
+
SingGuardClient,
|
|
27
|
+
policy_name,
|
|
28
|
+
rule_letter,
|
|
29
|
+
rule_matches,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
__all__ = [
|
|
33
|
+
"DEFAULT_EXPOSED_LABELS",
|
|
34
|
+
"NUDENET_LABELS",
|
|
35
|
+
"THINKING_MODES",
|
|
36
|
+
"CheckFailed",
|
|
37
|
+
"GuardCheckFailed",
|
|
38
|
+
"GuardResult",
|
|
39
|
+
"ModerationConfigError",
|
|
40
|
+
"NsfwDetection",
|
|
41
|
+
"NsfwResult",
|
|
42
|
+
"NudeNetCheckFailed",
|
|
43
|
+
"NudeNetClient",
|
|
44
|
+
"NudeNetLabel",
|
|
45
|
+
"SingGuardClient",
|
|
46
|
+
"exposed_score",
|
|
47
|
+
"policy_name",
|
|
48
|
+
"rule_letter",
|
|
49
|
+
"rule_matches",
|
|
50
|
+
]
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""The one HTTP path the moderation clients share. Stdlib only."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import base64
|
|
6
|
+
import http.client
|
|
7
|
+
import json
|
|
8
|
+
import urllib.error
|
|
9
|
+
import urllib.request
|
|
10
|
+
from urllib.parse import unquote, urlsplit, urlunsplit
|
|
11
|
+
|
|
12
|
+
# Ignore HTTP(S)_PROXY: images go straight to the configured server, never
|
|
13
|
+
# through whatever proxy happens to be in the environment.
|
|
14
|
+
_OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class ModerationConfigError(ValueError):
|
|
18
|
+
"""A moderation client was configured with an unusable URL or option."""
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class CheckFailed(RuntimeError):
|
|
22
|
+
"""One image could not be checked (network error, bad response)."""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def service_url(base: str, path: str, service: str) -> str:
|
|
26
|
+
"""*base* (a server's origin) with *path* appended to its path; a *base*
|
|
27
|
+
whose path already ends in *path* is used as is. A query string is kept.
|
|
28
|
+
|
|
29
|
+
The error names *service* but not *base*: a URL can carry a credential, and
|
|
30
|
+
callers surface this message.
|
|
31
|
+
"""
|
|
32
|
+
parts = urlsplit(base.strip())
|
|
33
|
+
if parts.scheme not in {"http", "https"} or not parts.hostname:
|
|
34
|
+
raise ModerationConfigError(f"the {service} URL must be an http(s) URL")
|
|
35
|
+
base_path = parts.path.rstrip("/")
|
|
36
|
+
if not base_path.endswith(path):
|
|
37
|
+
base_path += path
|
|
38
|
+
return urlunsplit((parts.scheme, parts.netloc, base_path, parts.query, ""))
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _split_credentials(url: str) -> tuple[str, dict[str, str]]:
|
|
42
|
+
"""*url* without its ``user:password@`` part, plus the Basic auth header it
|
|
43
|
+
stood for (urllib would otherwise take the userinfo for part of the host)."""
|
|
44
|
+
parts = urlsplit(url)
|
|
45
|
+
if parts.username is None:
|
|
46
|
+
return url, {}
|
|
47
|
+
netloc = parts.netloc.rpartition("@")[2]
|
|
48
|
+
userinfo = f"{unquote(parts.username)}:{unquote(parts.password or '')}"
|
|
49
|
+
auth = f"Basic {base64.b64encode(userinfo.encode()).decode('ascii')}"
|
|
50
|
+
return urlunsplit(parts._replace(netloc=netloc)), {"Authorization": auth}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def post_json(
|
|
54
|
+
url: str, body: bytes, content_type: str, timeout: float, service: str, failed: type[CheckFailed]
|
|
55
|
+
) -> object:
|
|
56
|
+
"""POST *body* to *url* and return the decoded JSON answer.
|
|
57
|
+
|
|
58
|
+
Every transport or decoding problem is raised as *failed*, so a caller
|
|
59
|
+
checking many images can record it against the one image.
|
|
60
|
+
"""
|
|
61
|
+
url, headers = _split_credentials(url)
|
|
62
|
+
req = urllib.request.Request(url, data=body, headers={"Content-Type": content_type, **headers}, method="POST")
|
|
63
|
+
try:
|
|
64
|
+
with _OPENER.open(req, timeout=timeout) as resp: # noqa: S310 - caller-configured URL
|
|
65
|
+
return json.loads(resp.read())
|
|
66
|
+
except urllib.error.HTTPError as exc:
|
|
67
|
+
exc.close() # the exception outlives the request as __cause__; its socket need not
|
|
68
|
+
raise failed(f"{service} returned HTTP {exc.code}") from exc
|
|
69
|
+
except (urllib.error.URLError, http.client.HTTPException, TimeoutError, OSError) as exc:
|
|
70
|
+
raise failed(f"{service} request failed: {exc}") from exc
|
|
71
|
+
except ValueError as exc:
|
|
72
|
+
raise failed(f"{service} returned a body that is not JSON") from exc
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"""NudeNet client: nudity detection through a NudeNet server over HTTP.
|
|
2
|
+
|
|
3
|
+
The suite never imports NudeNet (AGPL-3.0): it posts images to a NudeNet
|
|
4
|
+
server (``ghcr.io/notai-tech/nudenet`` or any server speaking the same API)
|
|
5
|
+
and records the detections. Running that server is the deployer's choice;
|
|
6
|
+
cortex and the stages stay MIT.
|
|
7
|
+
|
|
8
|
+
This module is also the suite's one definition of what counts as nudity:
|
|
9
|
+
:data:`DEFAULT_EXPOSED_LABELS`. Faces, covered body parts, and exposed feet,
|
|
10
|
+
belly, armpits or male chest are detected too but do not count towards
|
|
11
|
+
:attr:`NsfwResult.exposed_score`.
|
|
12
|
+
|
|
13
|
+
API: ``POST /infer`` with each image as a multipart field ``f1``, ``f2``, …,
|
|
14
|
+
answered by ``{"prediction": [[{"class", "score", "box"}, ...], ...],
|
|
15
|
+
"success": true}`` in field order. ``box`` is ``[x, y, w, h]`` in source
|
|
16
|
+
pixels, and the server drops detections scoring under
|
|
17
|
+
:data:`MIN_REPORTED_SCORE` (as does :func:`to_result`, whatever the server).
|
|
18
|
+
|
|
19
|
+
Note: the upstream ``ghcr.io/notai-tech/nudenet`` image reports ``y`` offset by
|
|
20
|
+
the letterbox padding on non-square images (scores are unaffected).
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import math
|
|
26
|
+
import re
|
|
27
|
+
import uuid
|
|
28
|
+
from typing import Literal
|
|
29
|
+
|
|
30
|
+
from pydantic import BaseModel, Field
|
|
31
|
+
|
|
32
|
+
from argus_cortex.moderation._http import CheckFailed, post_json, service_url
|
|
33
|
+
|
|
34
|
+
# NudeNet's 18 detector classes, verbatim (they are its API vocabulary).
|
|
35
|
+
NudeNetLabel = Literal[
|
|
36
|
+
"FEMALE_GENITALIA_COVERED",
|
|
37
|
+
"FACE_FEMALE",
|
|
38
|
+
"BUTTOCKS_EXPOSED",
|
|
39
|
+
"FEMALE_BREAST_EXPOSED",
|
|
40
|
+
"FEMALE_GENITALIA_EXPOSED",
|
|
41
|
+
"MALE_BREAST_EXPOSED",
|
|
42
|
+
"ANUS_EXPOSED",
|
|
43
|
+
"FEET_EXPOSED",
|
|
44
|
+
"BELLY_COVERED",
|
|
45
|
+
"FEET_COVERED",
|
|
46
|
+
"ARMPITS_COVERED",
|
|
47
|
+
"ARMPITS_EXPOSED",
|
|
48
|
+
"FACE_MALE",
|
|
49
|
+
"BELLY_EXPOSED",
|
|
50
|
+
"MALE_GENITALIA_EXPOSED",
|
|
51
|
+
"ANUS_COVERED",
|
|
52
|
+
"FEMALE_BREAST_COVERED",
|
|
53
|
+
"BUTTOCKS_COVERED",
|
|
54
|
+
]
|
|
55
|
+
NUDENET_LABELS: tuple[NudeNetLabel, ...] = NudeNetLabel.__args__
|
|
56
|
+
# The classes an image's exposed score (its "total" NSFW score) is taken over.
|
|
57
|
+
DEFAULT_EXPOSED_LABELS: tuple[NudeNetLabel, ...] = (
|
|
58
|
+
"BUTTOCKS_EXPOSED",
|
|
59
|
+
"FEMALE_BREAST_EXPOSED",
|
|
60
|
+
"FEMALE_GENITALIA_EXPOSED",
|
|
61
|
+
"MALE_GENITALIA_EXPOSED",
|
|
62
|
+
"ANUS_EXPOSED",
|
|
63
|
+
)
|
|
64
|
+
# NudeNet servers report nothing scoring under this.
|
|
65
|
+
MIN_REPORTED_SCORE = 0.2
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class NsfwDetection(BaseModel):
|
|
69
|
+
"""One NudeNet detection."""
|
|
70
|
+
|
|
71
|
+
label: NudeNetLabel
|
|
72
|
+
score: float
|
|
73
|
+
bbox: list[float] = Field(..., min_length=4, max_length=4, description="[x, y, w, h]")
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class NsfwResult(BaseModel):
|
|
77
|
+
"""NudeNet's view of one image.
|
|
78
|
+
|
|
79
|
+
Only detections scoring >= :data:`MIN_REPORTED_SCORE` are kept, so a class
|
|
80
|
+
missing from ``labels`` scored below that. ``exposed_score`` is the highest score among
|
|
81
|
+
:data:`DEFAULT_EXPOSED_LABELS`; filters can recompute it over another set.
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
detections: list[NsfwDetection] = Field(default_factory=list)
|
|
85
|
+
# Highest score per class, highest first.
|
|
86
|
+
labels: dict[NudeNetLabel, float] = Field(default_factory=dict)
|
|
87
|
+
exposed_score: float = 0.0
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class NudeNetCheckFailed(CheckFailed):
|
|
91
|
+
"""One image could not be checked by NudeNet."""
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def exposed_score(labels: dict[str, float], exposed: tuple[str, ...] | list[str] = DEFAULT_EXPOSED_LABELS) -> float:
|
|
95
|
+
"""The highest score among the *exposed* classes (0.0 when none was detected)."""
|
|
96
|
+
return max((labels[c] for c in exposed if c in labels), default=0.0)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _is_number(value: object) -> bool:
|
|
100
|
+
# json.loads accepts bare NaN/Infinity; a NaN score would compare False
|
|
101
|
+
# against every threshold and pass the image as safe.
|
|
102
|
+
return isinstance(value, int | float) and not isinstance(value, bool) and math.isfinite(value)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def to_result(raw: list) -> NsfwResult:
|
|
106
|
+
"""One image's detections as recorded, plus the per-class maxima and exposed score."""
|
|
107
|
+
known = set(NUDENET_LABELS)
|
|
108
|
+
dets: list[NsfwDetection] = []
|
|
109
|
+
for d in raw:
|
|
110
|
+
if not isinstance(d, dict):
|
|
111
|
+
continue
|
|
112
|
+
label, score, box = d.get("class"), d.get("score"), d.get("box")
|
|
113
|
+
if (
|
|
114
|
+
not isinstance(label, str)
|
|
115
|
+
or label not in known
|
|
116
|
+
or not _is_number(score)
|
|
117
|
+
or score < MIN_REPORTED_SCORE
|
|
118
|
+
or not isinstance(box, list)
|
|
119
|
+
or len(box) != 4
|
|
120
|
+
or not all(_is_number(v) for v in box)
|
|
121
|
+
):
|
|
122
|
+
continue # an unknown class or malformed entry is not evidence either way
|
|
123
|
+
dets.append(NsfwDetection(label=label, score=round(float(score), 4), bbox=[round(float(v), 1) for v in box]))
|
|
124
|
+
dets.sort(key=lambda d: -d.score)
|
|
125
|
+
labels: dict[str, float] = {}
|
|
126
|
+
for d in dets:
|
|
127
|
+
labels.setdefault(d.label, d.score)
|
|
128
|
+
return NsfwResult(detections=dets, labels=labels, exposed_score=exposed_score(labels))
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _multipart(data: bytes, filename: str) -> tuple[bytes, str]:
|
|
132
|
+
# The name lands in a header: keep it to characters that cannot break it.
|
|
133
|
+
filename = re.sub(r"[^A-Za-z0-9._-]", "_", filename)[-100:] or "image"
|
|
134
|
+
boundary = uuid.uuid4().hex
|
|
135
|
+
head = (
|
|
136
|
+
f"--{boundary}\r\n"
|
|
137
|
+
f'Content-Disposition: form-data; name="f1"; filename="{filename}"\r\n'
|
|
138
|
+
"Content-Type: application/octet-stream\r\n\r\n"
|
|
139
|
+
).encode()
|
|
140
|
+
return head + data + f"\r\n--{boundary}--\r\n".encode(), f"multipart/form-data; boundary={boundary}"
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
class NudeNetClient:
|
|
144
|
+
"""A NudeNet server at *base_url* (its origin, e.g. ``http://nudenet:8080``).
|
|
145
|
+
|
|
146
|
+
Thread-safe: :meth:`check` holds no state, so one client can serve a pool.
|
|
147
|
+
"""
|
|
148
|
+
|
|
149
|
+
def __init__(self, base_url: str, timeout: float = 30.0) -> None:
|
|
150
|
+
self.url = service_url(base_url, "/infer", "NudeNet")
|
|
151
|
+
self.timeout = timeout
|
|
152
|
+
|
|
153
|
+
def check(self, data: bytes, filename: str = "image") -> NsfwResult:
|
|
154
|
+
"""Post one image and parse the answer; raises :class:`NudeNetCheckFailed`."""
|
|
155
|
+
body, content_type = _multipart(data, filename)
|
|
156
|
+
payload = post_json(self.url, body, content_type, self.timeout, "NudeNet", NudeNetCheckFailed)
|
|
157
|
+
prediction = payload.get("prediction") if isinstance(payload, dict) else None
|
|
158
|
+
if not (isinstance(payload, dict) and payload.get("success") is True and isinstance(prediction, list)):
|
|
159
|
+
raise NudeNetCheckFailed("NudeNet did not report success")
|
|
160
|
+
if len(prediction) != 1 or not isinstance(prediction[0], list):
|
|
161
|
+
raise NudeNetCheckFailed(f"NudeNet returned {len(prediction)} results for one image")
|
|
162
|
+
return to_result(prediction[0])
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
"""SingGuard client: a policy verdict from a SingGuard server over HTTP.
|
|
2
|
+
|
|
3
|
+
Where NudeNet scores nudity per body part, SingGuard judges an image (and
|
|
4
|
+
optionally text) against a whole moderation policy and answers with a verdict
|
|
5
|
+
and the rule that fired. Its default rules are A. Sexual Content Risk, B.
|
|
6
|
+
Real-World Crimes & Public Safety, C. Unethical Behavior (hate, self-harm,
|
|
7
|
+
gore), D. Cybersecurity, E. Agent Safety, F. Politically Sensitive and G.
|
|
8
|
+
Animal Abuse. A custom policy text replaces them. It gives no scores.
|
|
9
|
+
|
|
10
|
+
API: ``POST /api/guard/check`` with JSON ``{input, image_base64, policy,
|
|
11
|
+
thinking}``, answered by ``{safe, verdict, rule, explanation, categories:
|
|
12
|
+
[{code, label, why}], model, elapsed_ms}`` and, from the argus-cull
|
|
13
|
+
``singguard`` service, ``thinking``: the mode the server judged in, which
|
|
14
|
+
its ``SINGGUARD_FORCE_THINKING`` can make differ from the one asked for.
|
|
15
|
+
|
|
16
|
+
``thinking`` is ``fast`` (a verdict and rule), ``fast-slow`` (SingGuard's own
|
|
17
|
+
reasoning mode: a fast first-line verdict, then the reasoning and a final
|
|
18
|
+
answer), ``slow`` (the argus-cull service's variant with no first-line
|
|
19
|
+
verdict, so the reasoning is not anchored to a verdict made before it) or
|
|
20
|
+
``cascade`` (that service's ``fast`` for every image and ``slow`` for the ones
|
|
21
|
+
fast flags, slow's verdict standing). Servers without ``slow`` and ``cascade``
|
|
22
|
+
answer HTTP 400 to them.
|
|
23
|
+
|
|
24
|
+
Large images are downscaled before sending (Pillow, from the ``[images]``
|
|
25
|
+
extra, imported only when downscaling is on).
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import base64
|
|
31
|
+
import hashlib
|
|
32
|
+
import importlib.util
|
|
33
|
+
import io
|
|
34
|
+
import json
|
|
35
|
+
|
|
36
|
+
from pydantic import BaseModel, Field
|
|
37
|
+
|
|
38
|
+
from argus_cortex.moderation._http import CheckFailed, ModerationConfigError, post_json, service_url
|
|
39
|
+
|
|
40
|
+
THINKING_MODES = ("fast", "fast-slow", "slow", "cascade")
|
|
41
|
+
# The guard model sees far fewer pixels than a camera original, and base64
|
|
42
|
+
# JSON inflates every byte by 4/3.
|
|
43
|
+
MAX_EDGE = 1536
|
|
44
|
+
MAX_BYTES = 4 * 1024 * 1024
|
|
45
|
+
_MIN_EDGE = 256 # below this, give up rather than send SingGuard a postage stamp
|
|
46
|
+
_NEEDS_PILLOW = "downscaling images needs: pip install argus-cortex[images] (or pass downscale=False)"
|
|
47
|
+
_EXPLANATION_MAX = 4000
|
|
48
|
+
_FIELD_MAX = 200
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class GuardResult(BaseModel):
|
|
52
|
+
"""SingGuard's verdict on one image. A verdict, not a score: SingGuard
|
|
53
|
+
reports which policy rules fired, not how confident it is."""
|
|
54
|
+
|
|
55
|
+
safe: bool
|
|
56
|
+
# The rules that made the image unsafe, as the policy names them
|
|
57
|
+
# (e.g. "A. Sexual Content Risk"); empty when safe.
|
|
58
|
+
rules: list[str] = Field(default_factory=list)
|
|
59
|
+
explanation: str | None = None
|
|
60
|
+
# The guard model, and the policy judged against: "default" (SingGuard's
|
|
61
|
+
# own rules) or "custom-<sha256 prefix>" of the policy text.
|
|
62
|
+
model: str | None = None
|
|
63
|
+
policy: str = "default"
|
|
64
|
+
# The thinking mode the server judged in, when it says (the argus-cull
|
|
65
|
+
# service does; it can be forced server-side to differ from the request).
|
|
66
|
+
thinking: str | None = None
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class GuardCheckFailed(CheckFailed):
|
|
70
|
+
"""One image could not be checked by SingGuard."""
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def policy_name(text: str | None) -> str:
|
|
74
|
+
"""How a result records the policy: ``default``, or ``custom-`` + a sha256 prefix of *text*.
|
|
75
|
+
|
|
76
|
+
*text* is stripped first, as :class:`SingGuardClient` strips the policy it sends.
|
|
77
|
+
"""
|
|
78
|
+
return "default" if text is None else f"custom-{hashlib.sha256(text.strip().encode()).hexdigest()[:12]}"
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def rule_letter(rule: str) -> str:
|
|
82
|
+
"""*rule* by its letter (``"A"`` for ``"A. Sexual Content Risk"``), else in full."""
|
|
83
|
+
head, sep, _ = rule.strip().partition(".")
|
|
84
|
+
return head if sep and len(head) == 1 else rule
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def rule_matches(fired: str, wanted: str) -> bool:
|
|
88
|
+
"""*wanted* names *fired*: in full, or by its letter (``"A"`` or ``"A."`` for ``"A. Sexual Content Risk"``)."""
|
|
89
|
+
fired, wanted = fired.strip().casefold(), wanted.strip().casefold()
|
|
90
|
+
return fired == wanted or fired.startswith(f"{wanted.removesuffix('.')}.")
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _clip(value: object, limit: int) -> str | None:
|
|
94
|
+
if not isinstance(value, str) or not value.strip():
|
|
95
|
+
return None
|
|
96
|
+
return value.strip()[:limit]
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def to_result(payload: object, policy: str = "default") -> GuardResult:
|
|
100
|
+
"""SingGuard's answer as recorded. A missing verdict is a failure, not a pass."""
|
|
101
|
+
if not isinstance(payload, dict) or not isinstance(payload.get("safe"), bool):
|
|
102
|
+
raise GuardCheckFailed("SingGuard returned no verdict")
|
|
103
|
+
safe = payload["safe"]
|
|
104
|
+
rules: list[str] = []
|
|
105
|
+
if not safe:
|
|
106
|
+
categories = payload.get("categories")
|
|
107
|
+
for cat in categories if isinstance(categories, list) else []:
|
|
108
|
+
code = _clip(cat.get("code"), _FIELD_MAX) if isinstance(cat, dict) else None
|
|
109
|
+
if code and code not in rules:
|
|
110
|
+
rules.append(code)
|
|
111
|
+
rule = _clip(payload.get("rule"), _FIELD_MAX)
|
|
112
|
+
if not rules and rule:
|
|
113
|
+
rules.append(rule)
|
|
114
|
+
return GuardResult(
|
|
115
|
+
safe=safe,
|
|
116
|
+
rules=rules,
|
|
117
|
+
explanation=_clip(payload.get("explanation"), _EXPLANATION_MAX),
|
|
118
|
+
model=_clip(payload.get("model"), _FIELD_MAX),
|
|
119
|
+
policy=policy,
|
|
120
|
+
thinking=_clip(payload.get("thinking"), _FIELD_MAX),
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def downscale(data: bytes, max_edge: int = MAX_EDGE, max_bytes: int = MAX_BYTES) -> bytes:
|
|
125
|
+
"""*data* as is when small enough, else downscaled to a JPEG (a PNG when it
|
|
126
|
+
has alpha) of at most *max_edge* px and *max_bytes*, shrinking further if
|
|
127
|
+
the first encoding is still too large.
|
|
128
|
+
|
|
129
|
+
Raises :class:`GuardCheckFailed` for an image Pillow cannot read or shrink
|
|
130
|
+
under *max_bytes*.
|
|
131
|
+
"""
|
|
132
|
+
try:
|
|
133
|
+
from PIL import Image, ImageOps
|
|
134
|
+
except ImportError as exc: # pragma: no cover - exercised only without the extra
|
|
135
|
+
raise ModerationConfigError(_NEEDS_PILLOW) from exc
|
|
136
|
+
try:
|
|
137
|
+
with Image.open(io.BytesIO(data)) as im:
|
|
138
|
+
if max(im.size) <= max_edge and len(data) <= max_bytes:
|
|
139
|
+
return data
|
|
140
|
+
# Shrink first, so a JPEG decodes at reduced scale (draft mode);
|
|
141
|
+
# the box is square, so transposing after gives the same size.
|
|
142
|
+
im.thumbnail((max_edge, max_edge))
|
|
143
|
+
# Upright, as a viewer would see it: re-encoding drops the EXIF tag.
|
|
144
|
+
out = ImageOps.exif_transpose(im)
|
|
145
|
+
alpha = out.mode in ("RGBA", "LA", "PA") or (out.mode == "P" and "transparency" in out.info)
|
|
146
|
+
if not alpha:
|
|
147
|
+
out = out.convert("RGB")
|
|
148
|
+
edge = max(out.size)
|
|
149
|
+
while True:
|
|
150
|
+
buf = io.BytesIO()
|
|
151
|
+
if alpha:
|
|
152
|
+
# Flattening onto black would hide dark content drawn on a
|
|
153
|
+
# transparent background; keep the alpha, as a small PNG goes.
|
|
154
|
+
out.save(buf, "PNG", optimize=True)
|
|
155
|
+
else:
|
|
156
|
+
out.save(buf, "JPEG", quality=90)
|
|
157
|
+
if buf.tell() <= max_bytes:
|
|
158
|
+
return buf.getvalue()
|
|
159
|
+
if edge <= _MIN_EDGE:
|
|
160
|
+
raise GuardCheckFailed("image could not be shrunk under the SingGuard size limit")
|
|
161
|
+
edge = max(_MIN_EDGE, edge * 3 // 4)
|
|
162
|
+
out.thumbnail((edge, edge))
|
|
163
|
+
except (OSError, ValueError, Image.DecompressionBombError) as exc:
|
|
164
|
+
raise GuardCheckFailed(f"image could not be prepared for SingGuard: {exc}") from exc
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
class SingGuardClient:
|
|
168
|
+
"""A SingGuard server at *base_url* (its origin, e.g. ``http://guard:9150``).
|
|
169
|
+
|
|
170
|
+
*policy* is the policy text (``None`` = SingGuard's default rules);
|
|
171
|
+
*thinking* is one of :data:`THINKING_MODES`: ``fast`` (verdict and rule),
|
|
172
|
+
``fast-slow`` (also writes why), ``slow`` (writes why, with no fast
|
|
173
|
+
verdict ahead of the reasoning) or ``cascade`` (fast, then slow only for
|
|
174
|
+
what fast flagged); the last two need the argus-cull ``singguard`` service.
|
|
175
|
+
With *downscale* on (the default, needs ``[images]``), images over
|
|
176
|
+
:data:`MAX_EDGE` px or :data:`MAX_BYTES` are shrunk before sending.
|
|
177
|
+
Thread-safe: :meth:`check` holds no state.
|
|
178
|
+
"""
|
|
179
|
+
|
|
180
|
+
def __init__(
|
|
181
|
+
self,
|
|
182
|
+
base_url: str,
|
|
183
|
+
timeout: float = 90.0,
|
|
184
|
+
thinking: str = "fast",
|
|
185
|
+
policy: str | None = None,
|
|
186
|
+
downscale: bool = True,
|
|
187
|
+
) -> None:
|
|
188
|
+
if thinking not in THINKING_MODES:
|
|
189
|
+
raise ModerationConfigError(f"SingGuard thinking must be one of {', '.join(THINKING_MODES)}")
|
|
190
|
+
if policy is not None and not policy.strip():
|
|
191
|
+
raise ModerationConfigError("the SingGuard policy is empty (pass None for the default rules)")
|
|
192
|
+
if downscale and importlib.util.find_spec("PIL") is None:
|
|
193
|
+
# Fail at startup, not as a config error out of every check().
|
|
194
|
+
raise ModerationConfigError(_NEEDS_PILLOW)
|
|
195
|
+
self.url = service_url(base_url, "/api/guard/check", "SingGuard")
|
|
196
|
+
self.timeout = timeout
|
|
197
|
+
self.thinking = thinking
|
|
198
|
+
self.policy = policy.strip() if policy is not None else None
|
|
199
|
+
self.policy_name = policy_name(self.policy)
|
|
200
|
+
self.downscale = downscale
|
|
201
|
+
|
|
202
|
+
def check(self, data: bytes, text: str = "") -> GuardResult:
|
|
203
|
+
"""Judge one image, with *text* (e.g. the prompt that made it) judged alongside.
|
|
204
|
+
|
|
205
|
+
Raises :class:`GuardCheckFailed`.
|
|
206
|
+
"""
|
|
207
|
+
image = downscale(data) if self.downscale else data
|
|
208
|
+
body: dict[str, str] = {
|
|
209
|
+
"input": text,
|
|
210
|
+
"image_base64": base64.b64encode(image).decode("ascii"),
|
|
211
|
+
"thinking": self.thinking,
|
|
212
|
+
}
|
|
213
|
+
if self.policy is not None:
|
|
214
|
+
body["policy"] = self.policy
|
|
215
|
+
payload = post_json(
|
|
216
|
+
self.url, json.dumps(body).encode(), "application/json", self.timeout, "SingGuard", GuardCheckFailed
|
|
217
|
+
)
|
|
218
|
+
return to_result(payload, self.policy_name)
|
|
@@ -0,0 +1,359 @@
|
|
|
1
|
+
"""Moderation clients against fake NudeNet and SingGuard servers.
|
|
2
|
+
|
|
3
|
+
The fakes speak each server's API (shapes taken from live servers), so the
|
|
4
|
+
real urllib path, multipart/JSON encoding and response parsing all run.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import base64
|
|
10
|
+
import io
|
|
11
|
+
import json
|
|
12
|
+
import subprocess
|
|
13
|
+
import sys
|
|
14
|
+
import threading
|
|
15
|
+
from collections.abc import Iterator
|
|
16
|
+
from email.parser import BytesParser
|
|
17
|
+
from email.policy import HTTP
|
|
18
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
19
|
+
|
|
20
|
+
import pytest
|
|
21
|
+
from PIL import Image
|
|
22
|
+
|
|
23
|
+
from argus_cortex.moderation import (
|
|
24
|
+
DEFAULT_EXPOSED_LABELS,
|
|
25
|
+
GuardCheckFailed,
|
|
26
|
+
GuardResult,
|
|
27
|
+
ModerationConfigError,
|
|
28
|
+
NsfwResult,
|
|
29
|
+
NudeNetCheckFailed,
|
|
30
|
+
NudeNetClient,
|
|
31
|
+
SingGuardClient,
|
|
32
|
+
exposed_score,
|
|
33
|
+
nudenet,
|
|
34
|
+
policy_name,
|
|
35
|
+
rule_letter,
|
|
36
|
+
rule_matches,
|
|
37
|
+
singguard,
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
_FAIL = 99 # this red value gets an HTTP 500 from either fake
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _png(red: int, size: tuple[int, int] = (64, 48), mode: str = "RGB") -> bytes:
|
|
44
|
+
buf = io.BytesIO()
|
|
45
|
+
Image.new(mode, size, (red, 0, 0, 255)[: len(mode)]).save(buf, "PNG")
|
|
46
|
+
return buf.getvalue()
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _red(data: bytes) -> int:
|
|
50
|
+
return Image.open(io.BytesIO(data)).convert("RGB").getpixel((0, 0))[0]
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class _Fake(BaseHTTPRequestHandler):
|
|
54
|
+
seen: list[dict] = []
|
|
55
|
+
|
|
56
|
+
def log_message(self, *args) -> None: # keep test output quiet
|
|
57
|
+
pass
|
|
58
|
+
|
|
59
|
+
def _send(self, status: int, payload: object | None = None) -> None:
|
|
60
|
+
out = json.dumps(payload).encode() if payload is not None else b""
|
|
61
|
+
self.send_response(status)
|
|
62
|
+
self.send_header("Content-Type", "application/json")
|
|
63
|
+
self.send_header("Content-Length", str(len(out)))
|
|
64
|
+
self.end_headers()
|
|
65
|
+
self.wfile.write(out)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
# ---------------------------------------------------------------------------
|
|
69
|
+
# NudeNet
|
|
70
|
+
# ---------------------------------------------------------------------------
|
|
71
|
+
|
|
72
|
+
_DETECTIONS = {
|
|
73
|
+
10: [],
|
|
74
|
+
20: [
|
|
75
|
+
{"class": "FACE_FEMALE", "score": 0.91, "box": [10, 10, 30, 30]},
|
|
76
|
+
{"class": "BELLY_EXPOSED", "score": 0.4, "box": [10, 50, 30, 20]},
|
|
77
|
+
],
|
|
78
|
+
30: [
|
|
79
|
+
{"class": "FEMALE_BREAST_EXPOSED", "score": 0.72, "box": [5, 5, 20, 20]},
|
|
80
|
+
{"class": "FEMALE_BREAST_EXPOSED", "score": 0.66, "box": [30, 5, 20, 20]},
|
|
81
|
+
{"class": "FACE_FEMALE", "score": 0.8, "box": [10, 0, 10, 10]},
|
|
82
|
+
],
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class _NudeNet(_Fake):
|
|
87
|
+
def do_POST(self) -> None:
|
|
88
|
+
body = self.rfile.read(int(self.headers["Content-Length"]))
|
|
89
|
+
msg = BytesParser(policy=HTTP).parsebytes(
|
|
90
|
+
f"Content-Type: {self.headers['Content-Type']}\r\n\r\n".encode() + body
|
|
91
|
+
)
|
|
92
|
+
part = next(msg.iter_parts())
|
|
93
|
+
red = _red(part.get_content())
|
|
94
|
+
type(self).seen.append(
|
|
95
|
+
{"path": self.path, "field": part.get_param("name", header="content-disposition"),
|
|
96
|
+
"filename": part.get_filename(), "auth": self.headers["Authorization"]}
|
|
97
|
+
) # fmt: skip
|
|
98
|
+
if red == _FAIL:
|
|
99
|
+
self._send(500)
|
|
100
|
+
else:
|
|
101
|
+
self._send(200, {"prediction": [_DETECTIONS.get(red, [])], "success": True})
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class _SingGuard(_Fake):
|
|
105
|
+
answers: dict[int, dict] = {} # a test's overrides of _GUARD_ANSWERS, by red value
|
|
106
|
+
|
|
107
|
+
def do_POST(self) -> None:
|
|
108
|
+
body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
|
|
109
|
+
image = Image.open(io.BytesIO(base64.b64decode(body["image_base64"])))
|
|
110
|
+
red = image.convert("RGB").getpixel((0, 0))[0]
|
|
111
|
+
type(self).seen.append(
|
|
112
|
+
{"path": self.path, "size": image.size, "format": image.format,
|
|
113
|
+
**{k: v for k, v in body.items() if k != "image_base64"}}
|
|
114
|
+
) # fmt: skip
|
|
115
|
+
if red == _FAIL:
|
|
116
|
+
self._send(500)
|
|
117
|
+
else:
|
|
118
|
+
answer = self.answers.get(red) or _GUARD_ANSWERS.get(red, _GUARD_ANSWERS[10])
|
|
119
|
+
self._send(200, {**answer, "model": "SingGuard-4b", "elapsed_ms": 5})
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _serve(handler: type[_Fake]) -> Iterator[tuple[type[_Fake], str]]:
|
|
123
|
+
handler.seen = []
|
|
124
|
+
server = ThreadingHTTPServer(("127.0.0.1", 0), handler)
|
|
125
|
+
threading.Thread(target=server.serve_forever, daemon=True).start()
|
|
126
|
+
try:
|
|
127
|
+
yield handler, f"http://127.0.0.1:{server.server_port}"
|
|
128
|
+
finally:
|
|
129
|
+
server.shutdown()
|
|
130
|
+
server.server_close()
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
@pytest.fixture
|
|
134
|
+
def nudenet_server() -> Iterator[tuple[type[_Fake], str]]:
|
|
135
|
+
yield from _serve(_NudeNet)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
@pytest.fixture
|
|
139
|
+
def singguard_server() -> Iterator[tuple[type[_Fake], str]]:
|
|
140
|
+
yield from _serve(_SingGuard)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def test_nudenet_url() -> None:
|
|
144
|
+
assert NudeNetClient("http://nudenet:8080/").url == "http://nudenet:8080/infer"
|
|
145
|
+
assert NudeNetClient("https://n.lan/x/infer").url == "https://n.lan/x/infer"
|
|
146
|
+
with pytest.raises(ModerationConfigError) as exc:
|
|
147
|
+
NudeNetClient("ftp://user:secret@nudenet")
|
|
148
|
+
assert "secret" not in str(exc.value) # callers surface this message
|
|
149
|
+
# A host named like the path is still a host, and a query stays a query.
|
|
150
|
+
assert NudeNetClient("http://infer").url == "http://infer/infer"
|
|
151
|
+
assert NudeNetClient("http://h:8080?token=x").url == "http://h:8080/infer?token=x"
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def test_credentials_in_the_url_become_basic_auth(nudenet_server) -> None:
|
|
155
|
+
fake, url = nudenet_server
|
|
156
|
+
NudeNetClient(url.replace("http://", "http://us%40er:p%3Aw@"), timeout=5).check(_png(10))
|
|
157
|
+
assert fake.seen[0]["auth"] == "Basic " + base64.b64encode(b"us@er:p:w").decode()
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def test_nudenet_check(nudenet_server) -> None:
|
|
161
|
+
fake, url = nudenet_server
|
|
162
|
+
client = NudeNetClient(url, timeout=5)
|
|
163
|
+
assert client.check(_png(10)) == NsfwResult()
|
|
164
|
+
portrait = client.check(_png(20), 'we"ird\r\nname.png')
|
|
165
|
+
assert portrait.exposed_score == 0.0 # a face and a belly are not exposed nudity
|
|
166
|
+
assert portrait.labels == {"FACE_FEMALE": 0.91, "BELLY_EXPOSED": 0.4}
|
|
167
|
+
topless = client.check(_png(30))
|
|
168
|
+
assert topless.exposed_score == 0.72 and len(topless.detections) == 3
|
|
169
|
+
assert list(topless.labels) == ["FACE_FEMALE", "FEMALE_BREAST_EXPOSED"] # highest first
|
|
170
|
+
with pytest.raises(NudeNetCheckFailed, match="HTTP 500"):
|
|
171
|
+
client.check(_png(_FAIL))
|
|
172
|
+
assert {s["path"] for s in fake.seen} == {"/infer"} and {s["field"] for s in fake.seen} == {"f1"}
|
|
173
|
+
assert "we_ird__name.png" in {s["filename"] for s in fake.seen} # header-safe
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def test_nudenet_parsing_ignores_malformed_entries() -> None:
|
|
177
|
+
r = nudenet.to_result(
|
|
178
|
+
[
|
|
179
|
+
{"class": "BUTTOCKS_EXPOSED", "score": 0.6, "box": [0, 0, 1, 1]},
|
|
180
|
+
{"class": "NOT_A_CLASS", "score": 0.99, "box": [0, 0, 1, 1]},
|
|
181
|
+
{"class": "ANUS_EXPOSED", "score": "high", "box": [0, 0, 1, 1]},
|
|
182
|
+
{"class": "ANUS_EXPOSED", "score": True, "box": [0, 0, 1, 1]},
|
|
183
|
+
{"class": "ANUS_EXPOSED", "score": 0.9, "box": [0, 0, 1]},
|
|
184
|
+
{"class": "ANUS_EXPOSED", "score": float("nan"), "box": [0, 0, 1, 1]},
|
|
185
|
+
{"class": "ANUS_EXPOSED", "score": 0.9, "box": [0, 0, float("inf"), 1]},
|
|
186
|
+
{"class": "ANUS_EXPOSED", "score": nudenet.MIN_REPORTED_SCORE / 2, "box": [0, 0, 1, 1]},
|
|
187
|
+
{"class": ["ANUS_EXPOSED"], "score": 0.9, "box": [0, 0, 1, 1]},
|
|
188
|
+
{"class": {"x": 1}, "score": 0.9, "box": [0, 0, 1, 1]},
|
|
189
|
+
"not a detection",
|
|
190
|
+
]
|
|
191
|
+
)
|
|
192
|
+
assert r.labels == {"BUTTOCKS_EXPOSED": 0.6} and r.exposed_score == 0.6
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
@pytest.mark.parametrize(
|
|
196
|
+
"payload",
|
|
197
|
+
[{"prediction": [[]], "success": False}, {"prediction": [[], []], "success": True}, {"success": True}, ["x"]],
|
|
198
|
+
)
|
|
199
|
+
def test_nudenet_bad_responses_are_failures(monkeypatch: pytest.MonkeyPatch, payload) -> None:
|
|
200
|
+
monkeypatch.setattr(nudenet, "post_json", lambda *a, **k: payload)
|
|
201
|
+
with pytest.raises(NudeNetCheckFailed):
|
|
202
|
+
NudeNetClient("http://n").check(_png(10))
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def test_exposed_set_is_the_five_exposed_nudity_classes() -> None:
|
|
206
|
+
assert set(DEFAULT_EXPOSED_LABELS) == {
|
|
207
|
+
"BUTTOCKS_EXPOSED", "FEMALE_BREAST_EXPOSED", "FEMALE_GENITALIA_EXPOSED", "MALE_GENITALIA_EXPOSED",
|
|
208
|
+
"ANUS_EXPOSED",
|
|
209
|
+
} # fmt: skip
|
|
210
|
+
labels = {"FEET_EXPOSED": 0.9, "MALE_BREAST_EXPOSED": 0.9, "BUTTOCKS_EXPOSED": 0.3}
|
|
211
|
+
assert exposed_score(labels) == 0.3
|
|
212
|
+
assert exposed_score(labels, ["MALE_BREAST_EXPOSED"]) == 0.9
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
# ---------------------------------------------------------------------------
|
|
216
|
+
# SingGuard
|
|
217
|
+
# ---------------------------------------------------------------------------
|
|
218
|
+
|
|
219
|
+
_CRIME = "B. Real-World Crimes & Public Safety"
|
|
220
|
+
_GORE = "C. Unethical Behavior"
|
|
221
|
+
_GUARD_ANSWERS: dict[int, dict] = {
|
|
222
|
+
10: {"safe": True, "verdict": "safe", "rule": "Safe", "explanation": "Safe.", "categories": []},
|
|
223
|
+
20: {
|
|
224
|
+
"safe": False,
|
|
225
|
+
"verdict": "unsafe",
|
|
226
|
+
"rule": _CRIME,
|
|
227
|
+
"explanation": f"Triggered rule: {_CRIME}.",
|
|
228
|
+
"categories": [{"code": _CRIME, "label": _CRIME, "why": "-"}, {"code": _GORE, "label": _GORE, "why": "-"}],
|
|
229
|
+
},
|
|
230
|
+
30: {"safe": False, "verdict": "unsafe", "rule": _GORE, "explanation": "", "categories": 5}, # malformed
|
|
231
|
+
40: {"verdict": "safe"}, # no boolean verdict: a failed check, not a pass
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def test_singguard_config() -> None:
|
|
236
|
+
client = SingGuardClient("http://192.168.1.198:9150/")
|
|
237
|
+
assert client.url == "http://192.168.1.198:9150/api/guard/check"
|
|
238
|
+
assert client.policy is None and client.policy_name == "default" and client.thinking == "fast"
|
|
239
|
+
custom = SingGuardClient("http://g", policy=" ### A. Minors\n")
|
|
240
|
+
assert custom.policy == "### A. Minors" and custom.policy_name == policy_name("### A. Minors")
|
|
241
|
+
assert custom.policy_name == policy_name(" ### A. Minors\n") # a policy read from a file matches
|
|
242
|
+
assert custom.policy_name.startswith("custom-") and len(custom.policy_name) == len("custom-") + 12
|
|
243
|
+
assert [SingGuardClient("http://g", thinking=m).thinking for m in ("slow", "cascade")] == ["slow", "cascade"]
|
|
244
|
+
for kwargs in ({"thinking": "slower"}, {"thinking": "FAST"}, {"policy": " \n"}):
|
|
245
|
+
with pytest.raises(ModerationConfigError):
|
|
246
|
+
SingGuardClient("http://g", **kwargs)
|
|
247
|
+
with pytest.raises(ModerationConfigError):
|
|
248
|
+
SingGuardClient("gopher://g")
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def test_singguard_check(singguard_server) -> None:
|
|
252
|
+
fake, url = singguard_server
|
|
253
|
+
client = SingGuardClient(url, timeout=5, thinking="fast-slow", policy="### A. Minors")
|
|
254
|
+
safe = client.check(_png(10), text="a cat on a sofa")
|
|
255
|
+
assert safe == GuardResult(safe=True, explanation="Safe.", model="SingGuard-4b", policy=client.policy_name)
|
|
256
|
+
assert client.check(_png(20)).rules == [_CRIME, _GORE]
|
|
257
|
+
gore = client.check(_png(30))
|
|
258
|
+
assert gore.rules == [_GORE] and gore.explanation is None # no categories: the rule itself
|
|
259
|
+
with pytest.raises(GuardCheckFailed, match="no verdict"):
|
|
260
|
+
client.check(_png(40))
|
|
261
|
+
with pytest.raises(GuardCheckFailed, match="HTTP 500"):
|
|
262
|
+
client.check(_png(_FAIL))
|
|
263
|
+
assert {s["path"] for s in fake.seen} == {"/api/guard/check"}
|
|
264
|
+
assert fake.seen[0]["input"] == "a cat on a sofa" and fake.seen[1]["input"] == ""
|
|
265
|
+
assert {s["thinking"] for s in fake.seen} == {"fast-slow"} and {s["policy"] for s in fake.seen} == {"### A. Minors"}
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def test_singguard_default_policy_is_not_sent(singguard_server) -> None:
|
|
269
|
+
fake, url = singguard_server
|
|
270
|
+
SingGuardClient(url).check(_png(10))
|
|
271
|
+
assert "policy" not in fake.seen[0]
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def test_singguard_slow_is_sent_and_the_mode_used_is_recorded(singguard_server) -> None:
|
|
275
|
+
"""The argus-cull service answers with the mode it judged in, which a
|
|
276
|
+
server-side force can make differ from the request; it is recorded as sent.
|
|
277
|
+
A server that says nothing leaves it None."""
|
|
278
|
+
fake, url = singguard_server
|
|
279
|
+
client = SingGuardClient(url, timeout=5, thinking="slow")
|
|
280
|
+
assert client.check(_png(10)).thinking is None
|
|
281
|
+
fake.answers = {10: {**_GUARD_ANSWERS[10], "thinking": "fast-slow"}}
|
|
282
|
+
try:
|
|
283
|
+
assert client.check(_png(10)).thinking == "fast-slow"
|
|
284
|
+
finally:
|
|
285
|
+
fake.answers = {}
|
|
286
|
+
assert {s["thinking"] for s in fake.seen} == {"slow"}
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def test_singguard_downscales_large_images(singguard_server) -> None:
|
|
290
|
+
fake, url = singguard_server
|
|
291
|
+
SingGuardClient(url).check(_png(10, (4000, 3000)))
|
|
292
|
+
SingGuardClient(url).check(_png(10, (4000, 3000), "RGBA"))
|
|
293
|
+
SingGuardClient(url, downscale=False).check(_png(10, (4000, 3000)))
|
|
294
|
+
assert [(s["size"], s["format"]) for s in fake.seen] == [
|
|
295
|
+
((1536, 1152), "JPEG"),
|
|
296
|
+
((1536, 1152), "PNG"), # alpha kept, not flattened onto black
|
|
297
|
+
((4000, 3000), "PNG"),
|
|
298
|
+
]
|
|
299
|
+
small = _png(10)
|
|
300
|
+
assert singguard.downscale(small) is small
|
|
301
|
+
exif = Image.Exif()
|
|
302
|
+
exif[0x0112] = 6 # rotate 90 degrees to view
|
|
303
|
+
buf = io.BytesIO()
|
|
304
|
+
Image.new("RGB", (4000, 3000)).save(buf, "JPEG", exif=exif)
|
|
305
|
+
assert Image.open(io.BytesIO(singguard.downscale(buf.getvalue()))).size == (1152, 1536) # upright
|
|
306
|
+
with pytest.raises(GuardCheckFailed, match="prepared"):
|
|
307
|
+
singguard.downscale(b"not an image")
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def test_downscale_enforces_the_byte_limit() -> None:
|
|
311
|
+
noisy = Image.effect_noise((1200, 900), 100).convert("RGB")
|
|
312
|
+
buf = io.BytesIO()
|
|
313
|
+
noisy.save(buf, "PNG")
|
|
314
|
+
out = singguard.downscale(buf.getvalue(), max_bytes=100_000)
|
|
315
|
+
assert len(out) <= 100_000 and max(Image.open(io.BytesIO(out)).size) < 1200
|
|
316
|
+
with pytest.raises(GuardCheckFailed, match="size limit"):
|
|
317
|
+
singguard.downscale(buf.getvalue(), max_bytes=100)
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def test_singguard_without_pillow_fails_at_construction(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
321
|
+
monkeypatch.setattr(singguard.importlib.util, "find_spec", lambda name: None)
|
|
322
|
+
with pytest.raises(ModerationConfigError, match="images"):
|
|
323
|
+
SingGuardClient("http://g")
|
|
324
|
+
assert SingGuardClient("http://g", downscale=False).downscale is False
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
@pytest.mark.parametrize(
|
|
328
|
+
("fired", "wanted", "expected"),
|
|
329
|
+
[(_CRIME, "B", True), (_CRIME, "b.", True), (_CRIME, _CRIME.lower(), True), (_CRIME, "B. Real", False),
|
|
330
|
+
(_CRIME, "C", False), ("Minors", "minors", True)],
|
|
331
|
+
) # fmt: skip
|
|
332
|
+
def test_rule_matches(fired: str, wanted: str, expected: bool) -> None:
|
|
333
|
+
assert rule_matches(fired, wanted) is expected
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def test_rule_letter() -> None:
|
|
337
|
+
assert rule_letter(_CRIME) == "B" and rule_letter("Minors") == "Minors"
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
# ---------------------------------------------------------------------------
|
|
341
|
+
# Transport and packaging
|
|
342
|
+
# ---------------------------------------------------------------------------
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def test_proxies_in_the_environment_are_ignored(nudenet_server, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
346
|
+
_fake, url = nudenet_server
|
|
347
|
+
monkeypatch.setenv("HTTP_PROXY", "http://127.0.0.1:9") # nothing listens there
|
|
348
|
+
monkeypatch.setenv("http_proxy", "http://127.0.0.1:9")
|
|
349
|
+
assert NudeNetClient(url, timeout=5).check(_png(10)) == NsfwResult()
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def test_unreachable_server_is_a_failed_check() -> None:
|
|
353
|
+
with pytest.raises(NudeNetCheckFailed, match="request failed"):
|
|
354
|
+
NudeNetClient("http://127.0.0.1:9", timeout=2).check(_png(10))
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def test_import_does_not_pull_in_pillow() -> None:
|
|
358
|
+
code = "import sys, argus_cortex, argus_cortex.moderation; assert 'PIL' not in sys.modules"
|
|
359
|
+
subprocess.run([sys.executable, "-c", code], check=True)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|