argus-cortex 0.2.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- argus_cortex-0.4.0/.github/workflows/release.yml +120 -0
- argus_cortex-0.4.0/CHANGELOG.md +110 -0
- argus_cortex-0.4.0/CLAUDE.md +49 -0
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/PKG-INFO +43 -2
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/README.md +33 -0
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/pyproject.toml +9 -3
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/src/argus_cortex/__init__.py +38 -0
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/src/argus_cortex/_version.py +2 -2
- argus_cortex-0.4.0/src/argus_cortex/moderation/__init__.py +50 -0
- argus_cortex-0.4.0/src/argus_cortex/moderation/_http.py +72 -0
- argus_cortex-0.4.0/src/argus_cortex/moderation/nudenet.py +162 -0
- argus_cortex-0.4.0/src/argus_cortex/moderation/singguard.py +201 -0
- argus_cortex-0.4.0/src/argus_cortex/xmp.py +409 -0
- argus_cortex-0.4.0/tests/test_moderation.py +342 -0
- argus_cortex-0.4.0/tests/test_xmp.py +157 -0
- argus_cortex-0.2.0/.github/workflows/release.yml +0 -72
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/.copier-answers.yml +0 -0
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/.env.example +0 -0
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/.github/workflows/ci.yml +0 -0
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/.gitignore +0 -0
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/LICENSE +0 -0
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/Makefile +0 -0
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/src/argus_cortex/backends.py +0 -0
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/src/argus_cortex/py.typed +0 -0
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/src/argus_cortex/server.py +0 -0
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/src/argus_cortex/store/__init__.py +0 -0
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/src/argus_cortex/store/blob.py +0 -0
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/src/argus_cortex/store/config.py +0 -0
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/src/argus_cortex/store/errors.py +0 -0
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/src/argus_cortex/store/lineage.py +0 -0
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/src/argus_cortex/store/models.py +0 -0
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/src/argus_cortex/store/schema.py +0 -0
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/src/argus_cortex/store/vector.py +0 -0
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/src/argus_cortex/taxonomy.py +0 -0
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/src/argus_cortex/wire.py +0 -0
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/tests/test_backends.py +0 -0
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/tests/test_blob.py +0 -0
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/tests/test_server.py +0 -0
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/tests/test_smoke.py +0 -0
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/tests/test_store.py +0 -0
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/tests/test_store_pg_integration.py +0 -0
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/tests/test_taxonomy.py +0 -0
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/tests/test_vector.py +0 -0
- {argus_cortex-0.2.0 → argus_cortex-0.4.0}/tests/test_wire.py +0 -0
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
name: Release to PyPI
|
|
2
|
+
|
|
3
|
+
# Top-level (non-reusable) on purpose: PyPI trusted publishing cannot run inside
|
|
4
|
+
# a reusable workflow. Keep this file in sync across the suite via `copier update`.
|
|
5
|
+
|
|
6
|
+
on:
|
|
7
|
+
push:
|
|
8
|
+
tags: ["v*"]
|
|
9
|
+
|
|
10
|
+
permissions:
|
|
11
|
+
contents: read
|
|
12
|
+
id-token: write
|
|
13
|
+
|
|
14
|
+
jobs:
|
|
15
|
+
build:
|
|
16
|
+
runs-on: ubuntu-latest
|
|
17
|
+
steps:
|
|
18
|
+
- uses: actions/checkout@v4
|
|
19
|
+
with:
|
|
20
|
+
fetch-depth: 0
|
|
21
|
+
- uses: astral-sh/setup-uv@v6
|
|
22
|
+
with:
|
|
23
|
+
enable-cache: true
|
|
24
|
+
- run: uv build
|
|
25
|
+
- uses: actions/upload-artifact@v4
|
|
26
|
+
with:
|
|
27
|
+
name: dist
|
|
28
|
+
path: dist/
|
|
29
|
+
|
|
30
|
+
test-install:
|
|
31
|
+
runs-on: ubuntu-latest
|
|
32
|
+
needs: build
|
|
33
|
+
strategy:
|
|
34
|
+
matrix:
|
|
35
|
+
python-version: ["3.11", "3.12"]
|
|
36
|
+
steps:
|
|
37
|
+
- uses: actions/download-artifact@v4
|
|
38
|
+
with:
|
|
39
|
+
name: dist
|
|
40
|
+
path: dist/
|
|
41
|
+
- uses: astral-sh/setup-uv@v6
|
|
42
|
+
with:
|
|
43
|
+
enable-cache: true
|
|
44
|
+
- name: Install wheel and smoke-test import
|
|
45
|
+
run: |
|
|
46
|
+
uv venv --python ${{ matrix.python-version }}
|
|
47
|
+
uv pip install dist/*.whl
|
|
48
|
+
uv run --no-sync python -c "import argus_cortex; print(f'argus-cortex {argus_cortex.__version__} OK')"
|
|
49
|
+
|
|
50
|
+
publish-testpypi:
|
|
51
|
+
runs-on: ubuntu-latest
|
|
52
|
+
needs: test-install
|
|
53
|
+
environment: testpypi
|
|
54
|
+
steps:
|
|
55
|
+
- uses: actions/download-artifact@v4
|
|
56
|
+
with:
|
|
57
|
+
name: dist
|
|
58
|
+
path: dist/
|
|
59
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
60
|
+
with:
|
|
61
|
+
repository-url: https://test.pypi.org/legacy/
|
|
62
|
+
|
|
63
|
+
publish-pypi:
|
|
64
|
+
runs-on: ubuntu-latest
|
|
65
|
+
# Ordered *after* the TestPyPI rehearsal so its outcome is visible first, but
|
|
66
|
+
# deliberately not gated on it. A TestPyPI-only fault — a missing trusted
|
|
67
|
+
# publisher, an index outage — must not silently withhold the real release:
|
|
68
|
+
# that failure mode shipped argus-proof images for two tags with no wheel on
|
|
69
|
+
# PyPI, and nothing surfaced it because publish-image is independent.
|
|
70
|
+
# Only the build + install gate is required; a TestPyPI failure still marks
|
|
71
|
+
# the whole run failed, so the problem stays loud.
|
|
72
|
+
needs: [test-install, publish-testpypi]
|
|
73
|
+
if: "!cancelled() && needs.test-install.result == 'success'"
|
|
74
|
+
environment: pypi
|
|
75
|
+
steps:
|
|
76
|
+
- uses: actions/download-artifact@v4
|
|
77
|
+
with:
|
|
78
|
+
name: dist
|
|
79
|
+
path: dist/
|
|
80
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
81
|
+
|
|
82
|
+
github-release:
|
|
83
|
+
# The GitHub Release is the last step, so it only appears once the version is
|
|
84
|
+
# actually on PyPI. Its notes are this version's CHANGELOG.md section.
|
|
85
|
+
# Gated on publish-pypi's own result, not the default success(): publish-pypi
|
|
86
|
+
# runs even when the TestPyPI rehearsal failed, and that must not skip this.
|
|
87
|
+
runs-on: ubuntu-latest
|
|
88
|
+
needs: publish-pypi
|
|
89
|
+
if: "!cancelled() && needs.publish-pypi.result == 'success'"
|
|
90
|
+
permissions:
|
|
91
|
+
contents: write
|
|
92
|
+
steps:
|
|
93
|
+
- uses: actions/checkout@v4
|
|
94
|
+
- uses: actions/download-artifact@v4
|
|
95
|
+
with:
|
|
96
|
+
name: dist
|
|
97
|
+
path: dist/
|
|
98
|
+
- name: Extract release notes from CHANGELOG.md
|
|
99
|
+
run: |
|
|
100
|
+
version="${GITHUB_REF_NAME#v}"
|
|
101
|
+
awk -v head="## [${version}]" 'index($0, head) == 1 {found = 1; next} found && /^## \[/ {exit} found' \
|
|
102
|
+
CHANGELOG.md > notes.md
|
|
103
|
+
if ! grep -q '[^[:space:]]' notes.md; then
|
|
104
|
+
echo "::error::CHANGELOG.md has no section for ${version}"
|
|
105
|
+
exit 1
|
|
106
|
+
fi
|
|
107
|
+
- name: Create or update the GitHub Release
|
|
108
|
+
env:
|
|
109
|
+
GH_TOKEN: ${{ github.token }}
|
|
110
|
+
run: |
|
|
111
|
+
tag="${GITHUB_REF_NAME}"
|
|
112
|
+
prerelease=""
|
|
113
|
+
case "$tag" in *-*|*rc*|*a[0-9]*|*b[0-9]*) prerelease="--prerelease" ;; esac
|
|
114
|
+
if gh release view "$tag" >/dev/null 2>&1; then
|
|
115
|
+
# Re-run of the workflow: refresh notes and assets rather than failing.
|
|
116
|
+
gh release edit "$tag" --notes-file notes.md
|
|
117
|
+
gh release upload "$tag" dist/* --clobber
|
|
118
|
+
else
|
|
119
|
+
gh release create "$tag" dist/* --verify-tag --title "$tag" --notes-file notes.md $prerelease
|
|
120
|
+
fi
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## [Unreleased]
|
|
9
|
+
|
|
10
|
+
## [0.4.0] - 2026-10-26
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
- **`argus_cortex.moderation`** — content-moderation clients shared by the
|
|
14
|
+
stages, hoisted from argus-curator. Both talk HTTP to a server the deployer
|
|
15
|
+
runs; nothing imports a detector.
|
|
16
|
+
- `NudeNetClient(base_url, timeout)` — a NudeNet server's `/infer`, returning
|
|
17
|
+
`NsfwResult` (detections, highest score per class, exposed score) and
|
|
18
|
+
raising `NudeNetCheckFailed` per image. `NudeNetLabel` / `NUDENET_LABELS`
|
|
19
|
+
are NudeNet's 18 classes; `DEFAULT_EXPOSED_LABELS` (the five exposed
|
|
20
|
+
buttocks/breast/genitalia/anus classes) and `exposed_score()` are the
|
|
21
|
+
suite's one definition of nudity.
|
|
22
|
+
- `SingGuardClient(base_url, timeout, thinking, policy, downscale)` — a
|
|
23
|
+
SingGuard server's `/api/guard/check`, returning `GuardResult` (verdict,
|
|
24
|
+
the rules that fired, explanation, model, policy name) and raising
|
|
25
|
+
`GuardCheckFailed`. `rule_matches()` / `rule_letter()` match rules by
|
|
26
|
+
letter or full name; `policy_name()` names a custom policy by a sha256
|
|
27
|
+
prefix.
|
|
28
|
+
- Proxies in the environment are ignored: images go only to the configured
|
|
29
|
+
server. Config errors (`ModerationConfigError`) never echo the URL. A
|
|
30
|
+
`user:password@` in the URL is sent as HTTP Basic auth.
|
|
31
|
+
- `[images]` extra (Pillow), for `SingGuardClient`'s downscaling of images
|
|
32
|
+
over 1536 px / 4 MB; imported only when it downscales. Downscaling is on by
|
|
33
|
+
default, so without the extra `SingGuardClient(...)` raises
|
|
34
|
+
`ModerationConfigError` at construction unless given `downscale=False`.
|
|
35
|
+
|
|
36
|
+
### Changed
|
|
37
|
+
- `release.yml` now creates the **GitHub Release** for each `v*` tag once the
|
|
38
|
+
version is on PyPI, with that version's `CHANGELOG.md` section as notes and
|
|
39
|
+
the built sdist/wheel attached (a missing section fails the job; a re-run
|
|
40
|
+
edits the Release). Tags with `-`, `rc`, `a<n>` or `b<n>` are pre-releases.
|
|
41
|
+
|
|
42
|
+
## [0.3.0] - 2026-10-25
|
|
43
|
+
|
|
44
|
+
### Added
|
|
45
|
+
- **`argus_cortex.xmp`** — the suite's one XMP sidecar implementation, hoisted
|
|
46
|
+
from argus-lens so lens (keywords/caption) and argus-curator (face regions,
|
|
47
|
+
content rating) share a writer. `write_xmp()` / `read_xmp()` /
|
|
48
|
+
`merge_xmp()` / `parse_xmp()` / `render_xmp()` / `sidecar_path()` over an
|
|
49
|
+
`XmpData` model, all exported from the top-level package.
|
|
50
|
+
- Namespaces: `dc` (subject, description), `xmp` (Rating, Label), `lr`
|
|
51
|
+
(hierarchicalSubject), `Iptc4xmpExt` (PersonInImage), `mwg-rs` regions
|
|
52
|
+
(`XmpRegions` / `XmpRegion`, MWG centre-based normalised coordinates), and a
|
|
53
|
+
new `argus` namespace for `ContentRating`. The `argus` namespace URI
|
|
54
|
+
(`https://github.com/smk762/argus-cortex/ns/xmp/1.0/`) is now a wire
|
|
55
|
+
contract.
|
|
56
|
+
- **Writes merge**: each property set in `XmpData` replaces that property
|
|
57
|
+
(latest write wins, element or attribute form, across every
|
|
58
|
+
`rdf:Description`); everything else in the sidecar — Lightroom develop
|
|
59
|
+
settings, GPS, another stage's fields, foreign namespace prefixes — is
|
|
60
|
+
preserved. `None` leaves a property untouched, an empty value removes it;
|
|
61
|
+
`replace=True` discards the existing sidecar.
|
|
62
|
+
- Writes are atomic (temp file + rename) and keep the existing file mode
|
|
63
|
+
(`0644` for a new sidecar). An unparseable existing sidecar raises
|
|
64
|
+
`XmpError` and is never clobbered; a DTD / entity declaration is refused.
|
|
65
|
+
- Stdlib `xml.etree` + pydantic only, so the top-level import stays
|
|
66
|
+
pydantic-only.
|
|
67
|
+
- `CLAUDE.md` agent guide.
|
|
68
|
+
|
|
69
|
+
### Security
|
|
70
|
+
- **eMudhra/emSign CA incident.** The extras that make outbound TLS
|
|
71
|
+
connections (`[remote]` httpx, `[qdrant]`, `[s3]`) now require
|
|
72
|
+
`certifi>=2026.10.19`, which no longer trusts the compromised emSign
|
|
73
|
+
intermediates. See https://discuss.python.org/t/109576.
|
|
74
|
+
|
|
75
|
+
## [0.2.0] - 2026-07-21
|
|
76
|
+
|
|
77
|
+
### Added
|
|
78
|
+
- **Optional stateful-services layer** (`argus_cortex.store`), each store opt-in
|
|
79
|
+
via `CORTEX_*` env vars and a `Null*` no-op when unset:
|
|
80
|
+
- Phase 1 — Postgres lineage store (`source_asset → caption → human_edit →
|
|
81
|
+
dataset_membership → training_run`), backed by a psycopg connection pool
|
|
82
|
+
with an atomic `transaction()` boundary (`[postgres]` extra).
|
|
83
|
+
- Phase 2 — Qdrant vector store for image / tag-set embeddings
|
|
84
|
+
(`[qdrant]` extra).
|
|
85
|
+
- Phase 3 — MinIO/S3 content-addressed blob store, keyed on sha256
|
|
86
|
+
(`content_key()` == `source_asset.sha256`) (`[s3]` extra).
|
|
87
|
+
- **`argus_cortex.server`** — shared pure-ASGI `WriteGuard`, the
|
|
88
|
+
`constant_refuse()` / `cross_site_refuse()` predicates, `env_flag()` and the
|
|
89
|
+
`UNSAFE_METHODS` / `TRUTHY` / `FALSY` constants (`[server]` extra).
|
|
90
|
+
|
|
91
|
+
### Fixed
|
|
92
|
+
- Release: a TestPyPI failure no longer gates the real PyPI release.
|
|
93
|
+
|
|
94
|
+
## [0.1.0] - 2026-07-08
|
|
95
|
+
|
|
96
|
+
### Added
|
|
97
|
+
- Initial release (renamed from `argus-core`; that PyPI name was taken):
|
|
98
|
+
- `argus_cortex.taxonomy` — `TargetProfile`, `TargetStyle`, `TargetCategory`.
|
|
99
|
+
- `argus_cortex.wire` — versioned wire-schema toolkit (`make_versioned_base`,
|
|
100
|
+
`check_version` with an injectable error type, `render_schema` /
|
|
101
|
+
`wire_schema`, `VersionError`).
|
|
102
|
+
- `argus_cortex.backends` — the local/remote AI-backend contract
|
|
103
|
+
(`LocalBackend`, `RemoteBackend`, `RemoteProvider`, `with_retries`
|
|
104
|
+
retrying only transient errors, `resolve_device`).
|
|
105
|
+
|
|
106
|
+
[Unreleased]: https://github.com/smk762/argus-cortex/compare/v0.4.0...HEAD
|
|
107
|
+
[0.4.0]: https://github.com/smk762/argus-cortex/compare/v0.3.0...v0.4.0
|
|
108
|
+
[0.3.0]: https://github.com/smk762/argus-cortex/compare/v0.2.0...v0.3.0
|
|
109
|
+
[0.2.0]: https://github.com/smk762/argus-cortex/compare/v0.1.0...v0.2.0
|
|
110
|
+
[0.1.0]: https://github.com/smk762/argus-cortex/releases/tag/v0.1.0
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# CLAUDE.md — argus-cortex
|
|
2
|
+
|
|
3
|
+
Guidance for AI agents working in this repo. Human-facing usage lives in [README.md](README.md); this file is the orientation an agent needs to change code safely.
|
|
4
|
+
|
|
5
|
+
## What this is
|
|
6
|
+
|
|
7
|
+
The **shared foundation library** of the Argus suite (quarry → curator → lens → forge → proof). It holds the code the stages *share* so it lives in one place instead of copy-pasted per repo: the target taxonomy, the wire-schema versioning discipline, the local/remote AI-backend contract, and the optional stateful-services layer.
|
|
8
|
+
|
|
9
|
+
Naming: the working directory is `argus-core`, but the GitHub repo, the PyPI package, and the import name are all **`argus-cortex`** (`src/argus_cortex/`). Don't "fix" the mismatch.
|
|
10
|
+
|
|
11
|
+
Because everything here is imported by the downstream stages, **a change ripples across the whole suite** — treat this repo as an API, not a leaf. Read [README.md](README.md) for the *why* and worked examples; it is the source of truth for behaviour.
|
|
12
|
+
|
|
13
|
+
## Layout
|
|
14
|
+
|
|
15
|
+
`src/argus_cortex/` (top level is pydantic-only; everything heavier is behind an extra and lazily imported):
|
|
16
|
+
|
|
17
|
+
- `taxonomy.py` — the "moat" types every stage inherits verbatim: `TargetProfile`, `TargetStyle`/`TargetCategory` literals. Keep it dependency-light; a change here retrains everyone's expectations.
|
|
18
|
+
- `wire.py` — the versioned wire-schema toolkit consumers build their contracts on: `make_versioned_base(field_name, current, majors)` (a Pydantic base that stamps + version-checks a field like `proof_version`/`manifest_version`), `check_version()` (the major-compatibility gate), `render_schema()`/`wire_schema()` (the canonical committed-schema string behind each consumer's `schema --check` CLI), and `VersionError`.
|
|
19
|
+
- `backends.py` — the AI-backend contract: `Backend`/`LocalBackend`/`RemoteBackend` (point at a hosted service via `from_host(host, port)` or `from_provider(RemoteProvider.…)`), plus `with_retries()` and `resolve_device()`. httpx is lazy, behind `[remote]`.
|
|
20
|
+
- `server.py` — the shared FastAPI/ASGI scaffolding other repos' micro-servers mount: `WriteGuard` (pure-ASGI method gate), the `constant_refuse()` (read-only/replay) and `cross_site_refuse()` (cross-origin write) predicates, `env_flag()`, and the `UNSAFE_METHODS`/`TRUTHY`/`FALSY` constants. Starlette is behind `[server]`.
|
|
21
|
+
- `xmp.py` — the suite's XMP sidecar read/merge/write (`write_xmp`/`read_xmp`/`merge_xmp`/`parse_xmp`, `XmpData`/`XmpRegions`, `XmpError`), used by lens (keywords/caption) and curator (faces, content rating). Stdlib `xml.etree` only. **Writes merge — latest wins per property, foreign properties preserved; keep that.** `None` = leave untouched, empty = remove. A DTD is refused and an unparseable sidecar raises rather than being overwritten. The `argus` namespace URI is a wire contract — don't change it.
|
|
22
|
+
- `moderation/` — content-moderation clients over HTTP: `nudenet.py` (`NudeNetClient`, `NsfwResult`, `DEFAULT_EXPOSED_LABELS` — **the suite's one definition of nudity; don't widen it casually**, every stage's NSFW numbers move with it), `singguard.py` (`SingGuardClient`, `GuardResult`, `rule_matches`), `_http.py` (shared POST, proxy-free opener, `CheckFailed`/`ModerationConfigError`). Stdlib + pydantic; Pillow lazily, behind `[images]`. Never import NudeNet (AGPL) — HTTP only. URLs are passed in; stages own their env config.
|
|
23
|
+
- `store/` — the optional stateful-services layer, opt-in per `CORTEX_*` env var: `lineage.py` (Phase 1 Postgres DAG, `open_lineage_store()`), `vector.py` (Phase 2 Qdrant, `open_vector_store()`), `blob.py` (Phase 3 MinIO/S3, `open_blob_store()`), over `config.py`/`models.py`/`errors.py`. Each `open_*()` returns a `Null*` no-op when its URL is unset. Drivers are behind `[postgres]`/`[qdrant]`/`[s3]`.
|
|
24
|
+
|
|
25
|
+
There is **no CLI and no committed schema in this repo** — the `schema`/`schema --check` pattern is something `wire.py` *enables* for downstream repos, not something cortex runs on itself.
|
|
26
|
+
|
|
27
|
+
## Commands
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
make install # uv venv + editable install with [dev]
|
|
31
|
+
make test # uv run --no-sync pytest --tb=short -q
|
|
32
|
+
make lint # ruff check + format --check (pinned ruff 0.15.16)
|
|
33
|
+
make format # ruff format + check --fix
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Run one test: `uv run --no-sync pytest tests/test_wire.py::test_name -q`.
|
|
37
|
+
|
|
38
|
+
## Conventions & gotchas
|
|
39
|
+
|
|
40
|
+
- **This is shared code — changes ripple.** A `taxonomy.py` edit or a `wire.py`/`backends.py` semantics change lands in every stage on their next bump. Prefer additive, backward-compatible changes; if you break the wire contract, that is a **major** version (`check_version` is what refuses the old major downstream).
|
|
41
|
+
- **Keep the top-level import pydantic-only.** `import argus_cortex` must not pull in starlette/httpx/psycopg/qdrant/minio — those are optional extras, imported lazily inside the module that needs them (see the `try/except ImportError` → "install `argus-cortex[extra]`" pattern). Don't hoist an optional import to module top level.
|
|
42
|
+
- **Stores fail *open* to a no-op, not an error.** `open_*()` returns a `Null*` store when the `CORTEX_*` URL is unset, so a stage with no DB configured still runs. Preserve that; blob identity is content-addressed on sha256 (`content_key()` == `source_asset.sha256`) — storage, dedup, and lineage join share one identity.
|
|
43
|
+
- **`WriteGuard` is pure ASGI on purpose** (not `BaseHTTPMiddleware`), so it never buffers the SSE/NDJSON streams downstream servers return. Register it *before* `CORSMiddleware`. `env_flag()` warns on a mistyped protection flag rather than silently disabling it — keep that fail-safe behaviour.
|
|
44
|
+
- **Versioning is git-tag-derived** (`hatch-vcs`). Never hand-edit a version; `src/argus_cortex/_version.py` is generated (gitignored). Tag `vX.Y.Z` to release.
|
|
45
|
+
- Pydantic v2 everywhere; async tests via `asyncio_mode = auto`. Ruff pinned **0.15.16**, line-length 120, target py311.
|
|
46
|
+
|
|
47
|
+
## CI / release
|
|
48
|
+
|
|
49
|
+
CI runs via the shared [`argus-ci`](https://github.com/smk762/argus-ci) reusable workflow (ruff + pytest with the `dev` extra — which installs the real psycopg/qdrant/minio/starlette clients so each store's live driver path is exercised, not just fakes). Release publishes to PyPI (OIDC trusted publishing) on `v*` tags — **no container image, this is a library**. Scaffolded from [`argus-pkg-template`](https://github.com/smk762/argus-pkg-template); run `copier update` to pull tooling changes.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: argus-cortex
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Shared foundation for the Argus suite: taxonomy, versioned wire-schema tooling, and the local/remote AI backend contract
|
|
5
5
|
Project-URL: Repository, https://github.com/smk762/argus-cortex
|
|
6
6
|
Author: smk762
|
|
@@ -16,21 +16,29 @@ Classifier: Typing :: Typed
|
|
|
16
16
|
Requires-Python: >=3.11
|
|
17
17
|
Requires-Dist: pydantic>=2.7
|
|
18
18
|
Provides-Extra: dev
|
|
19
|
+
Requires-Dist: certifi>=2026.10.19; extra == 'dev'
|
|
20
|
+
Requires-Dist: httpx2>=2.7; extra == 'dev'
|
|
19
21
|
Requires-Dist: httpx>=0.27; extra == 'dev'
|
|
20
22
|
Requires-Dist: minio>=7.2; extra == 'dev'
|
|
23
|
+
Requires-Dist: pillow>=10.0; extra == 'dev'
|
|
21
24
|
Requires-Dist: psycopg[binary,pool]>=3.1; extra == 'dev'
|
|
22
25
|
Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
|
|
23
26
|
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
24
27
|
Requires-Dist: qdrant-client>=1.10; extra == 'dev'
|
|
25
28
|
Requires-Dist: ruff==0.15.16; extra == 'dev'
|
|
26
29
|
Requires-Dist: starlette>=0.37; extra == 'dev'
|
|
30
|
+
Provides-Extra: images
|
|
31
|
+
Requires-Dist: pillow>=10.0; extra == 'images'
|
|
27
32
|
Provides-Extra: postgres
|
|
28
33
|
Requires-Dist: psycopg[binary,pool]>=3.1; extra == 'postgres'
|
|
29
34
|
Provides-Extra: qdrant
|
|
35
|
+
Requires-Dist: certifi>=2026.10.19; extra == 'qdrant'
|
|
30
36
|
Requires-Dist: qdrant-client>=1.10; extra == 'qdrant'
|
|
31
37
|
Provides-Extra: remote
|
|
38
|
+
Requires-Dist: certifi>=2026.10.19; extra == 'remote'
|
|
32
39
|
Requires-Dist: httpx>=0.27; extra == 'remote'
|
|
33
40
|
Provides-Extra: s3
|
|
41
|
+
Requires-Dist: certifi>=2026.10.19; extra == 's3'
|
|
34
42
|
Requires-Dist: minio>=7.2; extra == 's3'
|
|
35
43
|
Provides-Extra: server
|
|
36
44
|
Requires-Dist: starlette>=0.37; extra == 'server'
|
|
@@ -48,6 +56,8 @@ Part of the [Argus suite](https://github.com/smk762?tab=repositories&q=argus) (q
|
|
|
48
56
|
- **`argus_cortex.wire`** — the versioned wire-schema toolkit: `check_version()` (major-compatibility gate), `make_versioned_base()` (a Pydantic base that stamps + checks a version field like `proof_version` / `manifest_version`), and `wire_schema()` / `render_schema()` for the committed-schema `schema --check` CLI pattern.
|
|
49
57
|
- **`argus_cortex.backends`** — the AI-backend contract generalised from argus-lens: `Backend` / `LocalBackend` / `RemoteBackend` (point at a hosted service by `base_url`, `host`/`port`, or a known `RemoteProvider`), plus `with_retries()` and `resolve_device()`. httpx lives behind the `[remote]` extra.
|
|
50
58
|
- **`argus_cortex.server`** — shared write-guard + env-flag scaffolding for the suite's FastAPI micro-servers: `WriteGuard` (a pure-ASGI method-gate that never buffers streaming responses), the `constant_refuse()` / `cross_site_refuse()` predicates (read-only/replay mode and cross-site-write protection are two configs of the one guard), the `UNSAFE_METHODS` / `TRUTHY` / `FALSY` constants, and `env_flag()` (warns on a mistyped protection flag rather than silently disabling it). Starlette lives behind the `[server]` extra.
|
|
59
|
+
- **`argus_cortex.xmp`** — the suite's one XMP sidecar implementation (hoisted from argus-lens): `write_xmp()` / `read_xmp()` / `merge_xmp()` / `parse_xmp()` over an `XmpData` model covering `dc` (keywords, description), `xmp` (Rating, Label), `lr` (hierarchical keywords), `Iptc4xmpExt` (PersonInImage), `mwg-rs` face/pet regions, and `argus:ContentRating`. Writes **merge**: each property you set replaces that property (latest write wins), everything else in the sidecar — Lightroom develop settings, GPS, another stage's fields — is preserved. `None` leaves a property alone, an empty value removes it. Writes are atomic; an unparseable existing sidecar raises `XmpError` instead of being clobbered. Stdlib only.
|
|
60
|
+
- **`argus_cortex.moderation`** — the suite's content-moderation clients, over HTTP to servers the deployer runs (nothing here imports a detector): `NudeNetClient` (a NudeNet server's `/infer` → `NsfwResult`: detections, per-class maxima, and an **exposed score**) and `SingGuardClient` (a SingGuard server's `/api/guard/check` → `GuardResult`: verdict, the policy rules that fired, explanation; optional custom policy text). `DEFAULT_EXPOSED_LABELS` is the suite's **one definition of nudity**: the five exposed buttocks/breast/genitalia/anus classes — faces, covered parts, and bare feet, belly, armpits or male chest never count. Stdlib + pydantic; Pillow (for SingGuard's downscaling of large images, on by default) is behind `[images]`: without it `SingGuardClient(...)` raises `ModerationConfigError` at construction unless given `downscale=False`. A `user:password@` in a URL is sent as HTTP Basic auth. URLs are passed in, never read from the environment here: each stage owns its config. NudeNet itself is AGPL-3.0, which is why it stays behind an HTTP boundary.
|
|
51
61
|
- **`argus_cortex.store`** — the optional **stateful services layer** (the suite's persistent "memory"). External and opt-in — every store is configured by `CORTEX_*` env vars and no-ops when its URL is unset.
|
|
52
62
|
- **Phase 1 — Postgres lineage store.** Persists the `source_asset → caption → human_edit → dataset_membership → training_run` DAG for LoRA reproducibility, edit capture, and the feedback loop. Backed by a psycopg connection **pool** (concurrent `asyncio.to_thread` calls are safe; a dropped connection is replaced), with a `transaction()` boundary for atomic multi-step writes. `open_lineage_store()` returns a no-op store when `CORTEX_PG_URL` is unset; psycopg lives behind the `[postgres]` extra.
|
|
53
63
|
- **Phase 2 — Qdrant vector store.** Stores image + tag-set embeddings for retrieval-augmented few-shot and near-duplicate curation; put a `caption_id`/`asset_id` in a point's payload and a search hit joins straight back to the lineage. `open_vector_store()` no-ops when `CORTEX_QDRANT_URL` is unset; qdrant-client lives behind the `[qdrant]` extra. cortex stores vectors it's *handed* — computing embeddings is the caller's job.
|
|
@@ -65,6 +75,37 @@ scorer = RemoteBackend.from_host("192.168.1.20", 9000, api_key="…")
|
|
|
65
75
|
nim = RemoteBackend.from_provider(RemoteProvider.NVIDIA_NIM, api_key="…")
|
|
66
76
|
```
|
|
67
77
|
|
|
78
|
+
```python
|
|
79
|
+
from argus_cortex.xmp import XmpData, XmpRegion, XmpRegions, write_xmp
|
|
80
|
+
|
|
81
|
+
# lens writes captions/keywords …
|
|
82
|
+
write_xmp("pics/beach.jpg", XmpData(keywords=["beach", "sunset"], description="a beach at sunset"))
|
|
83
|
+
# … curator later adds faces + a content rating to the same sidecar; lens's fields survive
|
|
84
|
+
write_xmp(
|
|
85
|
+
"pics/beach.jpg",
|
|
86
|
+
XmpData(
|
|
87
|
+
persons=["Ada"],
|
|
88
|
+
regions=XmpRegions(width=4000, height=3000, regions=[XmpRegion(x=0.52, y=0.4, w=0.1, h=0.18, name="Ada")]),
|
|
89
|
+
content_rating="safe",
|
|
90
|
+
),
|
|
91
|
+
)
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
```python
|
|
95
|
+
from argus_cortex.moderation import NudeNetClient, SingGuardClient, rule_matches
|
|
96
|
+
|
|
97
|
+
nudenet = NudeNetClient("http://nudenet:8080") # its origin; /infer is appended
|
|
98
|
+
result = nudenet.check(image_bytes, "beach.jpg")
|
|
99
|
+
result.exposed_score # 0.0-1.0 over DEFAULT_EXPOSED_LABELS; result.labels = per-class maxima
|
|
100
|
+
|
|
101
|
+
guard = SingGuardClient("http://guard:9150", policy=None) # None = SingGuard's default rules
|
|
102
|
+
verdict = guard.check(image_bytes, text="the prompt, if any")
|
|
103
|
+
verdict.safe, verdict.rules # False, ["B. Real-World Crimes & Public Safety"]
|
|
104
|
+
any(rule_matches(r, "B") for r in verdict.rules) # match a rule by letter or full name
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
Each `check()` raises `CheckFailed` (`NudeNetCheckFailed` / `GuardCheckFailed`) for a network error or an unusable answer, so a caller checking many images can record "unchecked" against the one image — and should treat unchecked as *not* safe. Bad configuration raises `ModerationConfigError`, whose message never echoes the URL.
|
|
108
|
+
|
|
68
109
|
```python
|
|
69
110
|
from argus_cortex.store import open_lineage_store, SourceAsset, Caption, HumanEdit
|
|
70
111
|
|
|
@@ -10,6 +10,8 @@ Part of the [Argus suite](https://github.com/smk762?tab=repositories&q=argus) (q
|
|
|
10
10
|
- **`argus_cortex.wire`** — the versioned wire-schema toolkit: `check_version()` (major-compatibility gate), `make_versioned_base()` (a Pydantic base that stamps + checks a version field like `proof_version` / `manifest_version`), and `wire_schema()` / `render_schema()` for the committed-schema `schema --check` CLI pattern.
|
|
11
11
|
- **`argus_cortex.backends`** — the AI-backend contract generalised from argus-lens: `Backend` / `LocalBackend` / `RemoteBackend` (point at a hosted service by `base_url`, `host`/`port`, or a known `RemoteProvider`), plus `with_retries()` and `resolve_device()`. httpx lives behind the `[remote]` extra.
|
|
12
12
|
- **`argus_cortex.server`** — shared write-guard + env-flag scaffolding for the suite's FastAPI micro-servers: `WriteGuard` (a pure-ASGI method-gate that never buffers streaming responses), the `constant_refuse()` / `cross_site_refuse()` predicates (read-only/replay mode and cross-site-write protection are two configs of the one guard), the `UNSAFE_METHODS` / `TRUTHY` / `FALSY` constants, and `env_flag()` (warns on a mistyped protection flag rather than silently disabling it). Starlette lives behind the `[server]` extra.
|
|
13
|
+
- **`argus_cortex.xmp`** — the suite's one XMP sidecar implementation (hoisted from argus-lens): `write_xmp()` / `read_xmp()` / `merge_xmp()` / `parse_xmp()` over an `XmpData` model covering `dc` (keywords, description), `xmp` (Rating, Label), `lr` (hierarchical keywords), `Iptc4xmpExt` (PersonInImage), `mwg-rs` face/pet regions, and `argus:ContentRating`. Writes **merge**: each property you set replaces that property (latest write wins), everything else in the sidecar — Lightroom develop settings, GPS, another stage's fields — is preserved. `None` leaves a property alone, an empty value removes it. Writes are atomic; an unparseable existing sidecar raises `XmpError` instead of being clobbered. Stdlib only.
|
|
14
|
+
- **`argus_cortex.moderation`** — the suite's content-moderation clients, over HTTP to servers the deployer runs (nothing here imports a detector): `NudeNetClient` (a NudeNet server's `/infer` → `NsfwResult`: detections, per-class maxima, and an **exposed score**) and `SingGuardClient` (a SingGuard server's `/api/guard/check` → `GuardResult`: verdict, the policy rules that fired, explanation; optional custom policy text). `DEFAULT_EXPOSED_LABELS` is the suite's **one definition of nudity**: the five exposed buttocks/breast/genitalia/anus classes — faces, covered parts, and bare feet, belly, armpits or male chest never count. Stdlib + pydantic; Pillow (for SingGuard's downscaling of large images, on by default) is behind `[images]`: without it `SingGuardClient(...)` raises `ModerationConfigError` at construction unless given `downscale=False`. A `user:password@` in a URL is sent as HTTP Basic auth. URLs are passed in, never read from the environment here: each stage owns its config. NudeNet itself is AGPL-3.0, which is why it stays behind an HTTP boundary.
|
|
13
15
|
- **`argus_cortex.store`** — the optional **stateful services layer** (the suite's persistent "memory"). External and opt-in — every store is configured by `CORTEX_*` env vars and no-ops when its URL is unset.
|
|
14
16
|
- **Phase 1 — Postgres lineage store.** Persists the `source_asset → caption → human_edit → dataset_membership → training_run` DAG for LoRA reproducibility, edit capture, and the feedback loop. Backed by a psycopg connection **pool** (concurrent `asyncio.to_thread` calls are safe; a dropped connection is replaced), with a `transaction()` boundary for atomic multi-step writes. `open_lineage_store()` returns a no-op store when `CORTEX_PG_URL` is unset; psycopg lives behind the `[postgres]` extra.
|
|
15
17
|
- **Phase 2 — Qdrant vector store.** Stores image + tag-set embeddings for retrieval-augmented few-shot and near-duplicate curation; put a `caption_id`/`asset_id` in a point's payload and a search hit joins straight back to the lineage. `open_vector_store()` no-ops when `CORTEX_QDRANT_URL` is unset; qdrant-client lives behind the `[qdrant]` extra. cortex stores vectors it's *handed* — computing embeddings is the caller's job.
|
|
@@ -27,6 +29,37 @@ scorer = RemoteBackend.from_host("192.168.1.20", 9000, api_key="…")
|
|
|
27
29
|
nim = RemoteBackend.from_provider(RemoteProvider.NVIDIA_NIM, api_key="…")
|
|
28
30
|
```
|
|
29
31
|
|
|
32
|
+
```python
|
|
33
|
+
from argus_cortex.xmp import XmpData, XmpRegion, XmpRegions, write_xmp
|
|
34
|
+
|
|
35
|
+
# lens writes captions/keywords …
|
|
36
|
+
write_xmp("pics/beach.jpg", XmpData(keywords=["beach", "sunset"], description="a beach at sunset"))
|
|
37
|
+
# … curator later adds faces + a content rating to the same sidecar; lens's fields survive
|
|
38
|
+
write_xmp(
|
|
39
|
+
"pics/beach.jpg",
|
|
40
|
+
XmpData(
|
|
41
|
+
persons=["Ada"],
|
|
42
|
+
regions=XmpRegions(width=4000, height=3000, regions=[XmpRegion(x=0.52, y=0.4, w=0.1, h=0.18, name="Ada")]),
|
|
43
|
+
content_rating="safe",
|
|
44
|
+
),
|
|
45
|
+
)
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
```python
|
|
49
|
+
from argus_cortex.moderation import NudeNetClient, SingGuardClient, rule_matches
|
|
50
|
+
|
|
51
|
+
nudenet = NudeNetClient("http://nudenet:8080") # its origin; /infer is appended
|
|
52
|
+
result = nudenet.check(image_bytes, "beach.jpg")
|
|
53
|
+
result.exposed_score # 0.0-1.0 over DEFAULT_EXPOSED_LABELS; result.labels = per-class maxima
|
|
54
|
+
|
|
55
|
+
guard = SingGuardClient("http://guard:9150", policy=None) # None = SingGuard's default rules
|
|
56
|
+
verdict = guard.check(image_bytes, text="the prompt, if any")
|
|
57
|
+
verdict.safe, verdict.rules # False, ["B. Real-World Crimes & Public Safety"]
|
|
58
|
+
any(rule_matches(r, "B") for r in verdict.rules) # match a rule by letter or full name
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Each `check()` raises `CheckFailed` (`NudeNetCheckFailed` / `GuardCheckFailed`) for a network error or an unusable answer, so a caller checking many images can record "unchecked" against the one image — and should treat unchecked as *not* safe. Bad configuration raises `ModerationConfigError`, whose message never echoes the URL.
|
|
62
|
+
|
|
30
63
|
```python
|
|
31
64
|
from argus_cortex.store import open_lineage_store, SourceAsset, Caption, HumanEdit
|
|
32
65
|
|
|
@@ -24,14 +24,17 @@ dependencies = [
|
|
|
24
24
|
]
|
|
25
25
|
|
|
26
26
|
[project.optional-dependencies]
|
|
27
|
-
remote = ["httpx>=0.27"]
|
|
27
|
+
remote = ["httpx>=0.27", "certifi>=2026.10.19"]
|
|
28
28
|
postgres = ["psycopg[binary,pool]>=3.1"]
|
|
29
|
-
qdrant = ["qdrant-client>=1.10"]
|
|
30
|
-
s3 = ["minio>=7.2"]
|
|
29
|
+
qdrant = ["qdrant-client>=1.10", "certifi>=2026.10.19"]
|
|
30
|
+
s3 = ["minio>=7.2", "certifi>=2026.10.19"]
|
|
31
31
|
# The shared ASGI write-guard + env-flag scaffolding for the suite's servers
|
|
32
32
|
# (argus_cortex.server). Starlette ships with FastAPI, so consumers already have
|
|
33
33
|
# it via their own [server] extra.
|
|
34
34
|
server = ["starlette>=0.37"]
|
|
35
|
+
# Pillow, for SingGuardClient's downscaling of large images
|
|
36
|
+
# (argus_cortex.moderation.singguard), imported only when it downscales.
|
|
37
|
+
images = ["pillow>=10.0"]
|
|
35
38
|
# The store drivers are in dev so each store's real client path (pool/enum/client
|
|
36
39
|
# construction, error wrapping) is exercised in CI, not just via fakes.
|
|
37
40
|
# starlette + httpx power the pure-ASGI guard tests (Starlette TestClient).
|
|
@@ -44,6 +47,9 @@ dev = [
|
|
|
44
47
|
"minio>=7.2",
|
|
45
48
|
"starlette>=0.37",
|
|
46
49
|
"httpx>=0.27",
|
|
50
|
+
"httpx2>=2.7",
|
|
51
|
+
"pillow>=10.0",
|
|
52
|
+
"certifi>=2026.10.19",
|
|
47
53
|
]
|
|
48
54
|
|
|
49
55
|
[project.urls]
|
|
@@ -23,6 +23,14 @@ from argus_cortex.backends import (
|
|
|
23
23
|
resolve_device,
|
|
24
24
|
with_retries,
|
|
25
25
|
)
|
|
26
|
+
from argus_cortex.moderation import (
|
|
27
|
+
CheckFailed,
|
|
28
|
+
GuardResult,
|
|
29
|
+
ModerationConfigError,
|
|
30
|
+
NsfwResult,
|
|
31
|
+
NudeNetClient,
|
|
32
|
+
SingGuardClient,
|
|
33
|
+
)
|
|
26
34
|
from argus_cortex.store import (
|
|
27
35
|
IMAGE_COLLECTION,
|
|
28
36
|
TAGSET_COLLECTION,
|
|
@@ -56,6 +64,18 @@ from argus_cortex.wire import (
|
|
|
56
64
|
schema_major,
|
|
57
65
|
wire_schema,
|
|
58
66
|
)
|
|
67
|
+
from argus_cortex.xmp import (
|
|
68
|
+
XmpData,
|
|
69
|
+
XmpError,
|
|
70
|
+
XmpRegion,
|
|
71
|
+
XmpRegions,
|
|
72
|
+
merge_xmp,
|
|
73
|
+
parse_xmp,
|
|
74
|
+
read_xmp,
|
|
75
|
+
render_xmp,
|
|
76
|
+
sidecar_path,
|
|
77
|
+
write_xmp,
|
|
78
|
+
)
|
|
59
79
|
|
|
60
80
|
__all__ = [
|
|
61
81
|
"__version__",
|
|
@@ -70,6 +90,17 @@ __all__ = [
|
|
|
70
90
|
"render_schema",
|
|
71
91
|
"schema_major",
|
|
72
92
|
"wire_schema",
|
|
93
|
+
# xmp sidecars
|
|
94
|
+
"XmpData",
|
|
95
|
+
"XmpError",
|
|
96
|
+
"XmpRegion",
|
|
97
|
+
"XmpRegions",
|
|
98
|
+
"merge_xmp",
|
|
99
|
+
"parse_xmp",
|
|
100
|
+
"read_xmp",
|
|
101
|
+
"render_xmp",
|
|
102
|
+
"sidecar_path",
|
|
103
|
+
"write_xmp",
|
|
73
104
|
# backends
|
|
74
105
|
"Backend",
|
|
75
106
|
"BackendError",
|
|
@@ -79,6 +110,13 @@ __all__ = [
|
|
|
79
110
|
"RemoteProvider",
|
|
80
111
|
"resolve_device",
|
|
81
112
|
"with_retries",
|
|
113
|
+
# moderation clients (NudeNet, SingGuard over HTTP)
|
|
114
|
+
"CheckFailed",
|
|
115
|
+
"GuardResult",
|
|
116
|
+
"ModerationConfigError",
|
|
117
|
+
"NsfwResult",
|
|
118
|
+
"NudeNetClient",
|
|
119
|
+
"SingGuardClient",
|
|
82
120
|
# store (optional stateful services layer)
|
|
83
121
|
"StoreConfig",
|
|
84
122
|
"StoreError",
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '0.
|
|
22
|
-
__version_tuple__ = version_tuple = (0,
|
|
21
|
+
__version__ = version = '0.4.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 4, 0)
|
|
23
23
|
|
|
24
24
|
__commit_id__ = commit_id = None
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""Content-moderation clients the stages share: NudeNet and SingGuard over HTTP.
|
|
2
|
+
|
|
3
|
+
Each client talks to a server the deployer runs; nothing here imports a
|
|
4
|
+
detector. Stdlib + pydantic only (Pillow, for SingGuard's downscaling, is
|
|
5
|
+
behind the ``[images]`` extra and imported only when used). Configuration is
|
|
6
|
+
the caller's: each stage reads its own environment and passes a URL in.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from argus_cortex.moderation._http import CheckFailed, ModerationConfigError
|
|
12
|
+
from argus_cortex.moderation.nudenet import (
|
|
13
|
+
DEFAULT_EXPOSED_LABELS,
|
|
14
|
+
NUDENET_LABELS,
|
|
15
|
+
NsfwDetection,
|
|
16
|
+
NsfwResult,
|
|
17
|
+
NudeNetCheckFailed,
|
|
18
|
+
NudeNetClient,
|
|
19
|
+
NudeNetLabel,
|
|
20
|
+
exposed_score,
|
|
21
|
+
)
|
|
22
|
+
from argus_cortex.moderation.singguard import (
|
|
23
|
+
THINKING_MODES,
|
|
24
|
+
GuardCheckFailed,
|
|
25
|
+
GuardResult,
|
|
26
|
+
SingGuardClient,
|
|
27
|
+
policy_name,
|
|
28
|
+
rule_letter,
|
|
29
|
+
rule_matches,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
__all__ = [
|
|
33
|
+
"DEFAULT_EXPOSED_LABELS",
|
|
34
|
+
"NUDENET_LABELS",
|
|
35
|
+
"THINKING_MODES",
|
|
36
|
+
"CheckFailed",
|
|
37
|
+
"GuardCheckFailed",
|
|
38
|
+
"GuardResult",
|
|
39
|
+
"ModerationConfigError",
|
|
40
|
+
"NsfwDetection",
|
|
41
|
+
"NsfwResult",
|
|
42
|
+
"NudeNetCheckFailed",
|
|
43
|
+
"NudeNetClient",
|
|
44
|
+
"NudeNetLabel",
|
|
45
|
+
"SingGuardClient",
|
|
46
|
+
"exposed_score",
|
|
47
|
+
"policy_name",
|
|
48
|
+
"rule_letter",
|
|
49
|
+
"rule_matches",
|
|
50
|
+
]
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""The one HTTP path the moderation clients share. Stdlib only."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import base64
|
|
6
|
+
import http.client
|
|
7
|
+
import json
|
|
8
|
+
import urllib.error
|
|
9
|
+
import urllib.request
|
|
10
|
+
from urllib.parse import unquote, urlsplit, urlunsplit
|
|
11
|
+
|
|
12
|
+
# Ignore HTTP(S)_PROXY: images go straight to the configured server, never
|
|
13
|
+
# through whatever proxy happens to be in the environment.
|
|
14
|
+
_OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class ModerationConfigError(ValueError):
|
|
18
|
+
"""A moderation client was configured with an unusable URL or option."""
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class CheckFailed(RuntimeError):
|
|
22
|
+
"""One image could not be checked (network error, bad response)."""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def service_url(base: str, path: str, service: str) -> str:
|
|
26
|
+
"""*base* (a server's origin) with *path* appended to its path; a *base*
|
|
27
|
+
whose path already ends in *path* is used as is. A query string is kept.
|
|
28
|
+
|
|
29
|
+
The error names *service* but not *base*: a URL can carry a credential, and
|
|
30
|
+
callers surface this message.
|
|
31
|
+
"""
|
|
32
|
+
parts = urlsplit(base.strip())
|
|
33
|
+
if parts.scheme not in {"http", "https"} or not parts.hostname:
|
|
34
|
+
raise ModerationConfigError(f"the {service} URL must be an http(s) URL")
|
|
35
|
+
base_path = parts.path.rstrip("/")
|
|
36
|
+
if not base_path.endswith(path):
|
|
37
|
+
base_path += path
|
|
38
|
+
return urlunsplit((parts.scheme, parts.netloc, base_path, parts.query, ""))
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _split_credentials(url: str) -> tuple[str, dict[str, str]]:
|
|
42
|
+
"""*url* without its ``user:password@`` part, plus the Basic auth header it
|
|
43
|
+
stood for (urllib would otherwise take the userinfo for part of the host)."""
|
|
44
|
+
parts = urlsplit(url)
|
|
45
|
+
if parts.username is None:
|
|
46
|
+
return url, {}
|
|
47
|
+
netloc = parts.netloc.rpartition("@")[2]
|
|
48
|
+
userinfo = f"{unquote(parts.username)}:{unquote(parts.password or '')}"
|
|
49
|
+
auth = f"Basic {base64.b64encode(userinfo.encode()).decode('ascii')}"
|
|
50
|
+
return urlunsplit(parts._replace(netloc=netloc)), {"Authorization": auth}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def post_json(
|
|
54
|
+
url: str, body: bytes, content_type: str, timeout: float, service: str, failed: type[CheckFailed]
|
|
55
|
+
) -> object:
|
|
56
|
+
"""POST *body* to *url* and return the decoded JSON answer.
|
|
57
|
+
|
|
58
|
+
Every transport or decoding problem is raised as *failed*, so a caller
|
|
59
|
+
checking many images can record it against the one image.
|
|
60
|
+
"""
|
|
61
|
+
url, headers = _split_credentials(url)
|
|
62
|
+
req = urllib.request.Request(url, data=body, headers={"Content-Type": content_type, **headers}, method="POST")
|
|
63
|
+
try:
|
|
64
|
+
with _OPENER.open(req, timeout=timeout) as resp: # noqa: S310 - caller-configured URL
|
|
65
|
+
return json.loads(resp.read())
|
|
66
|
+
except urllib.error.HTTPError as exc:
|
|
67
|
+
exc.close() # the exception outlives the request as __cause__; its socket need not
|
|
68
|
+
raise failed(f"{service} returned HTTP {exc.code}") from exc
|
|
69
|
+
except (urllib.error.URLError, http.client.HTTPException, TimeoutError, OSError) as exc:
|
|
70
|
+
raise failed(f"{service} request failed: {exc}") from exc
|
|
71
|
+
except ValueError as exc:
|
|
72
|
+
raise failed(f"{service} returned a body that is not JSON") from exc
|