shadowshield 0.6.1__tar.gz → 0.6.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {shadowshield-0.6.1 → shadowshield-0.6.2}/PKG-INFO +16 -12
- {shadowshield-0.6.1 → shadowshield-0.6.2}/README.md +15 -11
- {shadowshield-0.6.1 → shadowshield-0.6.2}/docs/BENCHMARKS.md +22 -10
- {shadowshield-0.6.1 → shadowshield-0.6.2}/docs/PRODUCTION_READINESS.md +24 -9
- {shadowshield-0.6.1 → shadowshield-0.6.2}/docs/RELEASING.md +24 -11
- {shadowshield-0.6.1 → shadowshield-0.6.2}/pyproject.toml +1 -1
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/__init__.py +1 -1
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/cli.py +1 -1
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/control.py +224 -37
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/detectors/transformer.py +95 -10
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/detectors/vector.py +84 -1
- shadowshield-0.6.2/src/shadowshield/eval/data/generalization_benchmark_v3.jsonl +40 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/eval/dataset.py +7 -6
- {shadowshield-0.6.1 → shadowshield-0.6.2}/tests/test_control.py +116 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/tests/test_generalization_benchmarks.py +57 -1
- shadowshield-0.6.2/tests/test_site_security.py +107 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/tests/test_transformer.py +135 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/tests/test_vector.py +145 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/.gitignore +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/LICENSE +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/docs/CODE_REVIEW.md +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/docs/COMPARISON.md +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/docs/MARKET_LANDSCAPE.md +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/docs/NEXT_STEPS.md +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/docs/OWASP_LLM_TOP10.md +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/docs/PLAN_REVIEW.md +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/docs/REPORTER_SDK_SPEC.md +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/docs/SAAS_STRATEGY.md +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/docs/UPGRADE_OPPORTUNITIES.md +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/docs/configuration.md +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/docs/detectors.md +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/docs/index.md +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/docs/plugins.md +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/docs/research/LANDSCAPE.md +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/docs/security-model.md +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/examples/agentic_security.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/examples/custom_detector.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/examples/langchain_integration.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/examples/openai_integration.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/examples/quickstart.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/_security.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/config/__init__.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/config/default.yaml +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/core/__init__.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/core/canary.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/core/config.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/core/coverage.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/core/engine.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/core/policy.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/core/session.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/core/shield.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/core/telemetry.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/core/types.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/detectors/__init__.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/detectors/alignment.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/detectors/anomaly.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/detectors/base.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/detectors/canary.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/detectors/data/attack_corpus.txt +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/detectors/encoding.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/detectors/exfiltration.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/detectors/jailbreak.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/detectors/llm_check.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/detectors/pii.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/detectors/prompt_injection.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/eval/__init__.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/eval/data/adversarial_benchmark.jsonl +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/eval/data/builtin_benchmark.jsonl +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/eval/data/generalization_benchmark_v1.jsonl +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/eval/data/generalization_benchmark_v2.jsonl +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/eval/harness.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/integrations/__init__.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/integrations/agentdojo.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/integrations/mcp.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/middleware/__init__.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/middleware/base.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/middleware/decorators.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/middleware/langchain.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/middleware/openai.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/plugins/__init__.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/plugins/base.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/plugins/manager.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/py.typed +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/reporter.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/responders/__init__.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/responders/base.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/responders/blocker.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/responders/isolator.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/responders/rate_limiter.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/responders/sanitizer.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/server.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/static/dashboard.html +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/utils/__init__.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/utils/logging.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/utils/scoring.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/src/shadowshield/utils/text.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/tests/__init__.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/tests/test_adversarial.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/tests/test_agentic.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/tests/test_cli.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/tests/test_config.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/tests/test_detectors.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/tests/test_http_security.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/tests/test_mcp_guard.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/tests/test_middleware.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/tests/test_multilingual.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/tests/test_pii_backends.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/tests/test_policy.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/tests/test_production.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/tests/test_prompt_injection.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/tests/test_responders.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/tests/test_server.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/tests/test_shield.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/tests/test_spans.py +0 -0
- {shadowshield-0.6.1 → shadowshield-0.6.2}/tests/test_telemetry.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: shadowshield
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.2
|
|
4
4
|
Summary: Unified open-source security shield for agentic AI systems — inspired by Sentinel & ShadowClaw.
|
|
5
5
|
Project-URL: Homepage, https://shadowshield.xyz
|
|
6
6
|
Project-URL: Documentation, https://github.com/0xsl1m/shadowshield#readme
|
|
@@ -149,8 +149,9 @@ print(result.safe_text) # safe fallback message
|
|
|
149
149
|
> in-distribution **regression baseline, not a SOTA claim**. We publish the humbling
|
|
150
150
|
> external numbers on purpose — a credible security tool shows its homework.
|
|
151
151
|
> The frozen blind semantic snapshots are harder still: v1 reaches 26.7% recall /
|
|
152
|
-
> 13.3% FPR
|
|
153
|
-
> `shadowshield benchmark --generalization
|
|
152
|
+
> 13.3% FPR, v2 reaches 0% / 10%, and v3 reaches 30% / 30%; the 90-row aggregate
|
|
153
|
+
> is 22.2% / 20%. Run `shadowshield benchmark --generalization all`; these gaps
|
|
154
|
+
> are public by design.
|
|
154
155
|
|
|
155
156
|
---
|
|
156
157
|
|
|
@@ -295,7 +296,7 @@ shadowshield detectors # list registered detectors
|
|
|
295
296
|
shadowshield init > shield.yaml # write an annotated default config
|
|
296
297
|
shadowshield benchmark # run the bundled offline benchmark
|
|
297
298
|
shadowshield benchmark --adversarial
|
|
298
|
-
shadowshield benchmark --generalization
|
|
299
|
+
shadowshield benchmark --generalization all # all frozen blind semantic snapshots
|
|
299
300
|
shadowshield serve # HTTP server + live dashboard (needs [dashboard])
|
|
300
301
|
```
|
|
301
302
|
|
|
@@ -329,15 +330,18 @@ export SHADOWSHIELD_ADMIN_KEY="$(openssl rand -hex 32)"
|
|
|
329
330
|
export SHADOWSHIELD_POLICY_KEY="$(openssl rand -hex 32)"
|
|
330
331
|
export SHADOWSHIELD_POLICY_STATE_KEY="$(openssl rand -hex 32)"
|
|
331
332
|
export SHADOWSHIELD_IMAGE_DIGEST="$(curl -fsSL \
|
|
332
|
-
https://github.com/0xsl1m/shadowshield/releases/download/v0.6.
|
|
333
|
+
https://github.com/0xsl1m/shadowshield/releases/download/v0.6.2/container-digest.txt)"
|
|
333
334
|
docker compose pull
|
|
334
335
|
docker compose up -d
|
|
335
336
|
```
|
|
336
337
|
|
|
337
|
-
The release workflow scans that exact image, publishes it to GHCR,
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
338
|
+
The release workflow scans that exact image, publishes it to GHCR, signs SLSA
|
|
339
|
+
provenance plus the CycloneDX SBOM with ephemeral GitHub OIDC/Sigstore identity,
|
|
340
|
+
verifies the registry-attached source/workflow claims and an anonymous digest
|
|
341
|
+
pull, and attaches the digest plus SBOM to the matching GitHub Release. Stable
|
|
342
|
+
release tags are accepted only from an exact green `main` commit. All four
|
|
343
|
+
secrets must be independent. Terminate TLS at a trusted ingress before exposing
|
|
344
|
+
it beyond localhost. See the
|
|
341
345
|
[production-readiness roadmap](docs/PRODUCTION_READINESS.md) for launch gates,
|
|
342
346
|
known scale limits, and the operator checklist.
|
|
343
347
|
|
|
@@ -347,14 +351,14 @@ Upgrading a control-plane volume from 0.6.0 requires an offline re-key because
|
|
|
347
351
|
```bash
|
|
348
352
|
# Load the existing scan/admin keys first so the new Compose file can resolve.
|
|
349
353
|
export SHADOWSHIELD_IMAGE_DIGEST="$(curl -fsSL \
|
|
350
|
-
https://github.com/0xsl1m/shadowshield/releases/download/v0.6.
|
|
354
|
+
https://github.com/0xsl1m/shadowshield/releases/download/v0.6.2/container-digest.txt)"
|
|
351
355
|
export SHADOWSHIELD_POLICY_KEY="<existing-0.6.0-policy-key>"
|
|
352
356
|
export SHADOWSHIELD_POLICY_STATE_KEY="$(openssl rand -hex 32)"
|
|
353
357
|
# Stop every writer and snapshot the volume before running the migration.
|
|
354
358
|
docker compose stop shadowshield
|
|
355
359
|
docker compose run --rm --no-deps shadowshield \
|
|
356
360
|
shadowshield migrate-policy-state --path /var/lib/shadowshield/policy-state.json
|
|
357
|
-
# Preserve the reported .pre-0.6.1.bak file, then start 0.6.
|
|
361
|
+
# Preserve the reported .pre-0.6.1.bak file, then start 0.6.2.
|
|
358
362
|
docker compose up -d
|
|
359
363
|
```
|
|
360
364
|
|
|
@@ -606,7 +610,7 @@ shadowshield serve --control --api-key SECRET # require X-API-Key / Bearer
|
|
|
606
610
|
shadowshield schema # config JSON Schema (editor/CI validation)
|
|
607
611
|
shadowshield owasp # OWASP LLM Top 10 (2025) coverage map
|
|
608
612
|
shadowshield benchmark --adversarial # curated regression set
|
|
609
|
-
shadowshield benchmark --generalization
|
|
613
|
+
shadowshield benchmark --generalization all # all frozen blind semantic snapshots
|
|
610
614
|
```
|
|
611
615
|
|
|
612
616
|
## Documentation
|
|
@@ -76,8 +76,9 @@ print(result.safe_text) # safe fallback message
|
|
|
76
76
|
> in-distribution **regression baseline, not a SOTA claim**. We publish the humbling
|
|
77
77
|
> external numbers on purpose — a credible security tool shows its homework.
|
|
78
78
|
> The frozen blind semantic snapshots are harder still: v1 reaches 26.7% recall /
|
|
79
|
-
> 13.3% FPR
|
|
80
|
-
> `shadowshield benchmark --generalization
|
|
79
|
+
> 13.3% FPR, v2 reaches 0% / 10%, and v3 reaches 30% / 30%; the 90-row aggregate
|
|
80
|
+
> is 22.2% / 20%. Run `shadowshield benchmark --generalization all`; these gaps
|
|
81
|
+
> are public by design.
|
|
81
82
|
|
|
82
83
|
---
|
|
83
84
|
|
|
@@ -222,7 +223,7 @@ shadowshield detectors # list registered detectors
|
|
|
222
223
|
shadowshield init > shield.yaml # write an annotated default config
|
|
223
224
|
shadowshield benchmark # run the bundled offline benchmark
|
|
224
225
|
shadowshield benchmark --adversarial
|
|
225
|
-
shadowshield benchmark --generalization
|
|
226
|
+
shadowshield benchmark --generalization all # all frozen blind semantic snapshots
|
|
226
227
|
shadowshield serve # HTTP server + live dashboard (needs [dashboard])
|
|
227
228
|
```
|
|
228
229
|
|
|
@@ -256,15 +257,18 @@ export SHADOWSHIELD_ADMIN_KEY="$(openssl rand -hex 32)"
|
|
|
256
257
|
export SHADOWSHIELD_POLICY_KEY="$(openssl rand -hex 32)"
|
|
257
258
|
export SHADOWSHIELD_POLICY_STATE_KEY="$(openssl rand -hex 32)"
|
|
258
259
|
export SHADOWSHIELD_IMAGE_DIGEST="$(curl -fsSL \
|
|
259
|
-
https://github.com/0xsl1m/shadowshield/releases/download/v0.6.
|
|
260
|
+
https://github.com/0xsl1m/shadowshield/releases/download/v0.6.2/container-digest.txt)"
|
|
260
261
|
docker compose pull
|
|
261
262
|
docker compose up -d
|
|
262
263
|
```
|
|
263
264
|
|
|
264
|
-
The release workflow scans that exact image, publishes it to GHCR,
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
265
|
+
The release workflow scans that exact image, publishes it to GHCR, signs SLSA
|
|
266
|
+
provenance plus the CycloneDX SBOM with ephemeral GitHub OIDC/Sigstore identity,
|
|
267
|
+
verifies the registry-attached source/workflow claims and an anonymous digest
|
|
268
|
+
pull, and attaches the digest plus SBOM to the matching GitHub Release. Stable
|
|
269
|
+
release tags are accepted only from an exact green `main` commit. All four
|
|
270
|
+
secrets must be independent. Terminate TLS at a trusted ingress before exposing
|
|
271
|
+
it beyond localhost. See the
|
|
268
272
|
[production-readiness roadmap](docs/PRODUCTION_READINESS.md) for launch gates,
|
|
269
273
|
known scale limits, and the operator checklist.
|
|
270
274
|
|
|
@@ -274,14 +278,14 @@ Upgrading a control-plane volume from 0.6.0 requires an offline re-key because
|
|
|
274
278
|
```bash
|
|
275
279
|
# Load the existing scan/admin keys first so the new Compose file can resolve.
|
|
276
280
|
export SHADOWSHIELD_IMAGE_DIGEST="$(curl -fsSL \
|
|
277
|
-
https://github.com/0xsl1m/shadowshield/releases/download/v0.6.
|
|
281
|
+
https://github.com/0xsl1m/shadowshield/releases/download/v0.6.2/container-digest.txt)"
|
|
278
282
|
export SHADOWSHIELD_POLICY_KEY="<existing-0.6.0-policy-key>"
|
|
279
283
|
export SHADOWSHIELD_POLICY_STATE_KEY="$(openssl rand -hex 32)"
|
|
280
284
|
# Stop every writer and snapshot the volume before running the migration.
|
|
281
285
|
docker compose stop shadowshield
|
|
282
286
|
docker compose run --rm --no-deps shadowshield \
|
|
283
287
|
shadowshield migrate-policy-state --path /var/lib/shadowshield/policy-state.json
|
|
284
|
-
# Preserve the reported .pre-0.6.1.bak file, then start 0.6.
|
|
288
|
+
# Preserve the reported .pre-0.6.1.bak file, then start 0.6.2.
|
|
285
289
|
docker compose up -d
|
|
286
290
|
```
|
|
287
291
|
|
|
@@ -533,7 +537,7 @@ shadowshield serve --control --api-key SECRET # require X-API-Key / Bearer
|
|
|
533
537
|
shadowshield schema # config JSON Schema (editor/CI validation)
|
|
534
538
|
shadowshield owasp # OWASP LLM Top 10 (2025) coverage map
|
|
535
539
|
shadowshield benchmark --adversarial # curated regression set
|
|
536
|
-
shadowshield benchmark --generalization
|
|
540
|
+
shadowshield benchmark --generalization all # all frozen blind semantic snapshots
|
|
537
541
|
```
|
|
538
542
|
|
|
539
543
|
## Documentation
|
|
@@ -15,6 +15,8 @@ shadowshield benchmark --adversarial
|
|
|
15
15
|
# Independently authored blind semantic snapshots
|
|
16
16
|
shadowshield benchmark --generalization v1
|
|
17
17
|
shadowshield benchmark --generalization v2
|
|
18
|
+
shadowshield benchmark --generalization v3
|
|
19
|
+
shadowshield benchmark --generalization all
|
|
18
20
|
|
|
19
21
|
# External public dataset (needs the datasets extra)
|
|
20
22
|
pip install "shadowshield[datasets]"
|
|
@@ -52,25 +54,35 @@ cases visible, so its perfect score is not a generalization claim.
|
|
|
52
54
|
|
|
53
55
|
## 3. Blind semantic snapshots (the anti-gaming result)
|
|
54
56
|
|
|
55
|
-
|
|
57
|
+
Three balanced snapshots were authored independently without detector or regex
|
|
56
58
|
context, then frozen. They deliberately pair attacks with benign text containing
|
|
57
59
|
similar roleplay, authority, debug, and policy vocabulary.
|
|
58
60
|
|
|
59
|
-
Isolation protocol: the authoring
|
|
60
|
-
names, required row/balance counts, and a request for attack/benign
|
|
61
|
-
pairs.
|
|
62
|
-
|
|
61
|
+
Isolation protocol: the v1/v2 authoring tasks received only the five semantic
|
|
62
|
+
category names, required row/balance counts, and a request for attack/benign
|
|
63
|
+
contrast pairs. The v3 task received modality coverage requirements but no
|
|
64
|
+
detector source, signatures, or current predictions. The files were frozen before
|
|
65
|
+
detector tuning and are integrity-pinned in tests:
|
|
63
66
|
v1 SHA-256 `b3281ba1a42d266bb930bbb41943016d47b38dbc822ff7cff5131f3448a0248f`;
|
|
64
|
-
v2 SHA-256 `aa8b8c81c00a55bb65180e15ff743b6241d24845b3886e8e60b52b9b23db47fa
|
|
67
|
+
v2 SHA-256 `aa8b8c81c00a55bb65180e15ff743b6241d24845b3886e8e60b52b9b23db47fa`;
|
|
68
|
+
v3 SHA-256 `2285031e8143572311a522a4b6ec1a39c96a34ac2b42785f17818e8a145342bf`.
|
|
65
69
|
|
|
66
70
|
| snapshot | rows | TP/FP/TN/FN | recall | FPR | balanced accuracy |
|
|
67
71
|
|---|---:|---:|---:|---:|---:|
|
|
68
72
|
| v1 | 30 | 4/2/13/11 | 26.7% | 13.3% | 56.7% |
|
|
69
73
|
| v2 | 20 | 0/1/9/10 | 0% | 10% | 45% |
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
+
| v3 | 40 | 6/6/14/14 | 30% | 30% | 50% |
|
|
75
|
+
| **aggregate** | **90** | **10/9/36/35** | **22.2%** | **20%** | **51.1%** |
|
|
76
|
+
|
|
77
|
+
The v3 snapshot was opened only after a detector candidate and acceptance bar
|
|
78
|
+
were frozen. That candidate raised aggregate recall to 55.6% but failed the
|
|
79
|
+
predeclared false-positive and balanced-accuracy gates (24.4% FPR; 65.6%
|
|
80
|
+
balanced accuracy), so it was discarded in full. No v3-driven detector tuning
|
|
81
|
+
ships in 0.6.2; the table records the unchanged released core.
|
|
82
|
+
|
|
83
|
+
These snapshots expose the core deterministic tier's semantic-generalization
|
|
84
|
+
ceiling and are development evidence for a fresh-corpus classifier/conjunction
|
|
85
|
+
tranche, not numbers to hide or a set to tune against.
|
|
74
86
|
|
|
75
87
|
## 4. External: `deepset/prompt-injections` (out-of-distribution)
|
|
76
88
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Production readiness roadmap
|
|
2
2
|
|
|
3
|
-
**Baseline:** ShadowShield 0.6.
|
|
3
|
+
**Baseline:** ShadowShield 0.6.2 · **Audit date:** 2026-07-25
|
|
4
4
|
|
|
5
5
|
This roadmap is the release gate for the Python library and the optional HTTP
|
|
6
6
|
control plane. A checked item is implemented and locally verified; unchecked
|
|
@@ -10,12 +10,12 @@ items require an explicit operator or maintainer decision before public launch.
|
|
|
10
10
|
|
|
11
11
|
| Area | State | Evidence / remaining action |
|
|
12
12
|
|---|---|---|
|
|
13
|
-
| Correctness | Ready |
|
|
13
|
+
| Correctness | Ready | 290+ unit/integration cases; strict mypy; Ruff lint/format; CodeQL extended scanning |
|
|
14
14
|
| Packaging | Ready | Isolated sdist/wheel build, Twine metadata check, installed-wheel CI smoke test |
|
|
15
15
|
| Runtime security | Ready with configuration | Early API-key/Bearer auth, body limits, restricted CORS, immutable protection floor, non-root read-only container |
|
|
16
|
-
| Supply chain | Ready |
|
|
16
|
+
| Supply chain | Ready | Repository-enforced Action SHA pins, PyPI OIDC, exact-green-main release gates, pre-push image vulnerability gate, anonymous GHCR digest pull, registry-verified SLSA and CycloneDX attestations |
|
|
17
17
|
| Observability | Ready for single process | Content-free telemetry, Prometheus endpoint, bounded in-memory event feed |
|
|
18
|
-
| Detection quality | Beta | Curated adversarial: 100%/0% FPR; blind v1:
|
|
18
|
+
| Detection quality | Beta | Curated adversarial: 100%/0% FPR; blind v1–v3 aggregate: 22.2%/20%; v3: 30%/30%; deepset core: 23.3%/0% |
|
|
19
19
|
| Scale / HA | Not yet | Counters, events, configuration, and rate limits are process-local |
|
|
20
20
|
| Streaming | Not yet | Outputs must be buffered before scanning |
|
|
21
21
|
|
|
@@ -48,6 +48,19 @@ items require an explicit operator or maintainer decision before public launch.
|
|
|
48
48
|
- [x] Pin the base image by digest, gate the exact release image on Trivy, publish
|
|
49
49
|
it to GHCR, verify an anonymous digest pull, and attach its immutable digest
|
|
50
50
|
plus CycloneDX SBOM to the release.
|
|
51
|
+
- [x] Sign release-image SLSA provenance and the CycloneDX SBOM with ephemeral
|
|
52
|
+
GitHub OIDC/Sigstore identity, then verify repository, workflow, runner, and
|
|
53
|
+
source-commit claims from the OCI registry before completing the release.
|
|
54
|
+
- [x] Reject release publication unless a stable, non-prerelease tag matches the
|
|
55
|
+
package version, resolves to `main`, and has a successful exact-SHA `main` CI run.
|
|
56
|
+
- [x] Reject symlink/non-regular durable policy-state files, use unpredictable
|
|
57
|
+
exclusive temporary files, and revision-check reads, migrations, and replaces.
|
|
58
|
+
- [x] Coalesce concurrent failed Transformer/Vector load generations while giving
|
|
59
|
+
every waiter a fresh bounded exception traceback and allowing a later retry.
|
|
60
|
+
- [x] Enable Dependabot security updates, extended CodeQL scanning, secret
|
|
61
|
+
push-protection, and repository-level enforcement of full-SHA Action pins.
|
|
62
|
+
- [x] Publish a third independently authored blind snapshot and reject—not
|
|
63
|
+
tune—the first frozen candidate when it missed the predeclared FPR/accuracy bar.
|
|
51
64
|
|
|
52
65
|
## P1 — launch procedure (operator-owned)
|
|
53
66
|
|
|
@@ -70,11 +83,13 @@ items require an explicit operator or maintainer decision before public launch.
|
|
|
70
83
|
|
|
71
84
|
1. Add a rolling-window streaming scanner that blocks before unsafe output is emitted.
|
|
72
85
|
2. Move rate limits, event history, and metrics aggregation to shared external stores for multi-worker HA.
|
|
73
|
-
3.
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
86
|
+
3. Build a fresh independently sourced development corpus, then evaluate the
|
|
87
|
+
next detector candidate once on a new sealed snapshot and publish confidence intervals.
|
|
88
|
+
4. Add multi-hour soak and network fault-injection tests for reporters, judges, and hot policy updates.
|
|
89
|
+
5. Split the current administrator credential into narrower observe and mutation scopes.
|
|
90
|
+
6. Design cross-process model/index coordination and durable Reporter spooling.
|
|
91
|
+
7. Split unprivileged container build/scan from the write/OIDC publication job.
|
|
92
|
+
8. Refactor release evidence into a draft-first pipeline, then enable immutable releases.
|
|
78
93
|
|
|
79
94
|
## Go / no-go gates
|
|
80
95
|
|
|
@@ -8,9 +8,10 @@ this specific GitHub workflow directly. The pipeline is
|
|
|
8
8
|
The same GitHub Release also triggers
|
|
9
9
|
[`container-release.yml`](../.github/workflows/container-release.yml). It builds
|
|
10
10
|
one image from the release tag, generates a CycloneDX SBOM, rejects fixable
|
|
11
|
-
high/critical findings, pushes that exact scanned image to GHCR,
|
|
12
|
-
|
|
13
|
-
the
|
|
11
|
+
high/critical findings, pushes that exact scanned image to GHCR, signs SLSA
|
|
12
|
+
provenance and CycloneDX attestations through GitHub's Sigstore-backed OIDC
|
|
13
|
+
service, verifies the signed provenance and an anonymous pull by digest, and
|
|
14
|
+
attaches `container-digest.txt` plus the SBOM to the GitHub Release.
|
|
14
15
|
|
|
15
16
|
## One-time setup (PyPI account owner only)
|
|
16
17
|
|
|
@@ -62,13 +63,10 @@ Do not create the release while required checks are pending. A partial release
|
|
|
62
63
|
is recoverable by fixing the failed workflow and re-running it, but the PyPI
|
|
63
64
|
version itself cannot be replaced.
|
|
64
65
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
gh workflow run publish.yml --ref main
|
|
70
|
-
gh run watch # follow it
|
|
71
|
-
```
|
|
66
|
+
Both publication workflows independently reject non-stable tags, tag/source
|
|
67
|
+
version mismatches, commits outside `main`, and commits without a successful
|
|
68
|
+
exact-SHA `main` CI run. There is no manual-dispatch publication path; use
|
|
69
|
+
GitHub's **Re-run failed jobs** control to recover a failed release workflow.
|
|
72
70
|
|
|
73
71
|
## Verify
|
|
74
72
|
|
|
@@ -79,7 +77,22 @@ python -c "import shadowshield as ss; print(ss.__version__)"
|
|
|
79
77
|
|
|
80
78
|
gh release download vX.Y.Z \
|
|
81
79
|
--pattern container-digest.txt --pattern shadowshield-sbom.cdx.json
|
|
82
|
-
|
|
80
|
+
digest="$(cat container-digest.txt)"
|
|
81
|
+
image="ghcr.io/0xsl1m/shadowshield@$digest"
|
|
82
|
+
docker pull "$image"
|
|
83
|
+
gh attestation verify "oci://$image" \
|
|
84
|
+
--repo 0xsl1m/shadowshield \
|
|
85
|
+
--bundle-from-oci \
|
|
86
|
+
--signer-workflow 0xsl1m/shadowshield/.github/workflows/container-release.yml \
|
|
87
|
+
--source-digest "$(git rev-list -n 1 vX.Y.Z)" \
|
|
88
|
+
--deny-self-hosted-runners
|
|
89
|
+
gh attestation verify "oci://$image" \
|
|
90
|
+
--repo 0xsl1m/shadowshield \
|
|
91
|
+
--bundle-from-oci \
|
|
92
|
+
--predicate-type https://cyclonedx.org/bom \
|
|
93
|
+
--signer-workflow 0xsl1m/shadowshield/.github/workflows/container-release.yml \
|
|
94
|
+
--source-digest "$(git rev-list -n 1 vX.Y.Z)" \
|
|
95
|
+
--deny-self-hosted-runners
|
|
83
96
|
```
|
|
84
97
|
|
|
85
98
|
## Notes
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "shadowshield"
|
|
7
|
-
version = "0.6.
|
|
7
|
+
version = "0.6.2"
|
|
8
8
|
description = "Unified open-source security shield for agentic AI systems — inspired by Sentinel & ShadowClaw."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -242,7 +242,7 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
242
242
|
)
|
|
243
243
|
dataset_source.add_argument(
|
|
244
244
|
"--generalization",
|
|
245
|
-
choices=["v1", "v2", "all"],
|
|
245
|
+
choices=["v1", "v2", "v3", "all"],
|
|
246
246
|
default=None,
|
|
247
247
|
help="use an independently authored blind semantic snapshot",
|
|
248
248
|
)
|