shadowshield 0.6.2__tar.gz → 0.6.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. {shadowshield-0.6.2 → shadowshield-0.6.3}/PKG-INFO +6 -7
  2. {shadowshield-0.6.2 → shadowshield-0.6.3}/README.md +5 -5
  3. {shadowshield-0.6.2 → shadowshield-0.6.3}/docs/BENCHMARKS.md +35 -9
  4. {shadowshield-0.6.2 → shadowshield-0.6.3}/docs/PRODUCTION_READINESS.md +34 -11
  5. {shadowshield-0.6.2 → shadowshield-0.6.3}/docs/RELEASING.md +22 -5
  6. {shadowshield-0.6.2 → shadowshield-0.6.3}/docs/configuration.md +6 -5
  7. {shadowshield-0.6.2 → shadowshield-0.6.3}/docs/plugins.md +3 -1
  8. {shadowshield-0.6.2 → shadowshield-0.6.3}/docs/security-model.md +4 -2
  9. {shadowshield-0.6.2 → shadowshield-0.6.3}/pyproject.toml +11 -3
  10. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/__init__.py +1 -1
  11. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/_security.py +107 -20
  12. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/cli.py +11 -6
  13. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/config/default.yaml +4 -0
  14. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/control.py +57 -8
  15. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/core/config.py +6 -0
  16. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/core/engine.py +49 -10
  17. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/core/session.py +34 -19
  18. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/detectors/anomaly.py +2 -1
  19. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/detectors/base.py +2 -0
  20. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/detectors/exfiltration.py +6 -0
  21. shadowshield-0.6.3/src/shadowshield/eval/harness.py +432 -0
  22. {shadowshield-0.6.2 → shadowshield-0.6.3}/tests/test_cli.py +58 -0
  23. {shadowshield-0.6.2 → shadowshield-0.6.3}/tests/test_config.py +2 -0
  24. {shadowshield-0.6.2 → shadowshield-0.6.3}/tests/test_control.py +89 -1
  25. {shadowshield-0.6.2 → shadowshield-0.6.3}/tests/test_detectors.py +17 -0
  26. shadowshield-0.6.3/tests/test_eval_harness.py +240 -0
  27. shadowshield-0.6.3/tests/test_http_security.py +326 -0
  28. {shadowshield-0.6.2 → shadowshield-0.6.3}/tests/test_production.py +55 -0
  29. {shadowshield-0.6.2 → shadowshield-0.6.3}/tests/test_shield.py +49 -0
  30. {shadowshield-0.6.2 → shadowshield-0.6.3}/tests/test_site_security.py +13 -0
  31. shadowshield-0.6.3/tests/test_workflow_policy.py +114 -0
  32. shadowshield-0.6.2/src/shadowshield/eval/harness.py +0 -206
  33. shadowshield-0.6.2/tests/test_http_security.py +0 -138
  34. {shadowshield-0.6.2 → shadowshield-0.6.3}/.gitignore +0 -0
  35. {shadowshield-0.6.2 → shadowshield-0.6.3}/LICENSE +0 -0
  36. {shadowshield-0.6.2 → shadowshield-0.6.3}/docs/CODE_REVIEW.md +0 -0
  37. {shadowshield-0.6.2 → shadowshield-0.6.3}/docs/COMPARISON.md +0 -0
  38. {shadowshield-0.6.2 → shadowshield-0.6.3}/docs/MARKET_LANDSCAPE.md +0 -0
  39. {shadowshield-0.6.2 → shadowshield-0.6.3}/docs/NEXT_STEPS.md +0 -0
  40. {shadowshield-0.6.2 → shadowshield-0.6.3}/docs/OWASP_LLM_TOP10.md +0 -0
  41. {shadowshield-0.6.2 → shadowshield-0.6.3}/docs/PLAN_REVIEW.md +0 -0
  42. {shadowshield-0.6.2 → shadowshield-0.6.3}/docs/REPORTER_SDK_SPEC.md +0 -0
  43. {shadowshield-0.6.2 → shadowshield-0.6.3}/docs/SAAS_STRATEGY.md +0 -0
  44. {shadowshield-0.6.2 → shadowshield-0.6.3}/docs/UPGRADE_OPPORTUNITIES.md +0 -0
  45. {shadowshield-0.6.2 → shadowshield-0.6.3}/docs/detectors.md +0 -0
  46. {shadowshield-0.6.2 → shadowshield-0.6.3}/docs/index.md +0 -0
  47. {shadowshield-0.6.2 → shadowshield-0.6.3}/docs/research/LANDSCAPE.md +0 -0
  48. {shadowshield-0.6.2 → shadowshield-0.6.3}/examples/agentic_security.py +0 -0
  49. {shadowshield-0.6.2 → shadowshield-0.6.3}/examples/custom_detector.py +0 -0
  50. {shadowshield-0.6.2 → shadowshield-0.6.3}/examples/langchain_integration.py +0 -0
  51. {shadowshield-0.6.2 → shadowshield-0.6.3}/examples/openai_integration.py +0 -0
  52. {shadowshield-0.6.2 → shadowshield-0.6.3}/examples/quickstart.py +0 -0
  53. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/config/__init__.py +0 -0
  54. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/core/__init__.py +0 -0
  55. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/core/canary.py +0 -0
  56. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/core/coverage.py +0 -0
  57. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/core/policy.py +0 -0
  58. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/core/shield.py +0 -0
  59. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/core/telemetry.py +0 -0
  60. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/core/types.py +0 -0
  61. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/detectors/__init__.py +0 -0
  62. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/detectors/alignment.py +0 -0
  63. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/detectors/canary.py +0 -0
  64. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/detectors/data/attack_corpus.txt +0 -0
  65. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/detectors/encoding.py +0 -0
  66. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/detectors/jailbreak.py +0 -0
  67. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/detectors/llm_check.py +0 -0
  68. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/detectors/pii.py +0 -0
  69. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/detectors/prompt_injection.py +0 -0
  70. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/detectors/transformer.py +0 -0
  71. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/detectors/vector.py +0 -0
  72. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/eval/__init__.py +0 -0
  73. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/eval/data/adversarial_benchmark.jsonl +0 -0
  74. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/eval/data/builtin_benchmark.jsonl +0 -0
  75. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/eval/data/generalization_benchmark_v1.jsonl +0 -0
  76. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/eval/data/generalization_benchmark_v2.jsonl +0 -0
  77. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/eval/data/generalization_benchmark_v3.jsonl +0 -0
  78. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/eval/dataset.py +0 -0
  79. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/integrations/__init__.py +0 -0
  80. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/integrations/agentdojo.py +0 -0
  81. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/integrations/mcp.py +0 -0
  82. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/middleware/__init__.py +0 -0
  83. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/middleware/base.py +0 -0
  84. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/middleware/decorators.py +0 -0
  85. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/middleware/langchain.py +0 -0
  86. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/middleware/openai.py +0 -0
  87. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/plugins/__init__.py +0 -0
  88. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/plugins/base.py +0 -0
  89. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/plugins/manager.py +0 -0
  90. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/py.typed +0 -0
  91. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/reporter.py +0 -0
  92. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/responders/__init__.py +0 -0
  93. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/responders/base.py +0 -0
  94. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/responders/blocker.py +0 -0
  95. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/responders/isolator.py +0 -0
  96. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/responders/rate_limiter.py +0 -0
  97. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/responders/sanitizer.py +0 -0
  98. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/server.py +0 -0
  99. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/static/dashboard.html +0 -0
  100. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/utils/__init__.py +0 -0
  101. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/utils/logging.py +0 -0
  102. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/utils/scoring.py +0 -0
  103. {shadowshield-0.6.2 → shadowshield-0.6.3}/src/shadowshield/utils/text.py +0 -0
  104. {shadowshield-0.6.2 → shadowshield-0.6.3}/tests/__init__.py +0 -0
  105. {shadowshield-0.6.2 → shadowshield-0.6.3}/tests/test_adversarial.py +0 -0
  106. {shadowshield-0.6.2 → shadowshield-0.6.3}/tests/test_agentic.py +0 -0
  107. {shadowshield-0.6.2 → shadowshield-0.6.3}/tests/test_generalization_benchmarks.py +0 -0
  108. {shadowshield-0.6.2 → shadowshield-0.6.3}/tests/test_mcp_guard.py +0 -0
  109. {shadowshield-0.6.2 → shadowshield-0.6.3}/tests/test_middleware.py +0 -0
  110. {shadowshield-0.6.2 → shadowshield-0.6.3}/tests/test_multilingual.py +0 -0
  111. {shadowshield-0.6.2 → shadowshield-0.6.3}/tests/test_pii_backends.py +0 -0
  112. {shadowshield-0.6.2 → shadowshield-0.6.3}/tests/test_policy.py +0 -0
  113. {shadowshield-0.6.2 → shadowshield-0.6.3}/tests/test_prompt_injection.py +0 -0
  114. {shadowshield-0.6.2 → shadowshield-0.6.3}/tests/test_responders.py +0 -0
  115. {shadowshield-0.6.2 → shadowshield-0.6.3}/tests/test_server.py +0 -0
  116. {shadowshield-0.6.2 → shadowshield-0.6.3}/tests/test_spans.py +0 -0
  117. {shadowshield-0.6.2 → shadowshield-0.6.3}/tests/test_telemetry.py +0 -0
  118. {shadowshield-0.6.2 → shadowshield-0.6.3}/tests/test_transformer.py +0 -0
  119. {shadowshield-0.6.2 → shadowshield-0.6.3}/tests/test_vector.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: shadowshield
3
- Version: 0.6.2
3
+ Version: 0.6.3
4
4
  Summary: Unified open-source security shield for agentic AI systems — inspired by Sentinel & ShadowClaw.
5
5
  Project-URL: Homepage, https://shadowshield.xyz
6
6
  Project-URL: Documentation, https://github.com/0xsl1m/shadowshield#readme
@@ -28,7 +28,6 @@ Requires-Dist: httpx>=0.27
28
28
  Requires-Dist: pydantic>=2.5
29
29
  Requires-Dist: pyyaml>=6.0
30
30
  Requires-Dist: structlog>=24.1
31
- Requires-Dist: tiktoken>=0.6
32
31
  Provides-Extra: all
33
32
  Requires-Dist: datasets>=2.18; extra == 'all'
34
33
  Requires-Dist: fastapi>=0.110; extra == 'all'
@@ -207,8 +206,8 @@ pip install "shadowshield[dashboard]" # + FastAPI HTTP server & dashboard
207
206
  pip install "shadowshield[all]" # everything
208
207
  ```
209
208
 
210
- Core deps are intentionally small: `pydantic`, `structlog`, `pyyaml`, `httpx`,
211
- `tiktoken`. The ML classifier, Presidio PII, dataset loaders, and dashboard live
209
+ Core deps are intentionally small: `pydantic`, `structlog`, `pyyaml`, and
210
+ `httpx`. The ML classifier, Presidio PII, dataset loaders, and dashboard live
212
211
  behind extras — the default install pulls **no** heavy ML stack.
213
212
 
214
213
  ---
@@ -330,7 +329,7 @@ export SHADOWSHIELD_ADMIN_KEY="$(openssl rand -hex 32)"
330
329
  export SHADOWSHIELD_POLICY_KEY="$(openssl rand -hex 32)"
331
330
  export SHADOWSHIELD_POLICY_STATE_KEY="$(openssl rand -hex 32)"
332
331
  export SHADOWSHIELD_IMAGE_DIGEST="$(curl -fsSL \
333
- https://github.com/0xsl1m/shadowshield/releases/download/v0.6.2/container-digest.txt)"
332
+ https://github.com/0xsl1m/shadowshield/releases/download/v0.6.3/container-digest.txt)"
334
333
  docker compose pull
335
334
  docker compose up -d
336
335
  ```
@@ -351,14 +350,14 @@ Upgrading a control-plane volume from 0.6.0 requires an offline re-key because
351
350
  ```bash
352
351
  # Load the existing scan/admin keys first so the new Compose file can resolve.
353
352
  export SHADOWSHIELD_IMAGE_DIGEST="$(curl -fsSL \
354
- https://github.com/0xsl1m/shadowshield/releases/download/v0.6.2/container-digest.txt)"
353
+ https://github.com/0xsl1m/shadowshield/releases/download/v0.6.3/container-digest.txt)"
355
354
  export SHADOWSHIELD_POLICY_KEY="<existing-0.6.0-policy-key>"
356
355
  export SHADOWSHIELD_POLICY_STATE_KEY="$(openssl rand -hex 32)"
357
356
  # Stop every writer and snapshot the volume before running the migration.
358
357
  docker compose stop shadowshield
359
358
  docker compose run --rm --no-deps shadowshield \
360
359
  shadowshield migrate-policy-state --path /var/lib/shadowshield/policy-state.json
361
- # Preserve the reported .pre-0.6.1.bak file, then start 0.6.2.
360
+ # Preserve the reported .pre-0.6.1.bak file, then start 0.6.3.
362
361
  docker compose up -d
363
362
  ```
364
363
 
@@ -134,8 +134,8 @@ pip install "shadowshield[dashboard]" # + FastAPI HTTP server & dashboard
134
134
  pip install "shadowshield[all]" # everything
135
135
  ```
136
136
 
137
- Core deps are intentionally small: `pydantic`, `structlog`, `pyyaml`, `httpx`,
138
- `tiktoken`. The ML classifier, Presidio PII, dataset loaders, and dashboard live
137
+ Core deps are intentionally small: `pydantic`, `structlog`, `pyyaml`, and
138
+ `httpx`. The ML classifier, Presidio PII, dataset loaders, and dashboard live
139
139
  behind extras — the default install pulls **no** heavy ML stack.
140
140
 
141
141
  ---
@@ -257,7 +257,7 @@ export SHADOWSHIELD_ADMIN_KEY="$(openssl rand -hex 32)"
257
257
  export SHADOWSHIELD_POLICY_KEY="$(openssl rand -hex 32)"
258
258
  export SHADOWSHIELD_POLICY_STATE_KEY="$(openssl rand -hex 32)"
259
259
  export SHADOWSHIELD_IMAGE_DIGEST="$(curl -fsSL \
260
- https://github.com/0xsl1m/shadowshield/releases/download/v0.6.2/container-digest.txt)"
260
+ https://github.com/0xsl1m/shadowshield/releases/download/v0.6.3/container-digest.txt)"
261
261
  docker compose pull
262
262
  docker compose up -d
263
263
  ```
@@ -278,14 +278,14 @@ Upgrading a control-plane volume from 0.6.0 requires an offline re-key because
278
278
  ```bash
279
279
  # Load the existing scan/admin keys first so the new Compose file can resolve.
280
280
  export SHADOWSHIELD_IMAGE_DIGEST="$(curl -fsSL \
281
- https://github.com/0xsl1m/shadowshield/releases/download/v0.6.2/container-digest.txt)"
281
+ https://github.com/0xsl1m/shadowshield/releases/download/v0.6.3/container-digest.txt)"
282
282
  export SHADOWSHIELD_POLICY_KEY="<existing-0.6.0-policy-key>"
283
283
  export SHADOWSHIELD_POLICY_STATE_KEY="$(openssl rand -hex 32)"
284
284
  # Stop every writer and snapshot the volume before running the migration.
285
285
  docker compose stop shadowshield
286
286
  docker compose run --rm --no-deps shadowshield \
287
287
  shadowshield migrate-policy-state --path /var/lib/shadowshield/policy-state.json
288
- # Preserve the reported .pre-0.6.1.bak file, then start 0.6.2.
288
+ # Preserve the reported .pre-0.6.1.bak file, then start 0.6.3.
289
289
  docker compose up -d
290
290
  ```
291
291
 
@@ -26,14 +26,40 @@ pip install "shadowshield[transformers]"
26
26
  shadowshield benchmark --hf deepset/prompt-injections --split test --transformer
27
27
  ```
28
28
 
29
+ The CLI warms every configured detector before starting the latency clock. Its
30
+ JSON and text reports include a runtime-integrity section covering warmup,
31
+ readiness, and bounded per-detector failure counts. A benchmark with a warmup
32
+ failure, a detector that remains unready, or any detector error exits non-zero
33
+ and is marked **unreliable**; its confusion counts remain available for
34
+ diagnosis but must not be published as quality evidence. CLI benchmark shields
35
+ disable normal audit emission so logging I/O does not distort latency or corrupt
36
+ the command's machine-readable JSON stdout.
37
+
38
+ Programmatic callers can request the same behavior with
39
+ `evaluate_shield(shield, examples, warmup=True)`. The default remains
40
+ `warmup=False` for API compatibility. In both modes, readiness is checked after
41
+ the scans and per-scan detector error counters are aggregated.
42
+
43
+ Per-category output retains the original `total`, `flagged`, and flag-rate
44
+ fields, and additionally reports class-conditional TP/FP/TN/FN, recall, FPR,
45
+ and balanced accuracy. Recall is `null`/`n/a` for categories with no attack
46
+ rows; FPR is `null`/`n/a` for categories with no benign rows; balanced accuracy
47
+ is defined only when both classes are present.
48
+
49
+ Aggregate and per-category recall/FPR also include two-sided 95% Wilson score
50
+ intervals. Wilson intervals remain informative at small sample sizes and at
51
+ observed rates of 0% or 100%; an interval is `null`/`n/a` when that category has
52
+ no rows for the corresponding class. Existing scalar metric fields are
53
+ unchanged, and the interval fields are additive.
54
+
29
55
  ## 1. Bundled benchmark (in-distribution — a regression baseline)
30
56
 
31
57
  75 curated examples (40 attack / 35 benign, incl. 16 NotInject-style hard
32
58
  negatives). `balanced` mode:
33
59
 
34
- | recall | FPR | precision | F1 | p50 |
60
+ | recall (95% CI) | FPR (95% CI) | precision | F1 | p50 |
35
61
  |---:|---:|---:|---:|---:|
36
- | 100% | 0% | 100% | 100% | 0.16 ms |
62
+ | 100% [91.2%, 100%] | 0% [0%, 9.9%] | 100% | 100% | 0.16 ms |
37
63
 
38
64
  **This is a regression baseline and a smoke test — NOT a claim of real-world
39
65
  accuracy.** 100% on our own set just means we don't regress on the attack
@@ -45,9 +71,9 @@ The 36-row adversarial catalogue includes obfuscation, multilingual and indirect
45
71
  attacks, plus benign trigger-heavy counterexamples. It improved from
46
72
  15/2/16/3 to 18/0/18/0 (TP/FP/TN/FN):
47
73
 
48
- | recall | FPR | precision | F1 |
74
+ | recall (95% CI) | FPR (95% CI) | precision | F1 |
49
75
  |---:|---:|---:|---:|
50
- | 100% | 0% | 100% | 100% |
76
+ | 100% [82.4%, 100%] | 0% [0%, 17.6%] | 100% | 100% |
51
77
 
52
78
  This is still a curated regression set. The signatures were developed with these
53
79
  cases visible, so its perfect score is not a generalization claim.
@@ -67,12 +93,12 @@ v1 SHA-256 `b3281ba1a42d266bb930bbb41943016d47b38dbc822ff7cff5131f3448a0248f`;
67
93
  v2 SHA-256 `aa8b8c81c00a55bb65180e15ff743b6241d24845b3886e8e60b52b9b23db47fa`;
68
94
  v3 SHA-256 `2285031e8143572311a522a4b6ec1a39c96a34ac2b42785f17818e8a145342bf`.
69
95
 
70
- | snapshot | rows | TP/FP/TN/FN | recall | FPR | balanced accuracy |
96
+ | snapshot | rows | TP/FP/TN/FN | recall (95% CI) | FPR (95% CI) | balanced accuracy |
71
97
  |---|---:|---:|---:|---:|---:|
72
- | v1 | 30 | 4/2/13/11 | 26.7% | 13.3% | 56.7% |
73
- | v2 | 20 | 0/1/9/10 | 0% | 10% | 45% |
74
- | v3 | 40 | 6/6/14/14 | 30% | 30% | 50% |
75
- | **aggregate** | **90** | **10/9/36/35** | **22.2%** | **20%** | **51.1%** |
98
+ | v1 | 30 | 4/2/13/11 | 26.7% [10.9%, 52.0%] | 13.3% [3.7%, 37.9%] | 56.7% |
99
+ | v2 | 20 | 0/1/9/10 | 0% [0%, 27.8%] | 10% [1.8%, 40.4%] | 45% |
100
+ | v3 | 40 | 6/6/14/14 | 30% [14.5%, 51.9%] | 30% [14.5%, 51.9%] | 50% |
101
+ | **aggregate** | **90** | **10/9/36/35** | **22.2% [12.5%, 36.3%]** | **20% [10.9%, 33.8%]** | **51.1%** |
76
102
 
77
103
  The v3 snapshot was opened only after a detector candidate and acceptance bar
78
104
  were frozen. That candidate raised aggregate recall to 55.6% but failed the
@@ -1,20 +1,21 @@
1
1
  # Production readiness roadmap
2
2
 
3
- **Baseline:** ShadowShield 0.6.2 · **Audit date:** 2026-07-25
3
+ **Baseline:** ShadowShield 0.6.3 · **Audit date:** 2026-07-25
4
4
 
5
5
  This roadmap is the release gate for the Python library and the optional HTTP
6
- control plane. A checked item is implemented and locally verified; unchecked
7
- items require an explicit operator or maintainer decision before public launch.
6
+ control plane. A checked item is implemented and locally verified. The
7
+ operator-owned unchecked items do not block the library or static site, but they
8
+ must be resolved before exposing the optional HTTP control plane publicly.
8
9
 
9
10
  ## Current readiness
10
11
 
11
12
  | Area | State | Evidence / remaining action |
12
13
  |---|---|---|
13
- | Correctness | Ready | 290+ unit/integration cases; strict mypy; Ruff lint/format; CodeQL extended scanning |
14
- | Packaging | Ready | Isolated sdist/wheel build, Twine metadata check, installed-wheel CI smoke test |
15
- | Runtime security | Ready with configuration | Early API-key/Bearer auth, body limits, restricted CORS, immutable protection floor, non-root read-only container |
16
- | Supply chain | Ready | Repository-enforced Action SHA pins, PyPI OIDC, exact-green-main release gates, pre-push image vulnerability gate, anonymous GHCR digest pull, registry-verified SLSA and CycloneDX attestations |
17
- | Observability | Ready for single process | Content-free telemetry, Prometheus endpoint, bounded in-memory event feed |
14
+ | Correctness | Ready | 320 unit/integration cases; strict mypy; Ruff lint/format; CodeQL extended scanning |
15
+ | Packaging | Ready | Hash-locked build environment, byte-for-byte repeat build gate, Twine validation, installed-wheel smoke test |
16
+ | Runtime security | Ready with configuration | Early API-key/Bearer auth, bounded/deadlined body intake, detector-failure policy, restricted CORS, immutable protection floor, non-root read-only container |
17
+ | Supply chain | Ready | Digest/SHA-pinned build inputs, PyPI OIDC, exact-green-main release gates, immutable image/evidence handling, pre-push vulnerability gate, anonymous GHCR digest pull, registry-verified SLSA and CycloneDX attestations |
18
+ | Observability | Ready for single process | Content-free telemetry, detector-error counters, Prometheus endpoint, bounded in-memory event feed |
18
19
  | Detection quality | Beta | Curated adversarial: 100%/0% FPR; blind v1–v3 aggregate: 22.2%/20%; v3: 30%/30%; deepset core: 23.3%/0% |
19
20
  | Scale / HA | Not yet | Counters, events, configuration, and rate limits are process-local |
20
21
  | Streaming | Not yet | Outputs must be buffered before scanning |
@@ -23,7 +24,7 @@ items require an explicit operator or maintainer decision before public launch.
23
24
 
24
25
  - [x] Repair control-plane syntax and restore test collection.
25
26
  - [x] Make optional-dependency tests and typing deterministic with or without MCP installed.
26
- - [x] Enforce an 80% branch-aware coverage floor (current local result: 85%).
27
+ - [x] Enforce an 80% branch-aware coverage floor (current local result: 86%).
27
28
  - [x] Test every declared Python runtime, 3.10 through 3.14, in CI.
28
29
  - [x] Build and validate both distribution formats and smoke-test the installed wheel.
29
30
  - [x] Audit declared dependencies in CI (local audit: no known vulnerabilities).
@@ -61,6 +62,23 @@ items require an explicit operator or maintainer decision before public launch.
61
62
  push-protection, and repository-level enforcement of full-SHA Action pins.
62
63
  - [x] Publish a third independently authored blind snapshot and reject—not
63
64
  tune—the first frozen candidate when it missed the predeclared FPR/accuracy bar.
65
+ - [x] Make documented session guard calls record clean and blocked turns exactly
66
+ once so multi-turn and alignment checks receive the advertised trace.
67
+ - [x] Surface bounded, content-free detector failures in every result, audit,
68
+ benchmark, control event, and Prometheus metric; make strict mode fail closed.
69
+ - [x] Coalesce HTTP request frames, cap fragmentation, and apply one total body
70
+ deadline so authenticated slow/chunked bodies cannot pin all scan slots;
71
+ authenticate protected non-preflight methods before body/admission work.
72
+ - [x] Hash-lock release build and container dependencies, pin the Dockerfile
73
+ frontend, remove the unused mandatory tokenizer dependency, and require two
74
+ byte-identical distribution builds.
75
+ - [x] Upgrade maintained Actions to their Node 24 generations, bound every CI
76
+ job with a timeout, and refuse conflicting image tags or release assets while
77
+ allowing only attested exact-source digest recovery of an interrupted
78
+ container publication.
79
+ - [x] Make benchmark evidence fail visibly on warmup, readiness, or detector
80
+ errors and report class-conditional per-slice confusion metrics with 95%
81
+ Wilson confidence intervals.
64
82
 
65
83
  ## P1 — launch procedure (operator-owned)
66
84
 
@@ -73,7 +91,8 @@ items require an explicit operator or maintainer decision before public launch.
73
91
  - [ ] Set an explicit CORS allowlist or leave CORS disabled.
74
92
  - [ ] Run the adversarial and application-specific eval sets in permissive/shadow mode.
75
93
  - [ ] Choose thresholds from measured false-positive cost; record the accepted baseline.
76
- - [ ] Connect Prometheus and content-free telemetry; alert on error, block-rate, and latency shifts.
94
+ - [ ] Connect Prometheus and content-free telemetry; alert on detector errors,
95
+ readiness, block-rate, and latency shifts.
77
96
  - [ ] Load-test representative payload sizes and concurrency on the target instance class.
78
97
  - [ ] Record the deployed image digest and CI SBOM, then rehearse backup/restore
79
98
  and rollback against the target environment.
@@ -89,7 +108,11 @@ items require an explicit operator or maintainer decision before public launch.
89
108
  5. Split the current administrator credential into narrower observe and mutation scopes.
90
109
  6. Design cross-process model/index coordination and durable Reporter spooling.
91
110
  7. Split unprivileged container build/scan from the write/OIDC publication job.
92
- 8. Refactor release evidence into a draft-first pipeline, then enable immutable releases.
111
+ 8. Refactor release evidence into a draft-first pipeline with independently
112
+ promoted, already-scanned OCI artifacts.
113
+ 9. Pin optional model revisions, pre-cache them for offline production startup,
114
+ and bound model-loading time.
115
+ 10. Add vulnerability-audit matrices for every supported optional dependency stack.
93
116
 
94
117
  ## Go / no-go gates
95
118
 
@@ -13,6 +13,10 @@ provenance and CycloneDX attestations through GitHub's Sigstore-backed OIDC
13
13
  service, verifies the signed provenance and an anonymous pull by digest, and
14
14
  attaches `container-digest.txt` plus the SBOM to the GitHub Release.
15
15
 
16
+ Build tooling and the production container dependency graph are version- and
17
+ hash-locked in `requirements/build.lock` and `requirements/container.lock`.
18
+ Release jobs build the sdist and wheel twice and require byte-identical outputs.
19
+
16
20
  ## One-time setup (PyPI account owner only)
17
21
 
18
22
  This must be done in the PyPI web UI by the account that will own the project —
@@ -42,8 +46,15 @@ workflow deliberately fails its anonymous-pull gate if that happens.
42
46
  1. Let the first `Publish release container` run push the package.
43
47
  2. If the anonymous-pull step fails, open the `shadowshield` package settings
44
48
  under the `0xsl1m` account and change visibility to **Public**.
45
- 3. Re-run the failed workflow job. Do not publish the release digest to operators
46
- until the anonymous pull and evidence-attachment steps are green.
49
+ 3. Re-run the failed workflow job. It will reuse an existing image only when the
50
+ version and exact-commit tags resolve to the same digest and its revision and
51
+ version labels match the release **and** that digest already has trusted source
52
+ provenance from this workflow for the exact commit. It never overwrites a
53
+ conflicting or unattested tag. If an interruption happens after an image push
54
+ but before provenance is attached, delete the incomplete version/commit tags
55
+ from GHCR and re-run; the workflow deliberately refuses to bless them.
56
+ Do not publish the digest to operators until the anonymous-pull and
57
+ evidence-attachment steps are green.
47
58
 
48
59
  ## Cutting a release
49
60
 
@@ -59,9 +70,13 @@ gh release create vX.Y.Z --target "$merge_sha" \
59
70
  # -> PyPI publishing and exact-image scan/publish run independently
60
71
  ```
61
72
 
62
- Do not create the release while required checks are pending. A partial release
63
- is recoverable by fixing the failed workflow and re-running it, but the PyPI
64
- version itself cannot be replaced.
73
+ Do not create the release while required checks are pending. Once trusted source
74
+ provenance exists, a partial container release is recoverable by fixing the
75
+ failed workflow and re-running it: exact matching registry content is pulled,
76
+ rescanned, re-attested, and reused. An interruption between the first image push
77
+ and provenance attachment intentionally needs the incomplete GHCR tags removed
78
+ before retry. Conflicting tags and existing evidence assets are never
79
+ overwritten. The PyPI version itself cannot be replaced.
65
80
 
66
81
  Both publication workflows independently reject non-stable tags, tag/source
67
82
  version mismatches, commits outside `main`, and commits without a successful
@@ -99,6 +114,8 @@ gh attestation verify "oci://$image" \
99
114
 
100
115
  - **Versions are immutable.** A published version can never be re-uploaded, only
101
116
  *yanked*. Always confirm `twine check` is green (CI does this) before releasing.
117
+ - Regenerate dependency locks only in a dedicated reviewed PR, using the exact
118
+ commands recorded at the top of each lock file; CI audits both locked graphs.
102
119
  - Deploy the container by the exact `image@sha256` value in
103
120
  `container-digest.txt`, never by a mutable version tag or a local Compose
104
121
  rebuild. The release SBOM describes that exact pre-push-scanned image.
@@ -19,11 +19,11 @@ Generate an annotated starter file with `shadowshield init > shield.yaml`.
19
19
 
20
20
  ## Modes
21
21
 
22
- | Mode | block_threshold | medium→ | LLM check | rate limit |
23
- |---|---|---|---|---|
24
- | `strict` | 0.45 | block | on | on (30/60s) |
25
- | `balanced` | 0.65 | sanitize | off | off |
26
- | `permissive` | 0.85 | flag | off | off |
22
+ | Mode | block_threshold | medium→ | detector failure | LLM check | rate limit |
23
+ |---|---|---|---|---|---|
24
+ | `strict` | 0.45 | block | block | on | on (30/60s) |
25
+ | `balanced` | 0.65 | sanitize | report | off | off |
26
+ | `permissive` | 0.85 | flag | report | off | off |
27
27
 
28
28
  A mode seeds every default; any field you set in YAML or `for_mode(..., **kw)`
29
29
  layers on top.
@@ -34,6 +34,7 @@ layers on top.
34
34
  |---|---|
35
35
  | `mode` | preset posture (strict/balanced/permissive) |
36
36
  | `raise_on_block` | make `Shield.scan()` raise on a block (default false) |
37
+ | `fail_closed_on_detector_error` | block if any detector raises; failures are always reported as content-free metadata (strict defaults true) |
37
38
  | `block_threshold` | aggregate score that forces a block regardless of policy |
38
39
  | `policy.{none..critical}` | severity → decision mapping |
39
40
  | `detectors.<name>.enabled` | per-detector toggle |
@@ -56,7 +56,9 @@ shield = Shield.for_mode(
56
56
 
57
57
  A plugin that fails to import is **skipped**, not fatal — one broken third-party
58
58
  plugin can never disable the shield. A plugin detector that *raises during a scan*
59
- is caught by the engine's fail-safe wrapper and drops only its own contribution.
59
+ is caught by the engine's fail-safe wrapper, reported through bounded content-free
60
+ error metadata and metrics, and forces a block when
61
+ `fail_closed_on_detector_error` is enabled.
60
62
 
61
63
  ## Security note
62
64
 
@@ -67,8 +67,10 @@ identity to block on its own.
67
67
 
68
68
  ## Fail-safe defaults
69
69
 
70
- - **Detector errors fail safe.** A detector that raises drops only its own
71
- contribution; the scan continues with the other layers.
70
+ - **Detector errors are visible.** The other independent layers continue, while
71
+ bounded content-free metadata and metrics identify the failed detector. Strict
72
+ mode forces a block; balanced/permissive report the failure unless configured
73
+ to fail closed.
72
74
  - **`guard()` fails closed** (raises), **`filter()` fails soft** (returns the safe
73
75
  fallback). You choose per call site.
74
76
  - **Secrets are never echoed.** Secret matches are redacted from `Threat` records
@@ -1,10 +1,10 @@
1
1
  [build-system]
2
- requires = ["hatchling>=1.21"]
2
+ requires = ["hatchling==1.31.0"]
3
3
  build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "shadowshield"
7
- version = "0.6.2"
7
+ version = "0.6.3"
8
8
  description = "Unified open-source security shield for agentic AI systems — inspired by Sentinel & ShadowClaw."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -43,7 +43,6 @@ dependencies = [
43
43
  "structlog>=24.1",
44
44
  "pyyaml>=6.0",
45
45
  "httpx>=0.27",
46
- "tiktoken>=0.6",
47
46
  ]
48
47
 
49
48
  [project.optional-dependencies]
@@ -142,8 +141,17 @@ python_version = "3.10"
142
141
  strict = true
143
142
  warn_unused_ignores = true
144
143
  ignore_missing_imports = true
144
+ # Allow targeted external-stub exclusions to respect ``follow_imports = skip``.
145
+ follow_imports_for_stubs = true
145
146
  files = ["src/shadowshield"]
146
147
 
148
+ [[tool.mypy.overrides]]
149
+ # NumPy 2.5 publishes PEP 695 aliases that cannot be parsed while mypy checks
150
+ # our Python 3.10 syntax target. The optional dependency boundary stays Any;
151
+ # every ShadowShield source module is still checked explicitly and strictly.
152
+ module = ["numpy", "numpy.*"]
153
+ follow_imports = "skip"
154
+
147
155
  [tool.coverage.run]
148
156
  source = ["shadowshield"]
149
157
  branch = true
@@ -71,7 +71,7 @@ from .plugins import PluginManager, ShadowShieldPlugin
71
71
  from .reporter import Reporter, attach_reporter
72
72
  from .responders import Responder, spotlight
73
73
 
74
- __version__ = "0.6.2"
74
+ __version__ = "0.6.3"
75
75
 
76
76
  __all__ = [
77
77
  "__version__",
@@ -28,9 +28,13 @@ import hmac
28
28
  import json
29
29
  import os
30
30
  from collections.abc import Sequence
31
+ from math import isfinite
32
+ from time import monotonic
31
33
  from typing import Any
32
34
 
33
35
  MAX_HTTP_BODY_BYTES = 1_048_576
36
+ MAX_HTTP_BODY_FRAMES = 8_192
37
+ HTTP_BODY_READ_TIMEOUT_SECONDS = 15.0
34
38
  MAX_CONCURRENT_SCANS = 16
35
39
 
36
40
 
@@ -138,16 +142,39 @@ def is_loopback(host: str) -> bool:
138
142
 
139
143
 
140
144
  class RequestBodyLimitMiddleware:
141
- """Buffer and reject oversized HTTP bodies before framework JSON parsing.
145
+ """Coalesce and reject unsafe HTTP bodies before framework JSON parsing.
142
146
 
143
- These endpoints accept bounded JSON rather than streaming uploads. Buffering
144
- at most ``max_bytes`` gives deterministic 413 behavior even for chunked
145
- requests without a ``Content-Length`` header.
147
+ These endpoints accept bounded JSON rather than streaming uploads. The
148
+ aggregate buffer never exceeds ``max_bytes``, so many tiny ASGI frames cannot
149
+ amplify a legal body into unbounded Python object overhead. A frame ceiling
150
+ bounds receive-loop work, and one total deadline covers the complete body
151
+ rather than resetting for every chunk.
146
152
  """
147
153
 
148
- def __init__(self, app: Any, max_bytes: int = MAX_HTTP_BODY_BYTES) -> None:
154
+ def __init__(
155
+ self,
156
+ app: Any,
157
+ max_bytes: int = MAX_HTTP_BODY_BYTES,
158
+ *,
159
+ max_frames: int = MAX_HTTP_BODY_FRAMES,
160
+ read_timeout_seconds: float = HTTP_BODY_READ_TIMEOUT_SECONDS,
161
+ ) -> None:
162
+ if isinstance(max_frames, bool) or not isinstance(max_frames, int) or max_frames <= 0:
163
+ raise ValueError("max_frames must be a positive integer")
164
+ if isinstance(read_timeout_seconds, bool) or not isinstance(
165
+ read_timeout_seconds, (int, float)
166
+ ):
167
+ raise ValueError("read_timeout_seconds must be a finite positive number")
168
+ try:
169
+ normalized_timeout = float(read_timeout_seconds)
170
+ except OverflowError:
171
+ raise ValueError("read_timeout_seconds must be a finite positive number") from None
172
+ if not isfinite(normalized_timeout) or normalized_timeout <= 0:
173
+ raise ValueError("read_timeout_seconds must be a finite positive number")
149
174
  self.app = app
150
175
  self.max_bytes = max_bytes
176
+ self.max_frames = max_frames
177
+ self.read_timeout_seconds = normalized_timeout
151
178
 
152
179
  async def __call__(self, scope: dict[str, Any], receive: Any, send: Any) -> None:
153
180
  if scope.get("type") != "http":
@@ -165,27 +192,53 @@ class RequestBodyLimitMiddleware:
165
192
  await self._reject(send)
166
193
  return
167
194
 
168
- buffered: list[dict[str, Any]] = []
169
- received = 0
195
+ body = bytearray()
196
+ frame_count = 0
197
+ body_complete = False
198
+ terminal_message: dict[str, Any] | None = None
199
+ deadline = monotonic() + self.read_timeout_seconds
170
200
  while True:
171
- message = dict(await receive())
172
- buffered.append(message)
201
+ remaining = deadline - monotonic()
202
+ if remaining <= 0:
203
+ await self._reject_timeout(send)
204
+ return
205
+ try:
206
+ message = dict(await asyncio.wait_for(receive(), timeout=remaining))
207
+ except asyncio.TimeoutError:
208
+ await self._reject_timeout(send)
209
+ return
173
210
  if message.get("type") != "http.request":
211
+ terminal_message = message
174
212
  break
175
- received += len(message.get("body", b""))
176
- if received > self.max_bytes:
213
+
214
+ frame_count += 1
215
+ if frame_count > self.max_frames:
216
+ await self._reject_fragmented(send)
217
+ return
218
+ chunk = message.get("body", b"")
219
+ if len(body) + len(chunk) > self.max_bytes:
177
220
  await self._reject(send)
178
221
  return
222
+ body.extend(chunk)
179
223
  if not message.get("more_body", False):
224
+ body_complete = True
180
225
  break
181
226
 
182
- index = 0
227
+ aggregate_pending = bool(body) or body_complete
228
+ aggregate = bytes(body)
183
229
 
184
230
  async def replay_receive() -> dict[str, Any]:
185
- nonlocal index
186
- if index < len(buffered):
187
- message = buffered[index]
188
- index += 1
231
+ nonlocal aggregate_pending, terminal_message
232
+ if aggregate_pending:
233
+ aggregate_pending = False
234
+ return {
235
+ "type": "http.request",
236
+ "body": aggregate,
237
+ "more_body": not body_complete,
238
+ }
239
+ if terminal_message is not None:
240
+ message = terminal_message
241
+ terminal_message = None
189
242
  return message
190
243
  return {"type": "http.request", "body": b"", "more_body": False}
191
244
 
@@ -206,6 +259,36 @@ class RequestBodyLimitMiddleware:
206
259
  )
207
260
  await send({"type": "http.response.body", "body": body})
208
261
 
262
+ @staticmethod
263
+ async def _reject_fragmented(send: Any) -> None:
264
+ body = b'{"detail":"request body too fragmented"}'
265
+ await send(
266
+ {
267
+ "type": "http.response.start",
268
+ "status": 413,
269
+ "headers": [
270
+ (b"content-type", b"application/json"),
271
+ (b"content-length", str(len(body)).encode("ascii")),
272
+ ],
273
+ }
274
+ )
275
+ await send({"type": "http.response.body", "body": body})
276
+
277
+ @staticmethod
278
+ async def _reject_timeout(send: Any) -> None:
279
+ body = b'{"detail":"request body read timed out"}'
280
+ await send(
281
+ {
282
+ "type": "http.response.start",
283
+ "status": 408,
284
+ "headers": [
285
+ (b"content-type", b"application/json"),
286
+ (b"content-length", str(len(body)).encode("ascii")),
287
+ ],
288
+ }
289
+ )
290
+ await send({"type": "http.response.body", "body": body})
291
+
209
292
 
210
293
  class ConcurrencyLimitMiddleware:
211
294
  """Bound expensive scan/benchmark work while leaving health responsive."""
@@ -257,7 +340,13 @@ class ConcurrencyLimitMiddleware:
257
340
 
258
341
 
259
342
  class EarlyAuthMiddleware:
260
- """Authenticate protected routes before reading or parsing their bodies."""
343
+ """Authenticate protected routes before reading or parsing their bodies.
344
+
345
+ Configured CORS middleware sits outside this boundary and handles valid
346
+ browser preflights. Arbitrary ``OPTIONS`` requests that reach this layer are
347
+ authenticated like every other protected request, so they cannot consume
348
+ body-read or scan-admission capacity anonymously.
349
+ """
261
350
 
262
351
  def __init__(
263
352
  self,
@@ -279,10 +368,8 @@ class EarlyAuthMiddleware:
279
368
  protected = path in self.protected_paths or any(
280
369
  path.startswith(prefix) for prefix in self.protected_prefixes
281
370
  )
282
- method = str(scope.get("method", "GET")).upper()
283
371
  if (
284
372
  scope.get("type") == "http"
285
- and method != "OPTIONS"
286
373
  and protected
287
374
  and self.deny_when_unconfigured
288
375
  and not self.api_keys
@@ -300,7 +387,7 @@ class EarlyAuthMiddleware:
300
387
  )
301
388
  await send({"type": "http.response.body", "body": body})
302
389
  return
303
- if scope.get("type") == "http" and method != "OPTIONS" and protected and self.api_keys:
390
+ if scope.get("type") == "http" and protected and self.api_keys:
304
391
  headers = {
305
392
  key.decode("latin-1").lower(): value.decode("latin-1")
306
393
  for key, value in scope.get("headers", [])
@@ -111,11 +111,16 @@ def _cmd_benchmark(args: argparse.Namespace) -> int:
111
111
  else:
112
112
  examples = load_builtin()
113
113
 
114
- shield = Shield(
115
- ShieldConfig.for_mode(Mode(args.mode)),
116
- use_transformer=args.transformer or False,
117
- )
118
- report = evaluate_shield(shield, examples)
114
+ benchmark_config = ShieldConfig.for_mode(Mode(args.mode))
115
+ # Audit output is unrelated to detector quality, distorts latency, and can
116
+ # corrupt the benchmark command's JSON stdout. Benchmark reports still
117
+ # retain bounded runtime-integrity counters.
118
+ benchmark_config.logging.enabled = False
119
+ shield = Shield(benchmark_config, use_transformer=args.transformer or False)
120
+ # Model/index initialization is deliberately outside timed scans. Any
121
+ # warmup, readiness, or per-scan detector failure makes the benchmark
122
+ # explicitly unreliable and produces a non-zero exit status.
123
+ report = evaluate_shield(shield, examples, warmup=True)
119
124
 
120
125
  if args.json:
121
126
  print(json.dumps(report.to_dict(), indent=2))
@@ -128,7 +133,7 @@ def _cmd_benchmark(args: argparse.Namespace) -> int:
128
133
  )
129
134
  print(f"dataset: {src} mode: {args.mode}")
130
135
  print(report.format_text())
131
- return 0
136
+ return 0 if report.reliable else 1
132
137
 
133
138
 
134
139
  def _cmd_serve(args: argparse.Namespace) -> int:
@@ -14,6 +14,10 @@ mode: balanced
14
14
  # is non-throwing; use Shield.guard() for fail-closed ergonomics instead).
15
15
  raise_on_block: false
16
16
 
17
+ # Surface detector failures as content-free result metadata. Set true to force
18
+ # BLOCK after the other independent detectors finish (the strict preset does).
19
+ fail_closed_on_detector_error: false
20
+
17
21
  # Aggregate score (0..1) at/above which a payload is forced to BLOCK regardless
18
22
  # of the per-severity policy below. Lower = more cautious.
19
23
  block_threshold: 0.65