confidence-escalation 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- confidence_escalation-0.1.0/.github/workflows/ci.yml +72 -0
- confidence_escalation-0.1.0/.gitignore +78 -0
- confidence_escalation-0.1.0/LICENSE +21 -0
- confidence_escalation-0.1.0/PKG-INFO +290 -0
- confidence_escalation-0.1.0/README.md +248 -0
- confidence_escalation-0.1.0/pyproject.toml +70 -0
- confidence_escalation-0.1.0/src/confidence_escalation/__init__.py +69 -0
- confidence_escalation-0.1.0/src/confidence_escalation/adapters/__init__.py +1 -0
- confidence_escalation-0.1.0/src/confidence_escalation/adapters/autogen.py +98 -0
- confidence_escalation-0.1.0/src/confidence_escalation/adapters/crewai.py +89 -0
- confidence_escalation-0.1.0/src/confidence_escalation/adapters/google_adk.py +153 -0
- confidence_escalation-0.1.0/src/confidence_escalation/adapters/langchain.py +107 -0
- confidence_escalation-0.1.0/src/confidence_escalation/handlers.py +308 -0
- confidence_escalation-0.1.0/src/confidence_escalation/middleware.py +176 -0
- confidence_escalation-0.1.0/src/confidence_escalation/policy.py +167 -0
- confidence_escalation-0.1.0/src/confidence_escalation/scorer.py +169 -0
- confidence_escalation-0.1.0/tests/__init__.py +0 -0
- confidence_escalation-0.1.0/tests/test_handlers.py +123 -0
- confidence_escalation-0.1.0/tests/test_middleware.py +68 -0
- confidence_escalation-0.1.0/tests/test_policy.py +112 -0
- confidence_escalation-0.1.0/tests/test_scorer.py +103 -0
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
tags: ["v*"]
|
|
7
|
+
pull_request:
|
|
8
|
+
branches: [main]
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
test:
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
strategy:
|
|
14
|
+
matrix:
|
|
15
|
+
python-version: ["3.9", "3.10", "3.11", "3.12"]
|
|
16
|
+
|
|
17
|
+
steps:
|
|
18
|
+
- uses: actions/checkout@v4
|
|
19
|
+
|
|
20
|
+
- name: Set up Python ${{ matrix.python-version }}
|
|
21
|
+
uses: actions/setup-python@v5
|
|
22
|
+
with:
|
|
23
|
+
python-version: ${{ matrix.python-version }}
|
|
24
|
+
|
|
25
|
+
- name: Install dependencies
|
|
26
|
+
run: |
|
|
27
|
+
python -m pip install --upgrade pip
|
|
28
|
+
pip install ".[dev]"
|
|
29
|
+
|
|
30
|
+
- name: Run tests
|
|
31
|
+
run: |
|
|
32
|
+
python -m pytest tests/ -v --tb=short
|
|
33
|
+
|
|
34
|
+
- name: Run tests with coverage
|
|
35
|
+
if: matrix.python-version == '3.11'
|
|
36
|
+
run: |
|
|
37
|
+
pip install pytest-cov
|
|
38
|
+
python -m pytest tests/ --cov=src/confidence_escalation --cov-report=xml
|
|
39
|
+
|
|
40
|
+
- name: Upload coverage to Codecov
|
|
41
|
+
if: matrix.python-version == '3.11'
|
|
42
|
+
uses: codecov/codecov-action@v4
|
|
43
|
+
with:
|
|
44
|
+
token: ${{ secrets.CODECOV_TOKEN }}
|
|
45
|
+
fail_ci_if_error: false
|
|
46
|
+
|
|
47
|
+
publish:
|
|
48
|
+
needs: test
|
|
49
|
+
runs-on: ubuntu-latest
|
|
50
|
+
if: startsWith(github.ref, 'refs/tags/v')
|
|
51
|
+
|
|
52
|
+
steps:
|
|
53
|
+
- uses: actions/checkout@v4
|
|
54
|
+
|
|
55
|
+
- name: Set up Python
|
|
56
|
+
uses: actions/setup-python@v5
|
|
57
|
+
with:
|
|
58
|
+
python-version: "3.11"
|
|
59
|
+
|
|
60
|
+
- name: Install build tools
|
|
61
|
+
run: pip install build hatchling
|
|
62
|
+
|
|
63
|
+
- name: Build package
|
|
64
|
+
run: python -m build
|
|
65
|
+
|
|
66
|
+
- name: Publish to PyPI
|
|
67
|
+
env:
|
|
68
|
+
TWINE_USERNAME: __token__
|
|
69
|
+
TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
|
|
70
|
+
run: |
|
|
71
|
+
pip install twine
|
|
72
|
+
twine upload dist/*
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
__pycache__/
|
|
2
|
+
*.py[cod]
|
|
3
|
+
*$py.class
|
|
4
|
+
*.so
|
|
5
|
+
.Python
|
|
6
|
+
build/
|
|
7
|
+
develop-eggs/
|
|
8
|
+
dist/
|
|
9
|
+
downloads/
|
|
10
|
+
eggs/
|
|
11
|
+
.eggs/
|
|
12
|
+
lib/
|
|
13
|
+
lib64/
|
|
14
|
+
parts/
|
|
15
|
+
sdist/
|
|
16
|
+
var/
|
|
17
|
+
wheels/
|
|
18
|
+
share/python-wheels/
|
|
19
|
+
*.egg-info/
|
|
20
|
+
.installed.cfg
|
|
21
|
+
*.egg
|
|
22
|
+
MANIFEST
|
|
23
|
+
*.manifest
|
|
24
|
+
*.spec
|
|
25
|
+
pip-log.txt
|
|
26
|
+
pip-delete-this-directory.txt
|
|
27
|
+
htmlcov/
|
|
28
|
+
.tox/
|
|
29
|
+
.nox/
|
|
30
|
+
.coverage
|
|
31
|
+
.coverage.*
|
|
32
|
+
.cache
|
|
33
|
+
nosetests.xml
|
|
34
|
+
coverage.xml
|
|
35
|
+
*.cover
|
|
36
|
+
*.py,cover
|
|
37
|
+
.hypothesis/
|
|
38
|
+
.pytest_cache/
|
|
39
|
+
cover/
|
|
40
|
+
*.mo
|
|
41
|
+
*.pot
|
|
42
|
+
*.log
|
|
43
|
+
local_settings.py
|
|
44
|
+
db.sqlite3
|
|
45
|
+
db.sqlite3-journal
|
|
46
|
+
instance/
|
|
47
|
+
.webassets-cache
|
|
48
|
+
.scrapy
|
|
49
|
+
docs/_build/
|
|
50
|
+
.pybuilder/
|
|
51
|
+
target/
|
|
52
|
+
.ipynb_checkpoints
|
|
53
|
+
profile_default/
|
|
54
|
+
ipython_config.py
|
|
55
|
+
.pdm.toml
|
|
56
|
+
.pdm-python
|
|
57
|
+
.pdm-build/
|
|
58
|
+
__pypackages__/
|
|
59
|
+
celerybeat-schedule
|
|
60
|
+
celerybeat.pid
|
|
61
|
+
*.sage.py
|
|
62
|
+
.env
|
|
63
|
+
.venv
|
|
64
|
+
env/
|
|
65
|
+
venv/
|
|
66
|
+
ENV/
|
|
67
|
+
env.bak/
|
|
68
|
+
venv.bak/
|
|
69
|
+
.spyderproject.db
|
|
70
|
+
.spyproject
|
|
71
|
+
.ropeproject
|
|
72
|
+
/site
|
|
73
|
+
.mypy_cache/
|
|
74
|
+
.dmypy.json
|
|
75
|
+
dmypy.json
|
|
76
|
+
.pyre/
|
|
77
|
+
.pytype/
|
|
78
|
+
cython_debug/
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ashutosh Rana
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: confidence-escalation
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Framework-agnostic confidence-gated escalation middleware for LLM agents: multi-signal scoring (logprob, verbalized, tool risk), threshold policies, and escalation handlers for LangChain, CrewAI, AutoGen, and Google ADK.
|
|
5
|
+
Project-URL: Homepage, https://github.com/ashutoshrana/confidence-escalation
|
|
6
|
+
Project-URL: Documentation, https://github.com/ashutoshrana/confidence-escalation#readme
|
|
7
|
+
Project-URL: Repository, https://github.com/ashutoshrana/confidence-escalation
|
|
8
|
+
Project-URL: Issues, https://github.com/ashutoshrana/confidence-escalation/issues
|
|
9
|
+
Author-email: Ashutosh Rana <17191144+ashutoshrana@users.noreply.github.com>
|
|
10
|
+
License: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: agent-middleware,ai-governance,ai-safety,autogen,confidence-scoring,crewai,enterprise-ai,escalation,eu-ai-act,google-adk,human-in-loop,langchain,llm-agents,logprob,multi-signal,owasp-agentic-ai,regulated-industries,threshold-policy
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
22
|
+
Classifier: Topic :: Security
|
|
23
|
+
Requires-Python: >=3.9
|
|
24
|
+
Provides-Extra: adk
|
|
25
|
+
Requires-Dist: google-adk>=0.3.0; extra == 'adk'
|
|
26
|
+
Provides-Extra: all
|
|
27
|
+
Requires-Dist: crewai>=0.1.0; extra == 'all'
|
|
28
|
+
Requires-Dist: google-adk>=0.3.0; extra == 'all'
|
|
29
|
+
Requires-Dist: langchain-core>=0.1.0; extra == 'all'
|
|
30
|
+
Requires-Dist: pyautogen>=0.2.0; extra == 'all'
|
|
31
|
+
Provides-Extra: autogen
|
|
32
|
+
Requires-Dist: pyautogen>=0.2.0; extra == 'autogen'
|
|
33
|
+
Provides-Extra: crewai
|
|
34
|
+
Requires-Dist: crewai>=0.1.0; extra == 'crewai'
|
|
35
|
+
Provides-Extra: dev
|
|
36
|
+
Requires-Dist: pytest-asyncio>=0.21; extra == 'dev'
|
|
37
|
+
Requires-Dist: pytest-cov>=4.0; extra == 'dev'
|
|
38
|
+
Requires-Dist: pytest>=7.0; extra == 'dev'
|
|
39
|
+
Provides-Extra: langchain
|
|
40
|
+
Requires-Dist: langchain-core>=0.1.0; extra == 'langchain'
|
|
41
|
+
Description-Content-Type: text/markdown
|
|
42
|
+
|
|
43
|
+
# confidence-escalation
|
|
44
|
+
|
|
45
|
+
**Framework-agnostic confidence-gated escalation middleware for LLM agents.**
|
|
46
|
+
|
|
47
|
+
[](https://badge.fury.io/py/confidence-escalation)
|
|
48
|
+
[](https://www.python.org/downloads/)
|
|
49
|
+
[](https://opensource.org/licenses/MIT)
|
|
50
|
+
|
|
51
|
+
Multi-signal confidence scoring (logprob + verbalized + ASR + tool risk) with threshold-based escalation policies and pluggable handlers. Works with **LangChain**, **LangGraph**, **CrewAI**, **AutoGen**, **Google ADK**, and any Python agent framework.
|
|
52
|
+
|
|
53
|
+
Addresses **OWASP Agentic AI Top 10 ASI-09**: Human-Agent Trust Exploitation — prevents agents from taking high-stakes actions when confidence is insufficient.
|
|
54
|
+
|
|
55
|
+
---
|
|
56
|
+
|
|
57
|
+
## The Problem
|
|
58
|
+
|
|
59
|
+
LLM agents fail silently. When an agent is uncertain, it still returns a response — often confidently-worded — with no mechanism to:
|
|
60
|
+
- Detect that confidence is low before executing a high-risk tool call
|
|
61
|
+
- Route uncertain responses to a human reviewer
|
|
62
|
+
- Escalate to a stronger model when needed
|
|
63
|
+
- Produce a compliance audit trail of every escalation event
|
|
64
|
+
|
|
65
|
+
`confidence-escalation` solves all four.
|
|
66
|
+
|
|
67
|
+
---
|
|
68
|
+
|
|
69
|
+
## Features
|
|
70
|
+
|
|
71
|
+
- **Multi-signal scoring** — combine logprobs, verbalized confidence, and tool-call risk into a single composite score
|
|
72
|
+
- **Threshold policies** — single-threshold, dual-threshold (normal + critical), composite multi-policy chains
|
|
73
|
+
- **Pluggable handlers** — human-in-loop, model upgrade, tool restriction, compliance logging
|
|
74
|
+
- **Framework adapters** — LangChain callbacks, CrewAI step_callback, AutoGen reply function wrapper, Google ADK event interceptor
|
|
75
|
+
- **EU AI Act Article 12 audit logging** — structured JSON compliance log on every escalation
|
|
76
|
+
- **Zero required dependencies** — core library runs with no dependencies; framework integrations are optional extras
|
|
77
|
+
|
|
78
|
+
---
|
|
79
|
+
|
|
80
|
+
## Quick Start
|
|
81
|
+
|
|
82
|
+
### Installation
|
|
83
|
+
|
|
84
|
+
```bash
|
|
85
|
+
pip install confidence-escalation
|
|
86
|
+
# With LangChain:
|
|
87
|
+
pip install "confidence-escalation[langchain]"
|
|
88
|
+
# With all frameworks:
|
|
89
|
+
pip install "confidence-escalation[all]"
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
### Basic Scoring
|
|
93
|
+
|
|
94
|
+
```python
|
|
95
|
+
from confidence_escalation import MultiSignalConfidenceScorer
|
|
96
|
+
|
|
97
|
+
scorer = MultiSignalConfidenceScorer(
|
|
98
|
+
weights={"logprob": 0.5, "verbalized": 0.3, "tool_risk": -0.2}
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
score = scorer.score(
|
|
102
|
+
logprobs=[-0.1, -0.3, -0.2],
|
|
103
|
+
verbalized_response="I am 70% confident about this answer.",
|
|
104
|
+
tool_call_risk=0.15,
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
print(f"Confidence: {score.value:.3f}") # e.g. 0.712
|
|
108
|
+
print(f"Reliable: {score.is_reliable()}") # True (above 0.6 default)
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
### Threshold Policy + Human-in-Loop
|
|
112
|
+
|
|
113
|
+
```python
|
|
114
|
+
from confidence_escalation import (
|
|
115
|
+
ThresholdPolicy,
|
|
116
|
+
EscalationAction,
|
|
117
|
+
HumanInLoopHandler,
|
|
118
|
+
ComplianceLoggingHandler,
|
|
119
|
+
ConfidenceEscalationMiddleware,
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
def notify_human(ctx, result):
|
|
123
|
+
print(f"Routing to human review: session={ctx['session_id']}, confidence={result.confidence_score:.3f}")
|
|
124
|
+
|
|
125
|
+
policy = ThresholdPolicy(
|
|
126
|
+
threshold=0.65,
|
|
127
|
+
action=EscalationAction.HUMAN_IN_LOOP,
|
|
128
|
+
critical_threshold=0.3,
|
|
129
|
+
critical_action=EscalationAction.ABORT,
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
middleware = ConfidenceEscalationMiddleware(
|
|
133
|
+
policy=policy,
|
|
134
|
+
handlers=[
|
|
135
|
+
HumanInLoopHandler(callback=notify_human),
|
|
136
|
+
ComplianceLoggingHandler(),
|
|
137
|
+
],
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
result = middleware.call(
|
|
141
|
+
agent_step=lambda: my_llm.invoke(messages),
|
|
142
|
+
context={"session_id": "abc123", "model": "claude-sonnet-4-6"},
|
|
143
|
+
logprobs=[-0.4, -0.5],
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
if result["escalation"]["triggered"]:
|
|
147
|
+
print("Escalated — stopping agent execution.")
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
### Model Upgrade Handler
|
|
151
|
+
|
|
152
|
+
```python
|
|
153
|
+
from confidence_escalation import ModelUpgradeHandler, ThresholdPolicy, EscalationAction
|
|
154
|
+
|
|
155
|
+
handler = ModelUpgradeHandler(
|
|
156
|
+
upgrade_map={
|
|
157
|
+
"claude-haiku-4-5": "claude-sonnet-4-6",
|
|
158
|
+
"claude-sonnet-4-6": "claude-opus-4-7",
|
|
159
|
+
}
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
policy = ThresholdPolicy(threshold=0.7, action=EscalationAction.MODEL_UPGRADE)
|
|
163
|
+
result = policy.evaluate(score, context={"model": "claude-haiku-4-5"})
|
|
164
|
+
|
|
165
|
+
if result.triggered:
|
|
166
|
+
upgrade_info = handler.handle(result, context={"model": "claude-haiku-4-5"})
|
|
167
|
+
print(f"Retry with: {upgrade_info['upgraded_model']}")
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
### Tool Restriction
|
|
171
|
+
|
|
172
|
+
```python
|
|
173
|
+
from confidence_escalation import ToolRestrictionHandler, ThresholdPolicy, EscalationAction
|
|
174
|
+
|
|
175
|
+
handler = ToolRestrictionHandler(
|
|
176
|
+
high_risk_tools=["delete_record", "send_email", "execute_sql"],
|
|
177
|
+
allow_read_only=True,
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
policy = ThresholdPolicy(threshold=0.65, action=EscalationAction.TOOL_RESTRICTION)
|
|
181
|
+
result = policy.evaluate(score, context={"available_tools": ["get_customer", "delete_record"]})
|
|
182
|
+
|
|
183
|
+
if result.triggered:
|
|
184
|
+
restriction = handler.handle(result, context={"available_tools": agent_tools})
|
|
185
|
+
safe_tools = restriction["allowed_tools"]
|
|
186
|
+
# Re-invoke agent with only safe_tools
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
### LangChain Integration
|
|
190
|
+
|
|
191
|
+
```python
|
|
192
|
+
from confidence_escalation.adapters.langchain import LangChainEscalationAdapter
|
|
193
|
+
from confidence_escalation.handlers import HumanInLoopHandler
|
|
194
|
+
|
|
195
|
+
adapter = LangChainEscalationAdapter(
|
|
196
|
+
threshold=0.65,
|
|
197
|
+
handlers=[HumanInLoopHandler(raise_on_trigger=True)],
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
# Attach as LangChain callback
|
|
201
|
+
chain = LLMChain(llm=llm, callbacks=[adapter.as_callback()])
|
|
202
|
+
|
|
203
|
+
# Or call directly from a LangGraph node
|
|
204
|
+
def research_node(state):
|
|
205
|
+
response = llm.invoke(state["messages"])
|
|
206
|
+
try:
|
|
207
|
+
adapter.on_llm_end(response.content, logprobs=response.response_metadata.get("logprobs"))
|
|
208
|
+
except HumanInLoopHandler.HumanReviewRequired:
|
|
209
|
+
return {"status": "escalated"}
|
|
210
|
+
return {"response": response.content}
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
### CrewAI Integration
|
|
214
|
+
|
|
215
|
+
```python
|
|
216
|
+
from crewai import Agent
|
|
217
|
+
from confidence_escalation.adapters.crewai import CrewAIEscalationAdapter
|
|
218
|
+
|
|
219
|
+
adapter = CrewAIEscalationAdapter(threshold=0.65)
|
|
220
|
+
|
|
221
|
+
agent = Agent(
|
|
222
|
+
role="Research Specialist",
|
|
223
|
+
goal="Analyze market trends",
|
|
224
|
+
backstory="...",
|
|
225
|
+
step_callback=adapter.step_callback,
|
|
226
|
+
)
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
### Google ADK Integration
|
|
230
|
+
|
|
231
|
+
```python
|
|
232
|
+
from google.adk.agents import BaseAgent
|
|
233
|
+
from confidence_escalation.adapters.google_adk import ADKEscalationAdapter
|
|
234
|
+
|
|
235
|
+
class GovernedAgent(BaseAgent):
|
|
236
|
+
def __init__(self, *args, **kwargs):
|
|
237
|
+
super().__init__(*args, **kwargs)
|
|
238
|
+
self._escalation = ADKEscalationAdapter(threshold=0.65)
|
|
239
|
+
|
|
240
|
+
async def _run_async_impl(self, ctx):
|
|
241
|
+
async for event in self._llm_agent._run_async_impl(ctx):
|
|
242
|
+
if event.is_final_response():
|
|
243
|
+
result = self._escalation.evaluate_event(event, ctx)
|
|
244
|
+
if result["triggered"]:
|
|
245
|
+
yield self._escalation.build_escalation_event(result)
|
|
246
|
+
return
|
|
247
|
+
yield event
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
---
|
|
251
|
+
|
|
252
|
+
## Composite Policy Chains
|
|
253
|
+
|
|
254
|
+
```python
|
|
255
|
+
from confidence_escalation import ThresholdPolicy, EscalationAction
|
|
256
|
+
from confidence_escalation.policy import CompositePolicy
|
|
257
|
+
|
|
258
|
+
policy = CompositePolicy(policies=[
|
|
259
|
+
ThresholdPolicy(threshold=0.25, action=EscalationAction.ABORT),
|
|
260
|
+
ThresholdPolicy(threshold=0.55, action=EscalationAction.HUMAN_IN_LOOP),
|
|
261
|
+
ThresholdPolicy(threshold=0.75, action=EscalationAction.COMPLIANCE_LOG),
|
|
262
|
+
])
|
|
263
|
+
|
|
264
|
+
result = policy.evaluate(score, context={"session_id": "abc"})
|
|
265
|
+
# First matching threshold wins
|
|
266
|
+
```
|
|
267
|
+
|
|
268
|
+
---
|
|
269
|
+
|
|
270
|
+
## OWASP Agentic AI Coverage
|
|
271
|
+
|
|
272
|
+
| OWASP ASI ID | Risk | Coverage |
|
|
273
|
+
|-------------|------|----------|
|
|
274
|
+
| ASI-09 | Human-Agent Trust Exploitation | Confidence gating before high-stakes actions |
|
|
275
|
+
| ASI-02 | Tool Misuse | Tool restriction handler removes high-risk tools at low confidence |
|
|
276
|
+
| ASI-03 | Identity/Privilege Abuse | ComplianceLoggingHandler creates immutable audit trail |
|
|
277
|
+
|
|
278
|
+
---
|
|
279
|
+
|
|
280
|
+
## Related Packages
|
|
281
|
+
|
|
282
|
+
- [voice-ai-governance](https://github.com/ashutoshrana/voice-ai-governance) — HIPAA/FERPA/EU AI Act compliance for voice AI pipelines
|
|
283
|
+
- [regulated-ai-governance](https://github.com/ashutoshrana/regulated-ai-governance) — Runtime tool authorization and capability scoping
|
|
284
|
+
- [enterprise-rag-patterns](https://github.com/ashutoshrana/enterprise-rag-patterns) — FERPA/HIPAA/GDPR-compliant RAG patterns
|
|
285
|
+
|
|
286
|
+
---
|
|
287
|
+
|
|
288
|
+
## License
|
|
289
|
+
|
|
290
|
+
MIT License. See [LICENSE](LICENSE).
|
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
# confidence-escalation
|
|
2
|
+
|
|
3
|
+
**Framework-agnostic confidence-gated escalation middleware for LLM agents.**
|
|
4
|
+
|
|
5
|
+
[](https://badge.fury.io/py/confidence-escalation)
|
|
6
|
+
[](https://www.python.org/downloads/)
|
|
7
|
+
[](https://opensource.org/licenses/MIT)
|
|
8
|
+
|
|
9
|
+
Multi-signal confidence scoring (logprob + verbalized + ASR + tool risk) with threshold-based escalation policies and pluggable handlers. Works with **LangChain**, **LangGraph**, **CrewAI**, **AutoGen**, **Google ADK**, and any Python agent framework.
|
|
10
|
+
|
|
11
|
+
Addresses **OWASP Agentic AI Top 10 ASI-09**: Human-Agent Trust Exploitation — prevents agents from taking high-stakes actions when confidence is insufficient.
|
|
12
|
+
|
|
13
|
+
---
|
|
14
|
+
|
|
15
|
+
## The Problem
|
|
16
|
+
|
|
17
|
+
LLM agents fail silently. When an agent is uncertain, it still returns a response — often confidently-worded — with no mechanism to:
|
|
18
|
+
- Detect that confidence is low before executing a high-risk tool call
|
|
19
|
+
- Route uncertain responses to a human reviewer
|
|
20
|
+
- Escalate to a stronger model when needed
|
|
21
|
+
- Produce a compliance audit trail of every escalation event
|
|
22
|
+
|
|
23
|
+
`confidence-escalation` solves all four.
|
|
24
|
+
|
|
25
|
+
---
|
|
26
|
+
|
|
27
|
+
## Features
|
|
28
|
+
|
|
29
|
+
- **Multi-signal scoring** — combine logprobs, verbalized confidence, and tool-call risk into a single composite score
|
|
30
|
+
- **Threshold policies** — single-threshold, dual-threshold (normal + critical), composite multi-policy chains
|
|
31
|
+
- **Pluggable handlers** — human-in-loop, model upgrade, tool restriction, compliance logging
|
|
32
|
+
- **Framework adapters** — LangChain callbacks, CrewAI step_callback, AutoGen reply function wrapper, Google ADK event interceptor
|
|
33
|
+
- **EU AI Act Article 12 audit logging** — structured JSON compliance log on every escalation
|
|
34
|
+
- **Zero required dependencies** — core library runs with no dependencies; framework integrations are optional extras
|
|
35
|
+
|
|
36
|
+
---
|
|
37
|
+
|
|
38
|
+
## Quick Start
|
|
39
|
+
|
|
40
|
+
### Installation
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
pip install confidence-escalation
|
|
44
|
+
# With LangChain:
|
|
45
|
+
pip install "confidence-escalation[langchain]"
|
|
46
|
+
# With all frameworks:
|
|
47
|
+
pip install "confidence-escalation[all]"
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
### Basic Scoring
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
from confidence_escalation import MultiSignalConfidenceScorer
|
|
54
|
+
|
|
55
|
+
scorer = MultiSignalConfidenceScorer(
|
|
56
|
+
weights={"logprob": 0.5, "verbalized": 0.3, "tool_risk": -0.2}
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
score = scorer.score(
|
|
60
|
+
logprobs=[-0.1, -0.3, -0.2],
|
|
61
|
+
verbalized_response="I am 70% confident about this answer.",
|
|
62
|
+
tool_call_risk=0.15,
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
print(f"Confidence: {score.value:.3f}") # e.g. 0.712
|
|
66
|
+
print(f"Reliable: {score.is_reliable()}") # True (above 0.6 default)
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
### Threshold Policy + Human-in-Loop
|
|
70
|
+
|
|
71
|
+
```python
|
|
72
|
+
from confidence_escalation import (
|
|
73
|
+
ThresholdPolicy,
|
|
74
|
+
EscalationAction,
|
|
75
|
+
HumanInLoopHandler,
|
|
76
|
+
ComplianceLoggingHandler,
|
|
77
|
+
ConfidenceEscalationMiddleware,
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
def notify_human(ctx, result):
|
|
81
|
+
print(f"Routing to human review: session={ctx['session_id']}, confidence={result.confidence_score:.3f}")
|
|
82
|
+
|
|
83
|
+
policy = ThresholdPolicy(
|
|
84
|
+
threshold=0.65,
|
|
85
|
+
action=EscalationAction.HUMAN_IN_LOOP,
|
|
86
|
+
critical_threshold=0.3,
|
|
87
|
+
critical_action=EscalationAction.ABORT,
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
middleware = ConfidenceEscalationMiddleware(
|
|
91
|
+
policy=policy,
|
|
92
|
+
handlers=[
|
|
93
|
+
HumanInLoopHandler(callback=notify_human),
|
|
94
|
+
ComplianceLoggingHandler(),
|
|
95
|
+
],
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
result = middleware.call(
|
|
99
|
+
agent_step=lambda: my_llm.invoke(messages),
|
|
100
|
+
context={"session_id": "abc123", "model": "claude-sonnet-4-6"},
|
|
101
|
+
logprobs=[-0.4, -0.5],
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
if result["escalation"]["triggered"]:
|
|
105
|
+
print("Escalated — stopping agent execution.")
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
### Model Upgrade Handler
|
|
109
|
+
|
|
110
|
+
```python
|
|
111
|
+
from confidence_escalation import ModelUpgradeHandler, ThresholdPolicy, EscalationAction
|
|
112
|
+
|
|
113
|
+
handler = ModelUpgradeHandler(
|
|
114
|
+
upgrade_map={
|
|
115
|
+
"claude-haiku-4-5": "claude-sonnet-4-6",
|
|
116
|
+
"claude-sonnet-4-6": "claude-opus-4-7",
|
|
117
|
+
}
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
policy = ThresholdPolicy(threshold=0.7, action=EscalationAction.MODEL_UPGRADE)
|
|
121
|
+
result = policy.evaluate(score, context={"model": "claude-haiku-4-5"})
|
|
122
|
+
|
|
123
|
+
if result.triggered:
|
|
124
|
+
upgrade_info = handler.handle(result, context={"model": "claude-haiku-4-5"})
|
|
125
|
+
print(f"Retry with: {upgrade_info['upgraded_model']}")
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
### Tool Restriction
|
|
129
|
+
|
|
130
|
+
```python
|
|
131
|
+
from confidence_escalation import ToolRestrictionHandler, ThresholdPolicy, EscalationAction
|
|
132
|
+
|
|
133
|
+
handler = ToolRestrictionHandler(
|
|
134
|
+
high_risk_tools=["delete_record", "send_email", "execute_sql"],
|
|
135
|
+
allow_read_only=True,
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
policy = ThresholdPolicy(threshold=0.65, action=EscalationAction.TOOL_RESTRICTION)
|
|
139
|
+
result = policy.evaluate(score, context={"available_tools": ["get_customer", "delete_record"]})
|
|
140
|
+
|
|
141
|
+
if result.triggered:
|
|
142
|
+
restriction = handler.handle(result, context={"available_tools": agent_tools})
|
|
143
|
+
safe_tools = restriction["allowed_tools"]
|
|
144
|
+
# Re-invoke agent with only safe_tools
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
### LangChain Integration
|
|
148
|
+
|
|
149
|
+
```python
|
|
150
|
+
from confidence_escalation.adapters.langchain import LangChainEscalationAdapter
|
|
151
|
+
from confidence_escalation.handlers import HumanInLoopHandler
|
|
152
|
+
|
|
153
|
+
adapter = LangChainEscalationAdapter(
|
|
154
|
+
threshold=0.65,
|
|
155
|
+
handlers=[HumanInLoopHandler(raise_on_trigger=True)],
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
# Attach as LangChain callback
|
|
159
|
+
chain = LLMChain(llm=llm, callbacks=[adapter.as_callback()])
|
|
160
|
+
|
|
161
|
+
# Or call directly from a LangGraph node
|
|
162
|
+
def research_node(state):
|
|
163
|
+
response = llm.invoke(state["messages"])
|
|
164
|
+
try:
|
|
165
|
+
adapter.on_llm_end(response.content, logprobs=response.response_metadata.get("logprobs"))
|
|
166
|
+
except HumanInLoopHandler.HumanReviewRequired:
|
|
167
|
+
return {"status": "escalated"}
|
|
168
|
+
return {"response": response.content}
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
### CrewAI Integration
|
|
172
|
+
|
|
173
|
+
```python
|
|
174
|
+
from crewai import Agent
|
|
175
|
+
from confidence_escalation.adapters.crewai import CrewAIEscalationAdapter
|
|
176
|
+
|
|
177
|
+
adapter = CrewAIEscalationAdapter(threshold=0.65)
|
|
178
|
+
|
|
179
|
+
agent = Agent(
|
|
180
|
+
role="Research Specialist",
|
|
181
|
+
goal="Analyze market trends",
|
|
182
|
+
backstory="...",
|
|
183
|
+
step_callback=adapter.step_callback,
|
|
184
|
+
)
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
### Google ADK Integration
|
|
188
|
+
|
|
189
|
+
```python
|
|
190
|
+
from google.adk.agents import BaseAgent
|
|
191
|
+
from confidence_escalation.adapters.google_adk import ADKEscalationAdapter
|
|
192
|
+
|
|
193
|
+
class GovernedAgent(BaseAgent):
|
|
194
|
+
def __init__(self, *args, **kwargs):
|
|
195
|
+
super().__init__(*args, **kwargs)
|
|
196
|
+
self._escalation = ADKEscalationAdapter(threshold=0.65)
|
|
197
|
+
|
|
198
|
+
async def _run_async_impl(self, ctx):
|
|
199
|
+
async for event in self._llm_agent._run_async_impl(ctx):
|
|
200
|
+
if event.is_final_response():
|
|
201
|
+
result = self._escalation.evaluate_event(event, ctx)
|
|
202
|
+
if result["triggered"]:
|
|
203
|
+
yield self._escalation.build_escalation_event(result)
|
|
204
|
+
return
|
|
205
|
+
yield event
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
---
|
|
209
|
+
|
|
210
|
+
## Composite Policy Chains
|
|
211
|
+
|
|
212
|
+
```python
|
|
213
|
+
from confidence_escalation import ThresholdPolicy, EscalationAction
|
|
214
|
+
from confidence_escalation.policy import CompositePolicy
|
|
215
|
+
|
|
216
|
+
policy = CompositePolicy(policies=[
|
|
217
|
+
ThresholdPolicy(threshold=0.25, action=EscalationAction.ABORT),
|
|
218
|
+
ThresholdPolicy(threshold=0.55, action=EscalationAction.HUMAN_IN_LOOP),
|
|
219
|
+
ThresholdPolicy(threshold=0.75, action=EscalationAction.COMPLIANCE_LOG),
|
|
220
|
+
])
|
|
221
|
+
|
|
222
|
+
result = policy.evaluate(score, context={"session_id": "abc"})
|
|
223
|
+
# First matching threshold wins
|
|
224
|
+
```
|
|
225
|
+
|
|
226
|
+
---
|
|
227
|
+
|
|
228
|
+
## OWASP Agentic AI Coverage
|
|
229
|
+
|
|
230
|
+
| OWASP ASI ID | Risk | Coverage |
|
|
231
|
+
|-------------|------|----------|
|
|
232
|
+
| ASI-09 | Human-Agent Trust Exploitation | Confidence gating before high-stakes actions |
|
|
233
|
+
| ASI-02 | Tool Misuse | Tool restriction handler removes high-risk tools at low confidence |
|
|
234
|
+
| ASI-03 | Identity/Privilege Abuse | ComplianceLoggingHandler creates immutable audit trail |
|
|
235
|
+
|
|
236
|
+
---
|
|
237
|
+
|
|
238
|
+
## Related Packages
|
|
239
|
+
|
|
240
|
+
- [voice-ai-governance](https://github.com/ashutoshrana/voice-ai-governance) — HIPAA/FERPA/EU AI Act compliance for voice AI pipelines
|
|
241
|
+
- [regulated-ai-governance](https://github.com/ashutoshrana/regulated-ai-governance) — Runtime tool authorization and capability scoping
|
|
242
|
+
- [enterprise-rag-patterns](https://github.com/ashutoshrana/enterprise-rag-patterns) — FERPA/HIPAA/GDPR-compliant RAG patterns
|
|
243
|
+
|
|
244
|
+
---
|
|
245
|
+
|
|
246
|
+
## License
|
|
247
|
+
|
|
248
|
+
MIT License. See [LICENSE](LICENSE).
|