stateprobe 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stateprobe-0.3.0/LICENSE +21 -0
- stateprobe-0.3.0/PKG-INFO +205 -0
- stateprobe-0.3.0/README.md +167 -0
- stateprobe-0.3.0/pyproject.toml +63 -0
- stateprobe-0.3.0/setup.cfg +4 -0
- stateprobe-0.3.0/stateprobe/__init__.py +53 -0
- stateprobe-0.3.0/stateprobe/__main__.py +13 -0
- stateprobe-0.3.0/stateprobe/cli.py +2278 -0
- stateprobe-0.3.0/stateprobe/detector.py +401 -0
- stateprobe-0.3.0/stateprobe/engines/__init__.py +49 -0
- stateprobe-0.3.0/stateprobe/engines/base.py +100 -0
- stateprobe-0.3.0/stateprobe/engines/lab.py +401 -0
- stateprobe-0.3.0/stateprobe/engines/llm_judge.py +342 -0
- stateprobe-0.3.0/stateprobe/engines/static.py +118 -0
- stateprobe-0.3.0/stateprobe/enterprise/__init__.py +32 -0
- stateprobe-0.3.0/stateprobe/eval/__init__.py +14 -0
- stateprobe-0.3.0/stateprobe/eval/client.py +125 -0
- stateprobe-0.3.0/stateprobe/eval/scorer.py +242 -0
- stateprobe-0.3.0/stateprobe/html_report.py +511 -0
- stateprobe-0.3.0/stateprobe/lab/__init__.py +10 -0
- stateprobe-0.3.0/stateprobe/lab/cache.py +169 -0
- stateprobe-0.3.0/stateprobe/lab/deepseek_pairs.py +105 -0
- stateprobe-0.3.0/stateprobe/lab/probe.py +264 -0
- stateprobe-0.3.0/stateprobe/mcp_server.py +114 -0
- stateprobe-0.3.0/stateprobe/models.py +322 -0
- stateprobe-0.3.0/stateprobe/rewriter.py +256 -0
- stateprobe-0.3.0/stateprobe/rules.py +958 -0
- stateprobe-0.3.0/stateprobe/skill/__init__.py +53 -0
- stateprobe-0.3.0/stateprobe/skill/attention.py +1769 -0
- stateprobe-0.3.0/stateprobe/structural.py +176 -0
- stateprobe-0.3.0/stateprobe.egg-info/PKG-INFO +205 -0
- stateprobe-0.3.0/stateprobe.egg-info/SOURCES.txt +45 -0
- stateprobe-0.3.0/stateprobe.egg-info/dependency_links.txt +1 -0
- stateprobe-0.3.0/stateprobe.egg-info/entry_points.txt +3 -0
- stateprobe-0.3.0/stateprobe.egg-info/requires.txt +17 -0
- stateprobe-0.3.0/stateprobe.egg-info/top_level.txt +1 -0
- stateprobe-0.3.0/tests/test_cli.py +516 -0
- stateprobe-0.3.0/tests/test_cli_dialog.py +51 -0
- stateprobe-0.3.0/tests/test_detector.py +141 -0
- stateprobe-0.3.0/tests/test_engines.py +456 -0
- stateprobe-0.3.0/tests/test_engines_lab.py +386 -0
- stateprobe-0.3.0/tests/test_eval.py +79 -0
- stateprobe-0.3.0/tests/test_lab.py +50 -0
- stateprobe-0.3.0/tests/test_mcp_server.py +117 -0
- stateprobe-0.3.0/tests/test_rules.py +142 -0
- stateprobe-0.3.0/tests/test_skill.py +690 -0
- stateprobe-0.3.0/tests/test_structural.py +183 -0
stateprobe-0.3.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 StateProbe Contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: stateprobe
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: The attention layer for LLM agents — see what the model fires before it ships.
|
|
5
|
+
Author: StateProbe Contributors
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/Erye932/stateprobe
|
|
8
|
+
Project-URL: Issues, https://github.com/Erye932/stateprobe/issues
|
|
9
|
+
Keywords: llm,agent,mcp,claude-code,prompt-engineering,prompt-debugging,llm-evaluation,persona-vectors,activation-steering
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
20
|
+
Requires-Python: >=3.9
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
License-File: LICENSE
|
|
23
|
+
Requires-Dist: click>=8.1.0
|
|
24
|
+
Requires-Dist: rich>=13.7.0
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: mcp>=1.7.1; extra == "dev"
|
|
27
|
+
Requires-Dist: pytest>=7.4.0; extra == "dev"
|
|
28
|
+
Requires-Dist: pytest-cov>=4.1.0; extra == "dev"
|
|
29
|
+
Provides-Extra: lab
|
|
30
|
+
Requires-Dist: torch>=2.2.0; extra == "lab"
|
|
31
|
+
Requires-Dist: transformers>=4.40.0; extra == "lab"
|
|
32
|
+
Requires-Dist: accelerate>=0.28.0; extra == "lab"
|
|
33
|
+
Requires-Dist: safetensors>=0.4.0; extra == "lab"
|
|
34
|
+
Requires-Dist: numpy>=1.26.0; extra == "lab"
|
|
35
|
+
Provides-Extra: mcp
|
|
36
|
+
Requires-Dist: mcp>=1.7.1; extra == "mcp"
|
|
37
|
+
Dynamic: license-file
|
|
38
|
+
|
|
39
|
+
# StateProbe
|
|
40
|
+
|
|
41
|
+
[](https://pypi.org/project/stateprobe/)
|
|
42
|
+
[](LICENSE)
|
|
43
|
+
[](https://www.python.org/)
|
|
44
|
+
[](https://github.com/Erye932/stateprobe/actions/workflows/ci.yml)
|
|
45
|
+
|
|
46
|
+
> **The attention layer for LLM agents.**
|
|
47
|
+
|
|
48
|
+
Your agent already drifted — wrong focus, stale context, confidently editing files you never asked about. StateProbe catches it **before** the agent ships the answer. Works with **Claude Code**, **Cursor**, **Cline**, **Continue**, and any MCP host.
|
|
49
|
+
|
|
50
|
+
English | [简体中文](README.zh-CN.md)
|
|
51
|
+
|
|
52
|
+
---
|
|
53
|
+
|
|
54
|
+
## Why
|
|
55
|
+
|
|
56
|
+
LLM agents drift. They miss the user's actual point, get steered by stale context, or burn cycles on the wrong subtopic. Today's fix is "rewrite the prompt and pray." StateProbe gives you a sharper tool:
|
|
57
|
+
|
|
58
|
+
- **See** what the agent is about to focus on, before it answers
|
|
59
|
+
- **Decide** to continue, rewrite the focus, ask a boundary question, or cut stale context
|
|
60
|
+
- **Audit** the actual output afterwards and surface drift
|
|
61
|
+
|
|
62
|
+
Runs **locally**. Costs **zero LLM tokens** by default. Plugs into any MCP host.
|
|
63
|
+
|
|
64
|
+
## Install
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
pip install stateprobe
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
For MCP integration (Claude Code, Cursor, Cline, Continue):
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
pip install "stateprobe[mcp]"
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
For activation projection on open-source DeepSeek (optional):
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
pip install "stateprobe[lab]"
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
## 30-second demo
|
|
83
|
+
|
|
84
|
+
Before the agent answers, preview its planned focus:
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
stateprobe skill preview \
|
|
88
|
+
--context-text "Focus on safety; do not include deprecated APIs." \
|
|
89
|
+
--plan-text "I will list deprecated APIs and explain why they are unsafe."
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
After the agent answers, audit alignment with user requirements:
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
stateprobe skill overlay \
|
|
96
|
+
--context examples/skill_attention_context.txt \
|
|
97
|
+
--output examples/skill_attention_output.txt
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Or run the legacy prompt diagnostic, with the included
|
|
101
|
+
[`smart_but_not_answering`](demos/smart_but_not_answering) demo:
|
|
102
|
+
|
|
103
|
+
```bash
|
|
104
|
+
stateprobe demo
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
## What it gives back
|
|
108
|
+
|
|
109
|
+
`stateprobe skill preview` returns a JSON `activation_decision` — your agent host branches on it:
|
|
110
|
+
|
|
111
|
+
| Action | Meaning |
|
|
112
|
+
| --- | --- |
|
|
113
|
+
| `continue` | Aligned. Agent can speak. |
|
|
114
|
+
| `rewrite_planned_focus` | Plan misses user's actual must. **Don't ship.** Rewrite focus first. |
|
|
115
|
+
| `ask_boundary_question` | Visual / creative ambiguity. **Ask the user one yes/no first.** |
|
|
116
|
+
| `cut_context_contamination` | Agent is following stale context. **Cut the old direction first.** |
|
|
117
|
+
|
|
118
|
+
`stateprobe skill overlay` returns an `interrupt_level` (`ok` / `watch` / `interrupt`) plus `attention_gaps` and `control_levers` for the next turn.
|
|
119
|
+
|
|
120
|
+
Full schemas: [Skill spec](docs/SKILL_ATTENTION_HUD.md), [MCP server](docs/MCP_SERVER.md).
|
|
121
|
+
|
|
122
|
+
## Two product lines
|
|
123
|
+
|
|
124
|
+
| Line | What | Status |
|
|
125
|
+
| --- | --- | --- |
|
|
126
|
+
| **Skill — Agent Attention HUD** | External control layer. Text-to-text task attention. Works with closed and open models. | ✅ Available |
|
|
127
|
+
| **Lab — Activation Projection** | Projects prompt activations onto Persona Vectors on open-source DeepSeek-R1-Distill-Qwen. Opt-in. | ✅ Available |
|
|
128
|
+
| **Enterprise — Runtime Probe** | Future direction: hidden states, router traces, expert routing on open-source models. | 🛠 Not yet implemented |
|
|
129
|
+
|
|
130
|
+
**Boundary**: closed-source APIs (OpenAI, Claude) cannot expose hidden states — for them, StateProbe runs the text-level Skill layer only. Open-source models (DeepSeek, Qwen, Llama) unlock the Lab layer. OpenAI/Claude 物理上读不到 hidden states; the Lab layer requires open weights.
|
|
131
|
+
|
|
132
|
+
## How it differs
|
|
133
|
+
|
|
134
|
+
| | promptfoo | Guardrails AI | LangSmith | **StateProbe** |
|
|
135
|
+
| --- | --- | --- | --- | --- |
|
|
136
|
+
| Analyzes | Output quality | Output safety | Call traces | **Agent's planned attention before output** |
|
|
137
|
+
| When | After release | Runtime | Production | **Before each turn** |
|
|
138
|
+
| LLM API needed | Yes | Yes | Yes | **No (default)** |
|
|
139
|
+
|
|
140
|
+
Complementary, not competitive. promptfoo / Guardrails check what came out; StateProbe shapes what's about to come out.
|
|
141
|
+
|
|
142
|
+
## Architecture
|
|
143
|
+
|
|
144
|
+
Hybrid evidence pipeline ([ADR_009](docs/adr/009-hybrid-engine.md)): independent contributors emit confidence-weighted evidence, aggregated into 8 behavior axes. Static rules are always on (zero cost); the LLM and Lab layers are opt-in and stack on top.
|
|
145
|
+
|
|
146
|
+
| Layer | Purpose | Cost |
|
|
147
|
+
| --- | --- | --- |
|
|
148
|
+
| **Static Mode** (`StaticRuleContributor`) | Regex rules. Always on. | Zero |
|
|
149
|
+
| **LLM judge** (`LLMJudgeContributor`) | LLM semantic evidence. Opt-in via `--llm-augment`. | API call |
|
|
150
|
+
| **DeepSeek Lab** (`LabContributor`) | Hidden-state projection on DeepSeek-R1-Distill-Qwen-1.5B. Opt-in via `--lab-augment`. | Local GPU |
|
|
151
|
+
| **Black-box Eval** (independent) | Runs original / rewritten prompts on a target model and scores outputs. | API call |
|
|
152
|
+
|
|
153
|
+
Theoretical foundation:
|
|
154
|
+
|
|
155
|
+
- Anthropic — [Persona Vectors: Monitoring and Controlling Character Traits in Language Models](https://arxiv.org/abs/2507.21509)
|
|
156
|
+
- DeepSeek-AI — [DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning](https://arxiv.org/abs/2501.12948)
|
|
157
|
+
|
|
158
|
+
DeepSeek-first, not DeepSeek-only — see [DeepSeek roadmap](docs/DEEPSEEK_ROADMAP.md).
|
|
159
|
+
|
|
160
|
+
## Roadmap
|
|
161
|
+
|
|
162
|
+
- **v0.3** *(current)* — Skill HUD, MCP server, Lab activation projection on 4 axes
|
|
163
|
+
- **v0.3.1** — Remaining 4 axes; embedding contributor for offline fallback; VS Code / Cursor extension
|
|
164
|
+
- **v0.4** — MoE expert routing contributor on DeepSeek-MoE
|
|
165
|
+
- **v0.5** — Named steering vectors; output-time intervention API
|
|
166
|
+
|
|
167
|
+
See [CHANGELOG](CHANGELOG.md) for the full version history.
|
|
168
|
+
|
|
169
|
+
## Documentation
|
|
170
|
+
|
|
171
|
+
- [Skill spec](docs/SKILL_ATTENTION_HUD.md) — attention HUD reference
|
|
172
|
+
- [MCP server](docs/MCP_SERVER.md) — Claude Code / Cursor / Cline / Continue setup
|
|
173
|
+
- [Architecture](docs/ARCHITECTURE.md) — hybrid evidence pipeline
|
|
174
|
+
- [FAQ](docs/FAQ.md) — common objections answered
|
|
175
|
+
|
|
176
|
+
<details>
|
|
177
|
+
<summary><b>More docs</b> — evidence model, ADRs, roadmaps, contributor guides</summary>
|
|
178
|
+
|
|
179
|
+
- [Evidence model](docs/EVIDENCE_MODEL.md) — three-layer evidence boundaries
|
|
180
|
+
- [DeepSeek roadmap](docs/DEEPSEEK_ROADMAP.md) — DeepSeek-first, not DeepSeek-only
|
|
181
|
+
- [Architecture decisions](docs/adr/) — ADRs for hybrid pipeline and lab contributor
|
|
182
|
+
- [Publishing](docs/PUBLISHING.md) — release process
|
|
183
|
+
- [CHANGELOG](CHANGELOG.md) / [CITATION](CITATION.cff) / [CODE_OF_CONDUCT](CODE_OF_CONDUCT.md) / [CONTRIBUTING](CONTRIBUTING.md)
|
|
184
|
+
|
|
185
|
+
</details>
|
|
186
|
+
|
|
187
|
+
中文文档(含 China 镜像、PowerShell 编码 fix、完整命令样例):[README.zh-CN.md](README.zh-CN.md)
|
|
188
|
+
|
|
189
|
+
## Contributing
|
|
190
|
+
|
|
191
|
+
Rule library quality = the project's core value. If you find a prompt pattern that isn't detected, a misfire, or want a new target preset — open an issue or PR.
|
|
192
|
+
|
|
193
|
+
Each rule contribution must include: pattern / affected axis / direction / weight / **mechanism** / **paper citation**. See [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
194
|
+
|
|
195
|
+
```bash
|
|
196
|
+
python scripts/acceptance_check.py
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
## License
|
|
200
|
+
|
|
201
|
+
MIT — see [LICENSE](LICENSE).
|
|
202
|
+
|
|
203
|
+
---
|
|
204
|
+
|
|
205
|
+
Built on Anthropic interpretability and DeepSeek-AI open research. This tool turns those findings into something agent hosts and prompt engineers can use every day, without having to actually answer the question of how the model "thinks" — just what it's about to focus on next.
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
# StateProbe
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/stateprobe/)
|
|
4
|
+
[](LICENSE)
|
|
5
|
+
[](https://www.python.org/)
|
|
6
|
+
[](https://github.com/Erye932/stateprobe/actions/workflows/ci.yml)
|
|
7
|
+
|
|
8
|
+
> **The attention layer for LLM agents.**
|
|
9
|
+
|
|
10
|
+
Your agent already drifted — wrong focus, stale context, confidently editing files you never asked about. StateProbe catches it **before** the agent ships the answer. Works with **Claude Code**, **Cursor**, **Cline**, **Continue**, and any MCP host.
|
|
11
|
+
|
|
12
|
+
English | [简体中文](README.zh-CN.md)
|
|
13
|
+
|
|
14
|
+
---
|
|
15
|
+
|
|
16
|
+
## Why
|
|
17
|
+
|
|
18
|
+
LLM agents drift. They miss the user's actual point, get steered by stale context, or burn cycles on the wrong subtopic. Today's fix is "rewrite the prompt and pray." StateProbe gives you a sharper tool:
|
|
19
|
+
|
|
20
|
+
- **See** what the agent is about to focus on, before it answers
|
|
21
|
+
- **Decide** to continue, rewrite the focus, ask a boundary question, or cut stale context
|
|
22
|
+
- **Audit** the actual output afterwards and surface drift
|
|
23
|
+
|
|
24
|
+
Runs **locally**. Costs **zero LLM tokens** by default. Plugs into any MCP host.
|
|
25
|
+
|
|
26
|
+
## Install
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
pip install stateprobe
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
For MCP integration (Claude Code, Cursor, Cline, Continue):
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
pip install "stateprobe[mcp]"
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
For activation projection on open-source DeepSeek (optional):
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
pip install "stateprobe[lab]"
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
## 30-second demo
|
|
45
|
+
|
|
46
|
+
Before the agent answers, preview its planned focus:
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
stateprobe skill preview \
|
|
50
|
+
--context-text "Focus on safety; do not include deprecated APIs." \
|
|
51
|
+
--plan-text "I will list deprecated APIs and explain why they are unsafe."
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
After the agent answers, audit alignment with user requirements:
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
stateprobe skill overlay \
|
|
58
|
+
--context examples/skill_attention_context.txt \
|
|
59
|
+
--output examples/skill_attention_output.txt
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Or run the legacy prompt diagnostic, with the included
|
|
63
|
+
[`smart_but_not_answering`](demos/smart_but_not_answering) demo:
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
stateprobe demo
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
## What it gives back
|
|
70
|
+
|
|
71
|
+
`stateprobe skill preview` returns a JSON `activation_decision` — your agent host branches on it:
|
|
72
|
+
|
|
73
|
+
| Action | Meaning |
|
|
74
|
+
| --- | --- |
|
|
75
|
+
| `continue` | Aligned. Agent can speak. |
|
|
76
|
+
| `rewrite_planned_focus` | Plan misses user's actual must. **Don't ship.** Rewrite focus first. |
|
|
77
|
+
| `ask_boundary_question` | Visual / creative ambiguity. **Ask the user one yes/no first.** |
|
|
78
|
+
| `cut_context_contamination` | Agent is following stale context. **Cut the old direction first.** |
|
|
79
|
+
|
|
80
|
+
`stateprobe skill overlay` returns an `interrupt_level` (`ok` / `watch` / `interrupt`) plus `attention_gaps` and `control_levers` for the next turn.
|
|
81
|
+
|
|
82
|
+
Full schemas: [Skill spec](docs/SKILL_ATTENTION_HUD.md), [MCP server](docs/MCP_SERVER.md).
|
|
83
|
+
|
|
84
|
+
## Two product lines
|
|
85
|
+
|
|
86
|
+
| Line | What | Status |
|
|
87
|
+
| --- | --- | --- |
|
|
88
|
+
| **Skill — Agent Attention HUD** | External control layer. Text-to-text task attention. Works with closed and open models. | ✅ Available |
|
|
89
|
+
| **Lab — Activation Projection** | Projects prompt activations onto Persona Vectors on open-source DeepSeek-R1-Distill-Qwen. Opt-in. | ✅ Available |
|
|
90
|
+
| **Enterprise — Runtime Probe** | Future direction: hidden states, router traces, expert routing on open-source models. | 🛠 Not yet implemented |
|
|
91
|
+
|
|
92
|
+
**Boundary**: closed-source APIs (OpenAI, Claude) cannot expose hidden states — for them, StateProbe runs the text-level Skill layer only. Open-source models (DeepSeek, Qwen, Llama) unlock the Lab layer. OpenAI/Claude 物理上读不到 hidden states; the Lab layer requires open weights.
|
|
93
|
+
|
|
94
|
+
## How it differs
|
|
95
|
+
|
|
96
|
+
| | promptfoo | Guardrails AI | LangSmith | **StateProbe** |
|
|
97
|
+
| --- | --- | --- | --- | --- |
|
|
98
|
+
| Analyzes | Output quality | Output safety | Call traces | **Agent's planned attention before output** |
|
|
99
|
+
| When | After release | Runtime | Production | **Before each turn** |
|
|
100
|
+
| LLM API needed | Yes | Yes | Yes | **No (default)** |
|
|
101
|
+
|
|
102
|
+
Complementary, not competitive. promptfoo / Guardrails check what came out; StateProbe shapes what's about to come out.
|
|
103
|
+
|
|
104
|
+
## Architecture
|
|
105
|
+
|
|
106
|
+
Hybrid evidence pipeline ([ADR_009](docs/adr/009-hybrid-engine.md)): independent contributors emit confidence-weighted evidence, aggregated into 8 behavior axes. Static rules are always on (zero cost); the LLM and Lab layers are opt-in and stack on top.
|
|
107
|
+
|
|
108
|
+
| Layer | Purpose | Cost |
|
|
109
|
+
| --- | --- | --- |
|
|
110
|
+
| **Static Mode** (`StaticRuleContributor`) | Regex rules. Always on. | Zero |
|
|
111
|
+
| **LLM judge** (`LLMJudgeContributor`) | LLM semantic evidence. Opt-in via `--llm-augment`. | API call |
|
|
112
|
+
| **DeepSeek Lab** (`LabContributor`) | Hidden-state projection on DeepSeek-R1-Distill-Qwen-1.5B. Opt-in via `--lab-augment`. | Local GPU |
|
|
113
|
+
| **Black-box Eval** (independent) | Runs original / rewritten prompts on a target model and scores outputs. | API call |
|
|
114
|
+
|
|
115
|
+
Theoretical foundation:
|
|
116
|
+
|
|
117
|
+
- Anthropic — [Persona Vectors: Monitoring and Controlling Character Traits in Language Models](https://arxiv.org/abs/2507.21509)
|
|
118
|
+
- DeepSeek-AI — [DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning](https://arxiv.org/abs/2501.12948)
|
|
119
|
+
|
|
120
|
+
DeepSeek-first, not DeepSeek-only — see [DeepSeek roadmap](docs/DEEPSEEK_ROADMAP.md).
|
|
121
|
+
|
|
122
|
+
## Roadmap
|
|
123
|
+
|
|
124
|
+
- **v0.3** *(current)* — Skill HUD, MCP server, Lab activation projection on 4 axes
|
|
125
|
+
- **v0.3.1** — Remaining 4 axes; embedding contributor for offline fallback; VS Code / Cursor extension
|
|
126
|
+
- **v0.4** — MoE expert routing contributor on DeepSeek-MoE
|
|
127
|
+
- **v0.5** — Named steering vectors; output-time intervention API
|
|
128
|
+
|
|
129
|
+
See [CHANGELOG](CHANGELOG.md) for the full version history.
|
|
130
|
+
|
|
131
|
+
## Documentation
|
|
132
|
+
|
|
133
|
+
- [Skill spec](docs/SKILL_ATTENTION_HUD.md) — attention HUD reference
|
|
134
|
+
- [MCP server](docs/MCP_SERVER.md) — Claude Code / Cursor / Cline / Continue setup
|
|
135
|
+
- [Architecture](docs/ARCHITECTURE.md) — hybrid evidence pipeline
|
|
136
|
+
- [FAQ](docs/FAQ.md) — common objections answered
|
|
137
|
+
|
|
138
|
+
<details>
|
|
139
|
+
<summary><b>More docs</b> — evidence model, ADRs, roadmaps, contributor guides</summary>
|
|
140
|
+
|
|
141
|
+
- [Evidence model](docs/EVIDENCE_MODEL.md) — three-layer evidence boundaries
|
|
142
|
+
- [DeepSeek roadmap](docs/DEEPSEEK_ROADMAP.md) — DeepSeek-first, not DeepSeek-only
|
|
143
|
+
- [Architecture decisions](docs/adr/) — ADRs for hybrid pipeline and lab contributor
|
|
144
|
+
- [Publishing](docs/PUBLISHING.md) — release process
|
|
145
|
+
- [CHANGELOG](CHANGELOG.md) / [CITATION](CITATION.cff) / [CODE_OF_CONDUCT](CODE_OF_CONDUCT.md) / [CONTRIBUTING](CONTRIBUTING.md)
|
|
146
|
+
|
|
147
|
+
</details>
|
|
148
|
+
|
|
149
|
+
中文文档(含 China 镜像、PowerShell 编码 fix、完整命令样例):[README.zh-CN.md](README.zh-CN.md)
|
|
150
|
+
|
|
151
|
+
## Contributing
|
|
152
|
+
|
|
153
|
+
Rule library quality = the project's core value. If you find a prompt pattern that isn't detected, a misfire, or want a new target preset — open an issue or PR.
|
|
154
|
+
|
|
155
|
+
Each rule contribution must include: pattern / affected axis / direction / weight / **mechanism** / **paper citation**. See [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
156
|
+
|
|
157
|
+
```bash
|
|
158
|
+
python scripts/acceptance_check.py
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
## License
|
|
162
|
+
|
|
163
|
+
MIT — see [LICENSE](LICENSE).
|
|
164
|
+
|
|
165
|
+
---
|
|
166
|
+
|
|
167
|
+
Built on Anthropic interpretability and DeepSeek-AI open research. This tool turns those findings into something agent hosts and prompt engineers can use every day, without having to actually answer the question of how the model "thinks" — just what it's about to focus on next.
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "stateprobe"
|
|
7
|
+
version = "0.3.0"
|
|
8
|
+
description = "The attention layer for LLM agents — see what the model fires before it ships."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { text = "MIT" }
|
|
11
|
+
requires-python = ">=3.9"
|
|
12
|
+
authors = [
|
|
13
|
+
{ name = "StateProbe Contributors" }
|
|
14
|
+
]
|
|
15
|
+
keywords = ["llm", "agent", "mcp", "claude-code", "prompt-engineering", "prompt-debugging", "llm-evaluation", "persona-vectors", "activation-steering"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 3 - Alpha",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"License :: OSI Approved :: MIT License",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.9",
|
|
22
|
+
"Programming Language :: Python :: 3.10",
|
|
23
|
+
"Programming Language :: Python :: 3.11",
|
|
24
|
+
"Programming Language :: Python :: 3.12",
|
|
25
|
+
"Programming Language :: Python :: 3.13",
|
|
26
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
27
|
+
]
|
|
28
|
+
dependencies = [
|
|
29
|
+
"click>=8.1.0",
|
|
30
|
+
"rich>=13.7.0",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
[project.optional-dependencies]
|
|
34
|
+
dev = [
|
|
35
|
+
"mcp>=1.7.1",
|
|
36
|
+
"pytest>=7.4.0",
|
|
37
|
+
"pytest-cov>=4.1.0",
|
|
38
|
+
]
|
|
39
|
+
lab = [
|
|
40
|
+
"torch>=2.2.0",
|
|
41
|
+
"transformers>=4.40.0",
|
|
42
|
+
"accelerate>=0.28.0",
|
|
43
|
+
"safetensors>=0.4.0",
|
|
44
|
+
"numpy>=1.26.0",
|
|
45
|
+
]
|
|
46
|
+
mcp = [
|
|
47
|
+
"mcp>=1.7.1",
|
|
48
|
+
]
|
|
49
|
+
|
|
50
|
+
[project.scripts]
|
|
51
|
+
stateprobe = "stateprobe.cli:main"
|
|
52
|
+
stateprobe-mcp = "stateprobe.mcp_server:main"
|
|
53
|
+
|
|
54
|
+
[project.urls]
|
|
55
|
+
Homepage = "https://github.com/Erye932/stateprobe"
|
|
56
|
+
Issues = "https://github.com/Erye932/stateprobe/issues"
|
|
57
|
+
|
|
58
|
+
[tool.setuptools.packages.find]
|
|
59
|
+
where = ["."]
|
|
60
|
+
include = ["stateprobe*"]
|
|
61
|
+
|
|
62
|
+
[tool.setuptools.package-data]
|
|
63
|
+
stateprobe = ["templates/*.html"]
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""StateProbe — Prompt State Debugger.
|
|
2
|
+
|
|
3
|
+
Diagnose which behavioral vectors your prompt activates in an LLM,
|
|
4
|
+
identify pollution sources, and get rewrite suggestions to reach a target state.
|
|
5
|
+
|
|
6
|
+
Theoretical foundation:
|
|
7
|
+
- Anthropic, "Persona Vectors: Monitoring and Controlling Character Traits in
|
|
8
|
+
Language Models" (arXiv:2507.21509, 2025)
|
|
9
|
+
- DeepSeek-AI, "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via
|
|
10
|
+
Reinforcement Learning" (arXiv:2501.12948, 2025)
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
__version__ = "0.3.0"
|
|
14
|
+
|
|
15
|
+
from stateprobe.models import (
|
|
16
|
+
Axis,
|
|
17
|
+
AxisReading,
|
|
18
|
+
PollutionSource,
|
|
19
|
+
Report,
|
|
20
|
+
TargetPreset,
|
|
21
|
+
)
|
|
22
|
+
from stateprobe.detector import diagnose
|
|
23
|
+
from stateprobe.rewriter import suggest_rewrite
|
|
24
|
+
from stateprobe.engines import (
|
|
25
|
+
Engine,
|
|
26
|
+
EngineError,
|
|
27
|
+
EngineUnavailable,
|
|
28
|
+
EvidenceContributor,
|
|
29
|
+
LLMJudgeContributor,
|
|
30
|
+
LLMJudgeEngine,
|
|
31
|
+
StaticEngine,
|
|
32
|
+
StaticRuleContributor,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
__all__ = [
|
|
36
|
+
"Axis",
|
|
37
|
+
"AxisReading",
|
|
38
|
+
"PollutionSource",
|
|
39
|
+
"Report",
|
|
40
|
+
"TargetPreset",
|
|
41
|
+
"diagnose",
|
|
42
|
+
"suggest_rewrite",
|
|
43
|
+
"EvidenceContributor",
|
|
44
|
+
"EngineError",
|
|
45
|
+
"EngineUnavailable",
|
|
46
|
+
"StaticRuleContributor",
|
|
47
|
+
"LLMJudgeContributor",
|
|
48
|
+
# Deprecated aliases (removed in v0.3):
|
|
49
|
+
"Engine",
|
|
50
|
+
"StaticEngine",
|
|
51
|
+
"LLMJudgeEngine",
|
|
52
|
+
"__version__",
|
|
53
|
+
]
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""Entry point for ``python -m stateprobe``.
|
|
2
|
+
|
|
3
|
+
Forwards to the standard CLI defined by ``[project.scripts] stateprobe`` so
|
|
4
|
+
users who don't have the ``stateprobe`` script on PATH (e.g. ``pip install
|
|
5
|
+
--user`` without PATH set, or running directly from a checkout) can still
|
|
6
|
+
invoke the CLI in the conventional way.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from stateprobe.cli import main
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
if __name__ == "__main__":
|
|
13
|
+
main()
|