honestcode 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- honestcode-0.5.0/LICENSE +21 -0
- honestcode-0.5.0/PKG-INFO +669 -0
- honestcode-0.5.0/README.md +620 -0
- honestcode-0.5.0/honestcode/__init__.py +19 -0
- honestcode-0.5.0/honestcode/cli.py +428 -0
- honestcode-0.5.0/honestcode/graph/__init__.py +13 -0
- honestcode-0.5.0/honestcode/graph/graph_store.py +276 -0
- honestcode-0.5.0/honestcode/graph/watcher.py +116 -0
- honestcode-0.5.0/honestcode/guards/__init__.py +104 -0
- honestcode-0.5.0/honestcode/guards/guard.py +777 -0
- honestcode-0.5.0/honestcode/guards/store.py +457 -0
- honestcode-0.5.0/honestcode/guards/taxonomy.py +228 -0
- honestcode-0.5.0/honestcode/honest/__init__.py +19 -0
- honestcode-0.5.0/honestcode/honest/project_binding.py +200 -0
- honestcode-0.5.0/honestcode/honest/router.py +205 -0
- honestcode-0.5.0/honestcode/honest/symbol_index.py +446 -0
- honestcode-0.5.0/honestcode/integrations/__init__.py +14 -0
- honestcode-0.5.0/honestcode/integrations/pydantic_ai.py +304 -0
- honestcode-0.5.0/honestcode/mcp/__init__.py +46 -0
- honestcode-0.5.0/honestcode/mcp/knowledge_base.py +290 -0
- honestcode-0.5.0/honestcode/mcp/server.py +357 -0
- honestcode-0.5.0/honestcode/mcp/tools.py +2002 -0
- honestcode-0.5.0/honestcode/py.typed +0 -0
- honestcode-0.5.0/honestcode/sandbox/__init__.py +334 -0
- honestcode-0.5.0/honestcode/structure/__init__.py +6 -0
- honestcode-0.5.0/honestcode/structure/extractor.py +353 -0
- honestcode-0.5.0/honestcode/structure/multi_lang.py +267 -0
- honestcode-0.5.0/honestcode/structure/relations.py +47 -0
- honestcode-0.5.0/honestcode/structure/utils.py +262 -0
- honestcode-0.5.0/honestcode/verify/__init__.py +19 -0
- honestcode-0.5.0/honestcode/verify/evidence.py +195 -0
- honestcode-0.5.0/honestcode/verify/surface.py +497 -0
- honestcode-0.5.0/honestcode/verify/verify.py +1074 -0
- honestcode-0.5.0/honestcode.egg-info/PKG-INFO +669 -0
- honestcode-0.5.0/honestcode.egg-info/SOURCES.txt +55 -0
- honestcode-0.5.0/honestcode.egg-info/dependency_links.txt +1 -0
- honestcode-0.5.0/honestcode.egg-info/entry_points.txt +3 -0
- honestcode-0.5.0/honestcode.egg-info/requires.txt +24 -0
- honestcode-0.5.0/honestcode.egg-info/top_level.txt +1 -0
- honestcode-0.5.0/pyproject.toml +137 -0
- honestcode-0.5.0/setup.cfg +4 -0
- honestcode-0.5.0/tests/test_advanced.py +193 -0
- honestcode-0.5.0/tests/test_benchmarks.py +60 -0
- honestcode-0.5.0/tests/test_claude_hook.py +86 -0
- honestcode-0.5.0/tests/test_cli.py +206 -0
- honestcode-0.5.0/tests/test_encoding.py +106 -0
- honestcode-0.5.0/tests/test_false_positives.py +393 -0
- honestcode-0.5.0/tests/test_fixes.py +669 -0
- honestcode-0.5.0/tests/test_graph.py +306 -0
- honestcode-0.5.0/tests/test_guards.py +1249 -0
- honestcode-0.5.0/tests/test_hardening.py +565 -0
- honestcode-0.5.0/tests/test_more.py +196 -0
- honestcode-0.5.0/tests/test_pydantic_ai_integration.py +345 -0
- honestcode-0.5.0/tests/test_server.py +170 -0
- honestcode-0.5.0/tests/test_tools.py +142 -0
- honestcode-0.5.0/tests/test_value_drift.py +231 -0
- honestcode-0.5.0/tests/test_verify.py +263 -0
honestcode-0.5.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 HonestCode Team
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,669 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: honestcode
|
|
3
|
+
Version: 0.5.0
|
|
4
|
+
Summary: The deterministic verification layer for AI coding agents. Catch invented APIs, undefined symbols, and wrong calls before your agent moves on.
|
|
5
|
+
Author: HonestCode Team
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/Fengrru/honestcode
|
|
8
|
+
Project-URL: Documentation, https://github.com/Fengrru/honestcode#readme
|
|
9
|
+
Project-URL: Repository, https://github.com/Fengrru/honestcode
|
|
10
|
+
Project-URL: Issues, https://github.com/Fengrru/honestcode/issues
|
|
11
|
+
Keywords: mcp,model-context-protocol,verification,ai-coding-agent,hallucination-detection,invented-api,code-generation,static-analysis,llm,ai,code-analysis,agent-tooling
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
21
|
+
Classifier: Topic :: Software Development :: Code Generators
|
|
22
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
23
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
24
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
25
|
+
Classifier: Typing :: Typed
|
|
26
|
+
Requires-Python: >=3.10
|
|
27
|
+
Description-Content-Type: text/markdown
|
|
28
|
+
License-File: LICENSE
|
|
29
|
+
Requires-Dist: mcp<2.0,>=1.6.0
|
|
30
|
+
Requires-Dist: pyyaml>=6.0
|
|
31
|
+
Requires-Dist: tomli>=2.0.0; python_version < "3.11"
|
|
32
|
+
Provides-Extra: dev
|
|
33
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
34
|
+
Requires-Dist: pytest-asyncio>=0.24; extra == "dev"
|
|
35
|
+
Requires-Dist: pytest-cov>=4.0; extra == "dev"
|
|
36
|
+
Requires-Dist: ruff>=0.6; extra == "dev"
|
|
37
|
+
Requires-Dist: pre-commit>=3.0; extra == "dev"
|
|
38
|
+
Provides-Extra: pydantic-ai
|
|
39
|
+
Requires-Dist: pydantic-ai-slim>=2.55; extra == "pydantic-ai"
|
|
40
|
+
Requires-Dist: mcp<2.0,>=1.6.0; extra == "pydantic-ai"
|
|
41
|
+
Provides-Extra: multi-language
|
|
42
|
+
Requires-Dist: tree-sitter<0.26,>=0.21; extra == "multi-language"
|
|
43
|
+
Requires-Dist: tree-sitter-javascript>=0.21; extra == "multi-language"
|
|
44
|
+
Requires-Dist: tree-sitter-typescript>=0.21; extra == "multi-language"
|
|
45
|
+
Requires-Dist: tree-sitter-go>=0.21; extra == "multi-language"
|
|
46
|
+
Requires-Dist: tree-sitter-rust>=0.21; extra == "multi-language"
|
|
47
|
+
Requires-Dist: tree-sitter-java>=0.21; extra == "multi-language"
|
|
48
|
+
Dynamic: license-file
|
|
49
|
+
|
|
50
|
+
<div align="center">
|
|
51
|
+
|
|
52
|
+
# HonestCode
|
|
53
|
+
|
|
54
|
+
**The deterministic verification layer for AI coding agents.**
|
|
55
|
+
|
|
56
|
+
Catch invented APIs, undefined symbols, wrong calls, and other code hallucinations
|
|
57
|
+
before your agent moves on.
|
|
58
|
+
|
|
59
|
+

|
|
60
|
+
|
|
61
|
+
[](https://github.com/Fengrru/honestcode/actions/workflows/ci.yml)
|
|
62
|
+
[](LICENSE)
|
|
63
|
+
|
|
64
|
+
</div>
|
|
65
|
+
|
|
66
|
+
---
|
|
67
|
+
|
|
68
|
+
## The problem
|
|
69
|
+
|
|
70
|
+
AI coding agents hallucinate. They invent function names, fabricate library APIs,
|
|
71
|
+
and call methods that do not exist.
|
|
72
|
+
|
|
73
|
+
**HonestCode** is the layer that checks what the agent just wrote against the
|
|
74
|
+
repository it is supposed to be grounded in.
|
|
75
|
+
|
|
76
|
+
```text
|
|
77
|
+
Agent
|
|
78
|
+
↓
|
|
79
|
+
writes code
|
|
80
|
+
↓
|
|
81
|
+
HonestCode
|
|
82
|
+
↓
|
|
83
|
+
evidence
|
|
84
|
+
↓
|
|
85
|
+
Agent fixes
|
|
86
|
+
↓
|
|
87
|
+
verified code
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
No LLM calls. No network. Pure AST + symbol resolution.
|
|
91
|
+
|
|
92
|
+
---
|
|
93
|
+
|
|
94
|
+
## Demo
|
|
95
|
+
|
|
96
|
+
The agent was asked to *"add authentication using the existing `UserClient`."*
|
|
97
|
+
It generated:
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
client.refresh_token()
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
The real project only has:
|
|
104
|
+
|
|
105
|
+
```python
|
|
106
|
+
class UserClient:
|
|
107
|
+
def refresh(self): ...
|
|
108
|
+
def refresh_access_token(self): ...
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
HonestCode catches it deterministically:
|
|
112
|
+
|
|
113
|
+
```text
|
|
114
|
+
✗ verification failed — 1 issue in app/login_broken.py
|
|
115
|
+
|
|
116
|
+
[invented_api] app/login_broken.py:9
|
|
117
|
+
`UserClient.refresh_token()` does not exist.
|
|
118
|
+
did you mean: refresh_access_token()?
|
|
119
|
+
available methods:
|
|
120
|
+
- refresh()
|
|
121
|
+
- refresh_access_token()
|
|
122
|
+
confidence: deterministic · action: revise
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
Run it yourself:
|
|
126
|
+
|
|
127
|
+
```bash
|
|
128
|
+
cd demos/invented-api
|
|
129
|
+
python run.py
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
---
|
|
133
|
+
|
|
134
|
+
## Why not just use Ruff / Pyright?
|
|
135
|
+
|
|
136
|
+
HonestCode is not a replacement — it is a verification layer that answers a
|
|
137
|
+
specific question: *did the agent's code actually come from this repository?*
|
|
138
|
+
|
|
139
|
+
| Error | Ruff | Pyright | HonestCode |
|
|
140
|
+
|:--|:-:|:-:|:-:|
|
|
141
|
+
| Syntax error | ✓ | ✓ | ✓ |
|
|
142
|
+
| Type mismatch | — | ✓ | structural only |
|
|
143
|
+
| Undefined symbol | ✓ | ✓ | ✓ |
|
|
144
|
+
| **Invented API** | partial | partial | **core** |
|
|
145
|
+
| **Wrong method call** | — | partial | **core** |
|
|
146
|
+
| **Agent-generated API mismatch** | — | — | **core** |
|
|
147
|
+
|
|
148
|
+
The key difference is **repository grounding**. Ruff checks the file. Pyright
|
|
149
|
+
types the call. HonestCode checks whether the call resolves to a real symbol
|
|
150
|
+
that exists in the codebase the agent is editing. (HonestCode does not do full
|
|
151
|
+
type inference — its structural checks catch mismatched arities, iterating
|
|
152
|
+
`None`, and calling constants, not general type errors.)
|
|
153
|
+
|
|
154
|
+
What a call site is checked against depends on how the receiver's type is known:
|
|
155
|
+
an annotation (`client: UserClient`), a constructor assignment
|
|
156
|
+
(`client = UserClient()`), `self`/`cls`, and an inline constructor
|
|
157
|
+
(`UserClient().refresh()`) are all resolved. A receiver produced by a factory
|
|
158
|
+
function (`make_client().refresh()`) is **not** judged — function signatures
|
|
159
|
+
carry no return type in the surface model, and guessing there would trade a
|
|
160
|
+
missed finding for a false one.
|
|
161
|
+
|
|
162
|
+
---
|
|
163
|
+
|
|
164
|
+
## Install
|
|
165
|
+
|
|
166
|
+
One line — no clone, no virtualenv (once the package is on PyPI):
|
|
167
|
+
|
|
168
|
+
```bash
|
|
169
|
+
claude mcp add honestcode -- uvx honestcode-mcp
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
`uvx` fetches the server into an isolated environment on first use; install
|
|
173
|
+
[`uv`](https://docs.astral.sh/uv/) first if you do not have it. Until the
|
|
174
|
+
first PyPI release is published, install from a checkout:
|
|
175
|
+
|
|
176
|
+
```bash
|
|
177
|
+
git clone https://github.com/Fengrru/honestcode
|
|
178
|
+
cd honestcode
|
|
179
|
+
pip install -e .
|
|
180
|
+
claude mcp add honestcode -- honestcode-mcp
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
**Cursor / VS Code / Claude Desktop:** the `honestcode install` command probes
|
|
184
|
+
those clients' config files and registers the server for you:
|
|
185
|
+
|
|
186
|
+
```bash
|
|
187
|
+
honestcode install # all detected clients
|
|
188
|
+
honestcode install --client cursor --dry-run # preview without writing
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
Requires Python >= 3.10. 100% local.
|
|
192
|
+
|
|
193
|
+
---
|
|
194
|
+
|
|
195
|
+
## 30-second setup with Claude Code
|
|
196
|
+
|
|
197
|
+
```bash
|
|
198
|
+
# Add HonestCode as an MCP server
|
|
199
|
+
claude mcp add honestcode -- honestcode-mcp
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
Or add it manually to your Claude Code config (`~/.claude.json` on
|
|
203
|
+
macOS/Linux, `%USERPROFILE%\.claude.json` on Windows):
|
|
204
|
+
|
|
205
|
+
```json
|
|
206
|
+
{
|
|
207
|
+
"mcpServers": {
|
|
208
|
+
"honestcode": {
|
|
209
|
+
"command": "honestcode-mcp",
|
|
210
|
+
"args": []
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
Restart Claude Code. You now have two verification tools available:
|
|
217
|
+
|
|
218
|
+
* `scan_file(path)` — legacy tool, returns `issues` plus the new evidence shape.
|
|
219
|
+
* `verify_file(path)` — agent-facing tool, returns `status`, `findings` with
|
|
220
|
+
`evidence`, and a human-readable `text` summary.
|
|
221
|
+
|
|
222
|
+
Try prompting Claude:
|
|
223
|
+
|
|
224
|
+
```text
|
|
225
|
+
Add a login endpoint using the existing UserClient, then run verify_file on the
|
|
226
|
+
file you just wrote and fix anything it reports.
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
HonestCode auto-discovers the project root from the file path, indexes the
|
|
230
|
+
codebase, and returns grounded evidence.
|
|
231
|
+
|
|
232
|
+
### Making it automatic (no prompt discipline required)
|
|
233
|
+
|
|
234
|
+
MCP tools only run when the model chooses to call them. To make verification
|
|
235
|
+
unconditional, wire the hook in `scripts/claude_hook.py` into Claude Code's
|
|
236
|
+
PostToolUse event — every `.py` file the agent writes is verified, and a
|
|
237
|
+
finding blocks the edit with the evidence fed straight back into the
|
|
238
|
+
conversation:
|
|
239
|
+
|
|
240
|
+
```json
|
|
241
|
+
{
|
|
242
|
+
"hooks": {
|
|
243
|
+
"PostToolUse": [
|
|
244
|
+
{
|
|
245
|
+
"matcher": "Write|Edit",
|
|
246
|
+
"hooks": [
|
|
247
|
+
{
|
|
248
|
+
"type": "command",
|
|
249
|
+
"command": "python /absolute/path/to/honestcode/scripts/claude_hook.py"
|
|
250
|
+
}
|
|
251
|
+
]
|
|
252
|
+
}
|
|
253
|
+
]
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
```
|
|
257
|
+
|
|
258
|
+
Put that in `~/.claude/settings.json` (all projects) or
|
|
259
|
+
`.claude/settings.json` (one project). The hook never blocks on its own
|
|
260
|
+
failures: unreadable input, non-Python files, and verification crashes all
|
|
261
|
+
exit silently, so it can never turn into a wall that blocks every edit.
|
|
262
|
+
|
|
263
|
+
---
|
|
264
|
+
|
|
265
|
+
## Real example
|
|
266
|
+
|
|
267
|
+
```python
|
|
268
|
+
# auth/client.py
|
|
269
|
+
class UserClient:
|
|
270
|
+
def refresh_access_token(self): ...
|
|
271
|
+
|
|
272
|
+
# auth/login.py
|
|
273
|
+
from auth.client import UserClient
|
|
274
|
+
|
|
275
|
+
def login(client: UserClient):
|
|
276
|
+
client.refresh_token() # invented API
|
|
277
|
+
```
|
|
278
|
+
|
|
279
|
+
```python
|
|
280
|
+
>>> from honestcode.mcp.tools import verify_file
|
|
281
|
+
>>> verify_file("auth/login.py")
|
|
282
|
+
{
|
|
283
|
+
"status": "fail",
|
|
284
|
+
"file": "auth/login.py",
|
|
285
|
+
"findings": [
|
|
286
|
+
{
|
|
287
|
+
"line": 7,
|
|
288
|
+
"kind": "invented_api",
|
|
289
|
+
"symbol": "refresh_token",
|
|
290
|
+
"owner": "UserClient",
|
|
291
|
+
"message": "`UserClient.refresh_token()` does not exist.",
|
|
292
|
+
"evidence": {
|
|
293
|
+
"available_methods": ["refresh", "refresh_access_token"],
|
|
294
|
+
"did_you_mean": "refresh_access_token"
|
|
295
|
+
},
|
|
296
|
+
"confidence": "deterministic",
|
|
297
|
+
"action": "revise"
|
|
298
|
+
}
|
|
299
|
+
],
|
|
300
|
+
"text": "✗ verification failed — 1 issue in auth/login.py\n ..."
|
|
301
|
+
}
|
|
302
|
+
```
|
|
303
|
+
|
|
304
|
+
---
|
|
305
|
+
|
|
306
|
+
## Agent output protocol
|
|
307
|
+
|
|
308
|
+
`verify_file` returns evidence an agent can act on directly, not just an error
|
|
309
|
+
message:
|
|
310
|
+
|
|
311
|
+
```json
|
|
312
|
+
{
|
|
313
|
+
"status": "fail",
|
|
314
|
+
"file": "auth/client.py",
|
|
315
|
+
"line": 42,
|
|
316
|
+
"kind": "invented_api",
|
|
317
|
+
"symbol": "refresh_token",
|
|
318
|
+
"owner": "UserClient",
|
|
319
|
+
"message": "UserClient.refresh_token() does not exist.",
|
|
320
|
+
"evidence": {
|
|
321
|
+
"available_methods": ["refresh", "refresh_access_token"]
|
|
322
|
+
},
|
|
323
|
+
"confidence": "deterministic",
|
|
324
|
+
"action": "revise"
|
|
325
|
+
}
|
|
326
|
+
```
|
|
327
|
+
|
|
328
|
+
---
|
|
329
|
+
|
|
330
|
+
## How it works
|
|
331
|
+
|
|
332
|
+
1. **Auto-index** the project (or reuse the cached symbol index).
|
|
333
|
+
2. **Parse** the target file with the standard-library `ast` module.
|
|
334
|
+
3. **Resolve every call site** to a concrete symbol:
|
|
335
|
+
- infer the receiver type from annotations, constructors, and imports;
|
|
336
|
+
- reconstruct the class's member surface from the repository AST;
|
|
337
|
+
- mark the surface as *unknown* when a base class cannot be resolved or the
|
|
338
|
+
class defines `__getattr__`.
|
|
339
|
+
4. **Emit findings** with `kind`, `owner`, `evidence`, `confidence`, and `action`.
|
|
340
|
+
|
|
341
|
+
The loop is deterministic and auditable.
|
|
342
|
+
|
|
343
|
+
### Scope and limitations
|
|
344
|
+
|
|
345
|
+
The verifier stays silent wherever it cannot *prove* absence — that is the
|
|
346
|
+
feature, but it means detection has known boundaries:
|
|
347
|
+
|
|
348
|
+
* **Decorated classes are not judged for invented members.** Any class
|
|
349
|
+
decorator (including `@dataclass`) may add or rewrite members, so invented
|
|
350
|
+
method calls on such classes are not reported. Undecorated classes are fully
|
|
351
|
+
checked.
|
|
352
|
+
* **Standard-library member calls are not checked.** `from typing import
|
|
353
|
+
TypeVarr` (an invented *import*) is caught, but `os.path.joinn()` (an
|
|
354
|
+
invented *call* on a real module) is not — third-party and project members
|
|
355
|
+
are the covered surface.
|
|
356
|
+
* **Factory-function receivers are not judged.** `make_client().refresh()` is
|
|
357
|
+
skipped; annotations, constructor assignments, `self`/`cls`, and inline
|
|
358
|
+
constructors (`UserClient().refresh()`) are resolved.
|
|
359
|
+
* **`undefined_symbol` carries `confidence: high`, not `deterministic`.**
|
|
360
|
+
Only member-surface claims (invented APIs on a fully enumerated class) are
|
|
361
|
+
provable from the repository; name absence rests on the syntactic binding
|
|
362
|
+
model and keeps the lower confidence. Variables reassigned across types in
|
|
363
|
+
one scope are treated as unknown and never judged.
|
|
364
|
+
* **Graph tools are anchored to the last indexed project.** `scan_file` /
|
|
365
|
+
`verify_file` resolve the project root per file (including monorepo
|
|
366
|
+
sub-projects); `find_dead_code`, `explore_call_graph`, `search_code` and
|
|
367
|
+
friends operate on the project most recently passed to `index_project`.
|
|
368
|
+
* **`check_api` imports the base package on demand** — but only when it is
|
|
369
|
+
genuinely installed (or stdlib). A bare file dropped into the workspace is
|
|
370
|
+
never imported, so agent-authored code cannot be executed server-side
|
|
371
|
+
through this tool.
|
|
372
|
+
|
|
373
|
+
---
|
|
374
|
+
|
|
375
|
+
## MCP tools
|
|
376
|
+
|
|
377
|
+
By default `scan_file` and `verify_file` are exposed. Set `HONESTCODE_TOOLS=all`
|
|
378
|
+
to enable the full set, including:
|
|
379
|
+
|
|
380
|
+
| Tool | Purpose |
|
|
381
|
+
|:--|:--|
|
|
382
|
+
| `scan_file` | Scan a file for invented APIs, undefined calls, and wrong arities. |
|
|
383
|
+
| `verify_file` | Return the agent-facing structured evidence protocol. |
|
|
384
|
+
| `index_project` | Build or reuse the project symbol index. |
|
|
385
|
+
| `load_project_deps` | Load dependency APIs from `requirements.txt` / `pyproject.toml`. |
|
|
386
|
+
| `check_symbol` | Verify a symbol is defined. |
|
|
387
|
+
| `check_api` | Verify a library API call exists. |
|
|
388
|
+
| `validate_types` | Structural type checks. |
|
|
389
|
+
|
|
390
|
+
Dependency manifests (`requirements.txt`, `pyproject.toml`) are declared
|
|
391
|
+
automatically on the first scan, so imports from a dependency that is not
|
|
392
|
+
installed in the MCP server's interpreter are trusted rather than reported as
|
|
393
|
+
invented. Set `HONESTCODE_AUTO_DEPS=0` to disable that.
|
|
394
|
+
|
|
395
|
+
See the old tool reference below for the complete list.
|
|
396
|
+
|
|
397
|
+
---
|
|
398
|
+
|
|
399
|
+
## Crash-safe side effects
|
|
400
|
+
|
|
401
|
+
`verify_file` checks code before it runs. `guards/` covers what happens after:
|
|
402
|
+
an agent that retries a tool call can charge a card twice, and no durable
|
|
403
|
+
execution engine prevents it — Temporal and Restate both guarantee deterministic
|
|
404
|
+
replay of *orchestration* and leave exactly-once external effects to the
|
|
405
|
+
application.
|
|
406
|
+
|
|
407
|
+
```python
|
|
408
|
+
from honestcode.guards import EffectStore, SideEffectGuard, durable_scope
|
|
409
|
+
|
|
410
|
+
guard = SideEffectGuard(session_id="run-42", store_path=".agent/effects.db")
|
|
411
|
+
|
|
412
|
+
outcome = guard.run(
|
|
413
|
+
"charge",
|
|
414
|
+
target="cus_123",
|
|
415
|
+
payload={"cents": 4200},
|
|
416
|
+
execute=lambda: charge_card("cus_123", 4200),
|
|
417
|
+
approve_irreversible=True,
|
|
418
|
+
scope=durable_scope("order-4711", "charge", "cus_123"),
|
|
419
|
+
)
|
|
420
|
+
|
|
421
|
+
outcome.executed # False on the retry — a receipt, not a second charge
|
|
422
|
+
outcome.effect_id # stable across a restart, because the key is on disk
|
|
423
|
+
```
|
|
424
|
+
|
|
425
|
+
The interesting part is what `scope` does. Pass it and the effect's identity
|
|
426
|
+
comes from the scope rather than the session — which is the difference between a
|
|
427
|
+
retry being recognised and a customer being charged twice, because a restart *is*
|
|
428
|
+
a new session id. Pass nothing and the guard stays session-scoped.
|
|
429
|
+
|
|
430
|
+
Refusing is the default direction: an unrecognised action is treated as
|
|
431
|
+
irreversible rather than idempotent, and irreversible effects need explicit
|
|
432
|
+
approval. A run that dies between the call and the record leaves the effect
|
|
433
|
+
`executing`, and the next attempt reports the outcome as unknown instead of
|
|
434
|
+
guessing — which is what stops the retry. Scoped effects are recovered with
|
|
435
|
+
`guard.recover(scope=durable_scope(...))`, and `guard.effects(scope=...)` /
|
|
436
|
+
`guard.summary(scope=...)` report them; the records live under a namespace
|
|
437
|
+
derived from the scope so a restart can still find them.
|
|
438
|
+
|
|
439
|
+
For a framework, `integrations/` binds the guard to pydantic-ai, where a
|
|
440
|
+
committed effect replaces the tool result instead of re-running the tool. See
|
|
441
|
+
[`honestcode/integrations/README.md`](honestcode/integrations/README.md).
|
|
442
|
+
|
|
443
|
+
---
|
|
444
|
+
|
|
445
|
+
## Benchmark
|
|
446
|
+
|
|
447
|
+
HonestCode includes four benchmark suites that run automatically in CI on every
|
|
448
|
+
push and pull request to `main`:
|
|
449
|
+
|
|
450
|
+
### Agent-accuracy benchmark
|
|
451
|
+
|
|
452
|
+
`benchmarks/agent_accuracy/` is a deterministic, LLM-free benchmark that
|
|
453
|
+
measures how well HonestCode catches common agent hallucinations (**recall**).
|
|
454
|
+
|
|
455
|
+
Each task is a tiny agent episode: the agent writes a broken file, HonestCode
|
|
456
|
+
verifies it, then the file is replaced with the fix and verified again.
|
|
457
|
+
|
|
458
|
+
```bash
|
|
459
|
+
cd benchmarks/agent_accuracy
|
|
460
|
+
python run.py
|
|
461
|
+
```
|
|
462
|
+
|
|
463
|
+
Current results (6 tasks, deterministic verification):
|
|
464
|
+
|
|
465
|
+
| task | expected issue | broken detected | fixed clean | broken ms | fixed ms |
|
|
466
|
+
|---|---|---|---|---|---|
|
|
467
|
+
| invented_method | invented_api (`UserClient.refresh_token`) | yes | yes | 4.18 | 3.78 |
|
|
468
|
+
| invented_module_attr | invented_api (`Connection.query`) | yes | yes | 4.19 | 3.95 |
|
|
469
|
+
| undefined_import | undefined_symbol (`delete_user`) | yes | yes | 2.69 | 3.55 |
|
|
470
|
+
| wrong_signature | wrong_call (`add()`) | yes | yes | 3.56 | 3.35 |
|
|
471
|
+
| cross_file_new_symbol | invented_api (`fly`) | yes | yes | 3.85 | 2.45 |
|
|
472
|
+
| dep_alias_and_pypi_name | invented_api (`toll`) | yes | yes | 3.44 | 2.9 |
|
|
473
|
+
|
|
474
|
+
**Summary:** precision 1.0, recall 1.0, F1 1.0, false-positive rate 0.0, median
|
|
475
|
+
verify time 4–7 ms on a laptop.
|
|
476
|
+
|
|
477
|
+
The per-task milliseconds above are one run's output, not a guarantee — they
|
|
478
|
+
move a few tens of percent between runs on the same machine. What is stable, and
|
|
479
|
+
what the benchmark is gating on, is the first four columns plus the summary's
|
|
480
|
+
counts.
|
|
481
|
+
|
|
482
|
+
The summary line is `python benchmarks/agent_accuracy/run.py` verbatim rather
|
|
483
|
+
than a figure copied into this file, because a timing quoted here goes stale the
|
|
484
|
+
moment the code gets faster — as one did. Reproduce it with `--format markdown`.
|
|
485
|
+
|
|
486
|
+
The `cross_file_new_symbol` task pins the agent-session flow (write a module,
|
|
487
|
+
then a file importing it — the index must be fresh, never stale); the
|
|
488
|
+
`dep_alias_and_pypi_name` task pins dependency grounding through aliased
|
|
489
|
+
submodules and PyPI-named requirements (`PyYAML` → `yaml`).
|
|
490
|
+
|
|
491
|
+
### Real-world false-positive benchmark
|
|
492
|
+
|
|
493
|
+
`benchmarks/realworld_fp/` measures **precision on real code**: it scans a
|
|
494
|
+
vendored snapshot of `psf/requests` (19 files, ~6.4k LOC, Apache-2.0) and
|
|
495
|
+
asserts HonestCode reports **zero** findings. Real code exercises the
|
|
496
|
+
constructs synthetic tasks never do — stdlib imports, keyword-call style,
|
|
497
|
+
`@staticmethod` members, classes inheriting stdlib ABCs, compat re-export
|
|
498
|
+
modules — and any finding on this corpus is, by construction, a false
|
|
499
|
+
positive.
|
|
500
|
+
|
|
501
|
+
The same run also scans `benchmarks/realworld_fp/regressions/`: one tiny
|
|
502
|
+
project per false-positive class found in the wild (a receiver rebound across
|
|
503
|
+
types, `with ... as` bindings, monorepo sub-project imports, local binding
|
|
504
|
+
forms). Each scenario fails the benchmark by name if the regression returns.
|
|
505
|
+
|
|
506
|
+
```bash
|
|
507
|
+
python benchmarks/realworld_fp/run.py
|
|
508
|
+
# files: 25 findings: 0 elapsed: 0.7s (~26 ms/file)
|
|
509
|
+
```
|
|
510
|
+
|
|
511
|
+
The finding count is the claim; the elapsed time is indicative, and varies with
|
|
512
|
+
the machine.
|
|
513
|
+
|
|
514
|
+
### Value-drift benchmark
|
|
515
|
+
|
|
516
|
+
`benchmarks/value_drift/` asks a different question: when a formatting-tolerant
|
|
517
|
+
edit matcher resolves a search string, did the edit land on the region the
|
|
518
|
+
caller named, or silently on a near-miss sibling that carries a different value?
|
|
519
|
+
It runs ten scenarios across a guarded matcher cascade and an unguarded one, and
|
|
520
|
+
classifies each edit as `correct`, `silent_drift`, `no_match`, or `wrong_span`.
|
|
521
|
+
The guarded matcher must produce the intended edit on every scenario; a
|
|
522
|
+
`no_match` is a refusal, not a pass.
|
|
523
|
+
|
|
524
|
+
```bash
|
|
525
|
+
python benchmarks/value_drift/run.py # human-readable
|
|
526
|
+
python benchmarks/value_drift/run.py --json # machine-readable (status on stderr)
|
|
527
|
+
```
|
|
528
|
+
|
|
529
|
+
### Performance benchmark
|
|
530
|
+
|
|
531
|
+
`scripts/benchmark.py` measures the latency of core operations (index, scan,
|
|
532
|
+
graph, dead code detection, similarity search) on the repository itself.
|
|
533
|
+
|
|
534
|
+
```bash
|
|
535
|
+
python scripts/benchmark.py # text output
|
|
536
|
+
python scripts/benchmark.py --format markdown # table for README
|
|
537
|
+
python scripts/benchmark.py --repo psf/requests # benchmark a real-world repo
|
|
538
|
+
```
|
|
539
|
+
|
|
540
|
+
All four benchmarks run in CI (`.github/workflows/ci.yml`) as separate jobs:
|
|
541
|
+
`benchmark-accuracy`, `benchmark-realworld`, `benchmark-value-drift`, and
|
|
542
|
+
`benchmark-performance`. The accuracy job gates on precision and recall staying
|
|
543
|
+
at 1.0, the real-world job on any finding or failed scan, and the value-drift
|
|
544
|
+
job on every guarded scenario landing correctly. The performance job reports
|
|
545
|
+
numbers rather than gating on them.
|
|
546
|
+
|
|
547
|
+
See `benchmarks/agent_accuracy/README.md` for the dataset format and how to
|
|
548
|
+
add new accuracy tasks.
|
|
549
|
+
|
|
550
|
+
---
|
|
551
|
+
|
|
552
|
+
## Architecture
|
|
553
|
+
|
|
554
|
+
```
|
|
555
|
+
honestcode/
|
|
556
|
+
├── verify/ # Evidence protocol + repository-grounded verification
|
|
557
|
+
├── mcp/ # MCP server layer
|
|
558
|
+
├── honest/ # Symbol index + project binding
|
|
559
|
+
├── graph/ # Persistent call graph (SQLite)
|
|
560
|
+
├── structure/ # AST extraction
|
|
561
|
+
├── sandbox/ # Sandboxed execution
|
|
562
|
+
├── guards/ # Crash-safe side effects (stdlib-only, extractable)
|
|
563
|
+
├── integrations/ # Framework bindings — pydantic-ai today
|
|
564
|
+
└── cli.py # Command-line interface
|
|
565
|
+
```
|
|
566
|
+
|
|
567
|
+
`verify_file` is the agent interface. `scan_file` is exposed alongside it by
|
|
568
|
+
default and returns both the new evidence shape and the legacy `issues` list.
|
|
569
|
+
|
|
570
|
+
Two of those directories answer a different question from the rest. Everything
|
|
571
|
+
above `guards/` asks *is this code consistent with the repository*. `guards/`
|
|
572
|
+
asks *did this effect already happen* — an agent that retries a tool call after
|
|
573
|
+
a crash can charge a card twice, and durable-execution engines leave exactly-once
|
|
574
|
+
external effects to the application. `integrations/` binds the guard to a
|
|
575
|
+
framework; pydantic-ai's `wrap_tool_execute` is the first, and a tool result is
|
|
576
|
+
replaced rather than re-run when the effect has already been committed.
|
|
577
|
+
|
|
578
|
+
`guards/` is deliberately stdlib-only and does not depend on this package's
|
|
579
|
+
protocols, so it can be extracted into its own distribution. `tests/test_guards.py`
|
|
580
|
+
enforces that with an AST import check.
|
|
581
|
+
|
|
582
|
+
---
|
|
583
|
+
|
|
584
|
+
## Roadmap
|
|
585
|
+
|
|
586
|
+
### v0.1 — Repository grounding (done)
|
|
587
|
+
- symbol / API / function / class / method / import / call verification
|
|
588
|
+
- invented API detection with structured evidence
|
|
589
|
+
- auto-index on `scan_file`
|
|
590
|
+
|
|
591
|
+
### Beyond the roadmap — crash-safe side effects (now)
|
|
592
|
+
Not a verification layer, so it has no place in the v0.2/v0.3 sequence below:
|
|
593
|
+
`guards/` makes an effect idempotent across a crash, and `integrations/` binds
|
|
594
|
+
it to pydantic-ai. It answers "did this already happen", which is a runtime
|
|
595
|
+
question rather than a static one.
|
|
596
|
+
|
|
597
|
+
### v0.2 — Semantic contract verification
|
|
598
|
+
- function expects `UserID`, agent passes `User` → suspicious
|
|
599
|
+
|
|
600
|
+
### v0.3 — Execution verification
|
|
601
|
+
- static verification → tests → runtime evidence
|
|
602
|
+
|
|
603
|
+
### v1.0 — Verification Runtime for Coding Agents
|
|
604
|
+
```text
|
|
605
|
+
Coding Agent
|
|
606
|
+
│
|
|
607
|
+
┌───────▼───────┐
|
|
608
|
+
│ HonestCode │
|
|
609
|
+
│ Verification │
|
|
610
|
+
│ Runtime │
|
|
611
|
+
└───────┬───────┘
|
|
612
|
+
│
|
|
613
|
+
┌──────────────┼──────────────┐
|
|
614
|
+
↓ ↓ ↓
|
|
615
|
+
Static Semantic Runtime
|
|
616
|
+
Verification Verification Verification
|
|
617
|
+
│ │ │
|
|
618
|
+
└──────────────┼──────────────┘
|
|
619
|
+
↓
|
|
620
|
+
Evidence
|
|
621
|
+
↓
|
|
622
|
+
Agent
|
|
623
|
+
```
|
|
624
|
+
|
|
625
|
+
---
|
|
626
|
+
|
|
627
|
+
## Development
|
|
628
|
+
|
|
629
|
+
```bash
|
|
630
|
+
git clone https://github.com/Fengrru/honestcode.git
|
|
631
|
+
cd honestcode
|
|
632
|
+
python -m venv .venv
|
|
633
|
+
.venv\Scripts\activate # Windows
|
|
634
|
+
# source .venv/bin/activate # macOS/Linux
|
|
635
|
+
pip install -e ".[dev]"
|
|
636
|
+
pytest
|
|
637
|
+
ruff check honestcode tests scripts
|
|
638
|
+
```
|
|
639
|
+
|
|
640
|
+
Run benchmarks locally:
|
|
641
|
+
|
|
642
|
+
```bash
|
|
643
|
+
# Accuracy benchmark
|
|
644
|
+
python benchmarks/agent_accuracy/run.py
|
|
645
|
+
|
|
646
|
+
# Performance benchmark
|
|
647
|
+
python scripts/benchmark.py
|
|
648
|
+
```
|
|
649
|
+
|
|
650
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for pull request guidelines.
|
|
651
|
+
|
|
652
|
+
---
|
|
653
|
+
|
|
654
|
+
## Security
|
|
655
|
+
|
|
656
|
+
See [SECURITY.md](SECURITY.md) for vulnerability reporting.
|
|
657
|
+
|
|
658
|
+
---
|
|
659
|
+
|
|
660
|
+
## Telemetry
|
|
661
|
+
|
|
662
|
+
**HonestCode collects no telemetry.** There are no analytics libraries,
|
|
663
|
+
no background services, and no phone-home endpoints.
|
|
664
|
+
|
|
665
|
+
---
|
|
666
|
+
|
|
667
|
+
## License
|
|
668
|
+
|
|
669
|
+
[MIT](LICENSE) - Copyright (c) 2026 HonestCode Team
|