honestcode 0.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. honestcode-0.5.0/LICENSE +21 -0
  2. honestcode-0.5.0/PKG-INFO +669 -0
  3. honestcode-0.5.0/README.md +620 -0
  4. honestcode-0.5.0/honestcode/__init__.py +19 -0
  5. honestcode-0.5.0/honestcode/cli.py +428 -0
  6. honestcode-0.5.0/honestcode/graph/__init__.py +13 -0
  7. honestcode-0.5.0/honestcode/graph/graph_store.py +276 -0
  8. honestcode-0.5.0/honestcode/graph/watcher.py +116 -0
  9. honestcode-0.5.0/honestcode/guards/__init__.py +104 -0
  10. honestcode-0.5.0/honestcode/guards/guard.py +777 -0
  11. honestcode-0.5.0/honestcode/guards/store.py +457 -0
  12. honestcode-0.5.0/honestcode/guards/taxonomy.py +228 -0
  13. honestcode-0.5.0/honestcode/honest/__init__.py +19 -0
  14. honestcode-0.5.0/honestcode/honest/project_binding.py +200 -0
  15. honestcode-0.5.0/honestcode/honest/router.py +205 -0
  16. honestcode-0.5.0/honestcode/honest/symbol_index.py +446 -0
  17. honestcode-0.5.0/honestcode/integrations/__init__.py +14 -0
  18. honestcode-0.5.0/honestcode/integrations/pydantic_ai.py +304 -0
  19. honestcode-0.5.0/honestcode/mcp/__init__.py +46 -0
  20. honestcode-0.5.0/honestcode/mcp/knowledge_base.py +290 -0
  21. honestcode-0.5.0/honestcode/mcp/server.py +357 -0
  22. honestcode-0.5.0/honestcode/mcp/tools.py +2002 -0
  23. honestcode-0.5.0/honestcode/py.typed +0 -0
  24. honestcode-0.5.0/honestcode/sandbox/__init__.py +334 -0
  25. honestcode-0.5.0/honestcode/structure/__init__.py +6 -0
  26. honestcode-0.5.0/honestcode/structure/extractor.py +353 -0
  27. honestcode-0.5.0/honestcode/structure/multi_lang.py +267 -0
  28. honestcode-0.5.0/honestcode/structure/relations.py +47 -0
  29. honestcode-0.5.0/honestcode/structure/utils.py +262 -0
  30. honestcode-0.5.0/honestcode/verify/__init__.py +19 -0
  31. honestcode-0.5.0/honestcode/verify/evidence.py +195 -0
  32. honestcode-0.5.0/honestcode/verify/surface.py +497 -0
  33. honestcode-0.5.0/honestcode/verify/verify.py +1074 -0
  34. honestcode-0.5.0/honestcode.egg-info/PKG-INFO +669 -0
  35. honestcode-0.5.0/honestcode.egg-info/SOURCES.txt +55 -0
  36. honestcode-0.5.0/honestcode.egg-info/dependency_links.txt +1 -0
  37. honestcode-0.5.0/honestcode.egg-info/entry_points.txt +3 -0
  38. honestcode-0.5.0/honestcode.egg-info/requires.txt +24 -0
  39. honestcode-0.5.0/honestcode.egg-info/top_level.txt +1 -0
  40. honestcode-0.5.0/pyproject.toml +137 -0
  41. honestcode-0.5.0/setup.cfg +4 -0
  42. honestcode-0.5.0/tests/test_advanced.py +193 -0
  43. honestcode-0.5.0/tests/test_benchmarks.py +60 -0
  44. honestcode-0.5.0/tests/test_claude_hook.py +86 -0
  45. honestcode-0.5.0/tests/test_cli.py +206 -0
  46. honestcode-0.5.0/tests/test_encoding.py +106 -0
  47. honestcode-0.5.0/tests/test_false_positives.py +393 -0
  48. honestcode-0.5.0/tests/test_fixes.py +669 -0
  49. honestcode-0.5.0/tests/test_graph.py +306 -0
  50. honestcode-0.5.0/tests/test_guards.py +1249 -0
  51. honestcode-0.5.0/tests/test_hardening.py +565 -0
  52. honestcode-0.5.0/tests/test_more.py +196 -0
  53. honestcode-0.5.0/tests/test_pydantic_ai_integration.py +345 -0
  54. honestcode-0.5.0/tests/test_server.py +170 -0
  55. honestcode-0.5.0/tests/test_tools.py +142 -0
  56. honestcode-0.5.0/tests/test_value_drift.py +231 -0
  57. honestcode-0.5.0/tests/test_verify.py +263 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 HonestCode Team
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,669 @@
1
+ Metadata-Version: 2.4
2
+ Name: honestcode
3
+ Version: 0.5.0
4
+ Summary: The deterministic verification layer for AI coding agents. Catch invented APIs, undefined symbols, and wrong calls before your agent moves on.
5
+ Author: HonestCode Team
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/Fengrru/honestcode
8
+ Project-URL: Documentation, https://github.com/Fengrru/honestcode#readme
9
+ Project-URL: Repository, https://github.com/Fengrru/honestcode
10
+ Project-URL: Issues, https://github.com/Fengrru/honestcode/issues
11
+ Keywords: mcp,model-context-protocol,verification,ai-coding-agent,hallucination-detection,invented-api,code-generation,static-analysis,llm,ai,code-analysis,agent-tooling
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: Operating System :: OS Independent
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.10
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Programming Language :: Python :: 3.13
20
+ Classifier: Programming Language :: Python :: 3.14
21
+ Classifier: Topic :: Software Development :: Code Generators
22
+ Classifier: Topic :: Software Development :: Quality Assurance
23
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
24
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
25
+ Classifier: Typing :: Typed
26
+ Requires-Python: >=3.10
27
+ Description-Content-Type: text/markdown
28
+ License-File: LICENSE
29
+ Requires-Dist: mcp<2.0,>=1.6.0
30
+ Requires-Dist: pyyaml>=6.0
31
+ Requires-Dist: tomli>=2.0.0; python_version < "3.11"
32
+ Provides-Extra: dev
33
+ Requires-Dist: pytest>=8.0; extra == "dev"
34
+ Requires-Dist: pytest-asyncio>=0.24; extra == "dev"
35
+ Requires-Dist: pytest-cov>=4.0; extra == "dev"
36
+ Requires-Dist: ruff>=0.6; extra == "dev"
37
+ Requires-Dist: pre-commit>=3.0; extra == "dev"
38
+ Provides-Extra: pydantic-ai
39
+ Requires-Dist: pydantic-ai-slim>=2.55; extra == "pydantic-ai"
40
+ Requires-Dist: mcp<2.0,>=1.6.0; extra == "pydantic-ai"
41
+ Provides-Extra: multi-language
42
+ Requires-Dist: tree-sitter<0.26,>=0.21; extra == "multi-language"
43
+ Requires-Dist: tree-sitter-javascript>=0.21; extra == "multi-language"
44
+ Requires-Dist: tree-sitter-typescript>=0.21; extra == "multi-language"
45
+ Requires-Dist: tree-sitter-go>=0.21; extra == "multi-language"
46
+ Requires-Dist: tree-sitter-rust>=0.21; extra == "multi-language"
47
+ Requires-Dist: tree-sitter-java>=0.21; extra == "multi-language"
48
+ Dynamic: license-file
49
+
50
+ <div align="center">
51
+
52
+ # HonestCode
53
+
54
+ **The deterministic verification layer for AI coding agents.**
55
+
56
+ Catch invented APIs, undefined symbols, wrong calls, and other code hallucinations
57
+ before your agent moves on.
58
+
59
+ ![HonestCode invented API demo](demos/invented-api/demo.gif)
60
+
61
+ [![CI](https://github.com/Fengrru/honestcode/actions/workflows/ci.yml/badge.svg)](https://github.com/Fengrru/honestcode/actions/workflows/ci.yml)
62
+ [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)
63
+
64
+ </div>
65
+
66
+ ---
67
+
68
+ ## The problem
69
+
70
+ AI coding agents hallucinate. They invent function names, fabricate library APIs,
71
+ and call methods that do not exist.
72
+
73
+ **HonestCode** is the layer that checks what the agent just wrote against the
74
+ repository it is supposed to be grounded in.
75
+
76
+ ```text
77
+ Agent
78
+ ↓
79
+ writes code
80
+ ↓
81
+ HonestCode
82
+ ↓
83
+ evidence
84
+ ↓
85
+ Agent fixes
86
+ ↓
87
+ verified code
88
+ ```
89
+
90
+ No LLM calls. No network. Pure AST + symbol resolution.
91
+
92
+ ---
93
+
94
+ ## Demo
95
+
96
+ The agent was asked to *"add authentication using the existing `UserClient`."*
97
+ It generated:
98
+
99
+ ```python
100
+ client.refresh_token()
101
+ ```
102
+
103
+ The real project only has:
104
+
105
+ ```python
106
+ class UserClient:
107
+ def refresh(self): ...
108
+ def refresh_access_token(self): ...
109
+ ```
110
+
111
+ HonestCode catches it deterministically:
112
+
113
+ ```text
114
+ ✗ verification failed — 1 issue in app/login_broken.py
115
+
116
+ [invented_api] app/login_broken.py:9
117
+ `UserClient.refresh_token()` does not exist.
118
+ did you mean: refresh_access_token()?
119
+ available methods:
120
+ - refresh()
121
+ - refresh_access_token()
122
+ confidence: deterministic · action: revise
123
+ ```
124
+
125
+ Run it yourself:
126
+
127
+ ```bash
128
+ cd demos/invented-api
129
+ python run.py
130
+ ```
131
+
132
+ ---
133
+
134
+ ## Why not just use Ruff / Pyright?
135
+
136
+ HonestCode is not a replacement — it is a verification layer that answers a
137
+ specific question: *did the agent's code actually come from this repository?*
138
+
139
+ | Error | Ruff | Pyright | HonestCode |
140
+ |:--|:-:|:-:|:-:|
141
+ | Syntax error | ✓ | ✓ | ✓ |
142
+ | Type mismatch | — | ✓ | structural only |
143
+ | Undefined symbol | ✓ | ✓ | ✓ |
144
+ | **Invented API** | partial | partial | **core** |
145
+ | **Wrong method call** | — | partial | **core** |
146
+ | **Agent-generated API mismatch** | — | — | **core** |
147
+
148
+ The key difference is **repository grounding**. Ruff checks the file. Pyright
149
+ types the call. HonestCode checks whether the call resolves to a real symbol
150
+ that exists in the codebase the agent is editing. (HonestCode does not do full
151
+ type inference — its structural checks catch mismatched arities, iterating
152
+ `None`, and calling constants, not general type errors.)
153
+
154
+ What a call site is checked against depends on how the receiver's type is known:
155
+ an annotation (`client: UserClient`), a constructor assignment
156
+ (`client = UserClient()`), `self`/`cls`, and an inline constructor
157
+ (`UserClient().refresh()`) are all resolved. A receiver produced by a factory
158
+ function (`make_client().refresh()`) is **not** judged — function signatures
159
+ carry no return type in the surface model, and guessing there would trade a
160
+ missed finding for a false one.
161
+
162
+ ---
163
+
164
+ ## Install
165
+
166
+ One line — no clone, no virtualenv (once the package is on PyPI):
167
+
168
+ ```bash
169
+ claude mcp add honestcode -- uvx honestcode-mcp
170
+ ```
171
+
172
+ `uvx` fetches the server into an isolated environment on first use; install
173
+ [`uv`](https://docs.astral.sh/uv/) first if you do not have it. Until the
174
+ first PyPI release is published, install from a checkout:
175
+
176
+ ```bash
177
+ git clone https://github.com/Fengrru/honestcode
178
+ cd honestcode
179
+ pip install -e .
180
+ claude mcp add honestcode -- honestcode-mcp
181
+ ```
182
+
183
+ **Cursor / VS Code / Claude Desktop:** the `honestcode install` command probes
184
+ those clients' config files and registers the server for you:
185
+
186
+ ```bash
187
+ honestcode install # all detected clients
188
+ honestcode install --client cursor --dry-run # preview without writing
189
+ ```
190
+
191
+ Requires Python >= 3.10. 100% local.
192
+
193
+ ---
194
+
195
+ ## 30-second setup with Claude Code
196
+
197
+ ```bash
198
+ # Add HonestCode as an MCP server
199
+ claude mcp add honestcode -- honestcode-mcp
200
+ ```
201
+
202
+ Or add it manually to your Claude Code config (`~/.claude.json` on
203
+ macOS/Linux, `%USERPROFILE%\.claude.json` on Windows):
204
+
205
+ ```json
206
+ {
207
+ "mcpServers": {
208
+ "honestcode": {
209
+ "command": "honestcode-mcp",
210
+ "args": []
211
+ }
212
+ }
213
+ }
214
+ ```
215
+
216
+ Restart Claude Code. You now have two verification tools available:
217
+
218
+ * `scan_file(path)` — legacy tool, returns `issues` plus the new evidence shape.
219
+ * `verify_file(path)` — agent-facing tool, returns `status`, `findings` with
220
+ `evidence`, and a human-readable `text` summary.
221
+
222
+ Try prompting Claude:
223
+
224
+ ```text
225
+ Add a login endpoint using the existing UserClient, then run verify_file on the
226
+ file you just wrote and fix anything it reports.
227
+ ```
228
+
229
+ HonestCode auto-discovers the project root from the file path, indexes the
230
+ codebase, and returns grounded evidence.
231
+
232
+ ### Making it automatic (no prompt discipline required)
233
+
234
+ MCP tools only run when the model chooses to call them. To make verification
235
+ unconditional, wire the hook in `scripts/claude_hook.py` into Claude Code's
236
+ PostToolUse event — every `.py` file the agent writes is verified, and a
237
+ finding blocks the edit with the evidence fed straight back into the
238
+ conversation:
239
+
240
+ ```json
241
+ {
242
+ "hooks": {
243
+ "PostToolUse": [
244
+ {
245
+ "matcher": "Write|Edit",
246
+ "hooks": [
247
+ {
248
+ "type": "command",
249
+ "command": "python /absolute/path/to/honestcode/scripts/claude_hook.py"
250
+ }
251
+ ]
252
+ }
253
+ ]
254
+ }
255
+ }
256
+ ```
257
+
258
+ Put that in `~/.claude/settings.json` (all projects) or
259
+ `.claude/settings.json` (one project). The hook never blocks on its own
260
+ failures: unreadable input, non-Python files, and verification crashes all
261
+ exit silently, so it can never turn into a wall that blocks every edit.
262
+
263
+ ---
264
+
265
+ ## Real example
266
+
267
+ ```python
268
+ # auth/client.py
269
+ class UserClient:
270
+ def refresh_access_token(self): ...
271
+
272
+ # auth/login.py
273
+ from auth.client import UserClient
274
+
275
+ def login(client: UserClient):
276
+ client.refresh_token() # invented API
277
+ ```
278
+
279
+ ```python
280
+ >>> from honestcode.mcp.tools import verify_file
281
+ >>> verify_file("auth/login.py")
282
+ {
283
+ "status": "fail",
284
+ "file": "auth/login.py",
285
+ "findings": [
286
+ {
287
+ "line": 7,
288
+ "kind": "invented_api",
289
+ "symbol": "refresh_token",
290
+ "owner": "UserClient",
291
+ "message": "`UserClient.refresh_token()` does not exist.",
292
+ "evidence": {
293
+ "available_methods": ["refresh", "refresh_access_token"],
294
+ "did_you_mean": "refresh_access_token"
295
+ },
296
+ "confidence": "deterministic",
297
+ "action": "revise"
298
+ }
299
+ ],
300
+ "text": "✗ verification failed — 1 issue in auth/login.py\n ..."
301
+ }
302
+ ```
303
+
304
+ ---
305
+
306
+ ## Agent output protocol
307
+
308
+ `verify_file` returns evidence an agent can act on directly, not just an error
309
+ message:
310
+
311
+ ```json
312
+ {
313
+ "status": "fail",
314
+ "file": "auth/client.py",
315
+ "line": 42,
316
+ "kind": "invented_api",
317
+ "symbol": "refresh_token",
318
+ "owner": "UserClient",
319
+ "message": "UserClient.refresh_token() does not exist.",
320
+ "evidence": {
321
+ "available_methods": ["refresh", "refresh_access_token"]
322
+ },
323
+ "confidence": "deterministic",
324
+ "action": "revise"
325
+ }
326
+ ```
327
+
328
+ ---
329
+
330
+ ## How it works
331
+
332
+ 1. **Auto-index** the project (or reuse the cached symbol index).
333
+ 2. **Parse** the target file with the standard-library `ast` module.
334
+ 3. **Resolve every call site** to a concrete symbol:
335
+ - infer the receiver type from annotations, constructors, and imports;
336
+ - reconstruct the class's member surface from the repository AST;
337
+ - mark the surface as *unknown* when a base class cannot be resolved or the
338
+ class defines `__getattr__`.
339
+ 4. **Emit findings** with `kind`, `owner`, `evidence`, `confidence`, and `action`.
340
+
341
+ The loop is deterministic and auditable.
342
+
343
+ ### Scope and limitations
344
+
345
+ The verifier stays silent wherever it cannot *prove* absence — that is the
346
+ feature, but it means detection has known boundaries:
347
+
348
+ * **Decorated classes are not judged for invented members.** Any class
349
+ decorator (including `@dataclass`) may add or rewrite members, so invented
350
+ method calls on such classes are not reported. Undecorated classes are fully
351
+ checked.
352
+ * **Standard-library member calls are not checked.** `from typing import
353
+ TypeVarr` (an invented *import*) is caught, but `os.path.joinn()` (an
354
+ invented *call* on a real module) is not — third-party and project members
355
+ are the covered surface.
356
+ * **Factory-function receivers are not judged.** `make_client().refresh()` is
357
+ skipped; annotations, constructor assignments, `self`/`cls`, and inline
358
+ constructors (`UserClient().refresh()`) are resolved.
359
+ * **`undefined_symbol` carries `confidence: high`, not `deterministic`.**
360
+ Only member-surface claims (invented APIs on a fully enumerated class) are
361
+ provable from the repository; name absence rests on the syntactic binding
362
+ model and keeps the lower confidence. Variables reassigned across types in
363
+ one scope are treated as unknown and never judged.
364
+ * **Graph tools are anchored to the last indexed project.** `scan_file` /
365
+ `verify_file` resolve the project root per file (including monorepo
366
+ sub-projects); `find_dead_code`, `explore_call_graph`, `search_code` and
367
+ friends operate on the project most recently passed to `index_project`.
368
+ * **`check_api` imports the base package on demand** — but only when it is
369
+ genuinely installed (or stdlib). A bare file dropped into the workspace is
370
+ never imported, so agent-authored code cannot be executed server-side
371
+ through this tool.
372
+
373
+ ---
374
+
375
+ ## MCP tools
376
+
377
+ By default `scan_file` and `verify_file` are exposed. Set `HONESTCODE_TOOLS=all`
378
+ to enable the full set, including:
379
+
380
+ | Tool | Purpose |
381
+ |:--|:--|
382
+ | `scan_file` | Scan a file for invented APIs, undefined calls, and wrong arities. |
383
+ | `verify_file` | Return the agent-facing structured evidence protocol. |
384
+ | `index_project` | Build or reuse the project symbol index. |
385
+ | `load_project_deps` | Load dependency APIs from `requirements.txt` / `pyproject.toml`. |
386
+ | `check_symbol` | Verify a symbol is defined. |
387
+ | `check_api` | Verify a library API call exists. |
388
+ | `validate_types` | Structural type checks. |
389
+
390
+ Dependency manifests (`requirements.txt`, `pyproject.toml`) are declared
391
+ automatically on the first scan, so imports from a dependency that is not
392
+ installed in the MCP server's interpreter are trusted rather than reported as
393
+ invented. Set `HONESTCODE_AUTO_DEPS=0` to disable that.
394
+
395
+ See the old tool reference below for the complete list.
396
+
397
+ ---
398
+
399
+ ## Crash-safe side effects
400
+
401
+ `verify_file` checks code before it runs. `guards/` covers what happens after:
402
+ an agent that retries a tool call can charge a card twice, and no durable
403
+ execution engine prevents it — Temporal and Restate both guarantee deterministic
404
+ replay of *orchestration* and leave exactly-once external effects to the
405
+ application.
406
+
407
+ ```python
408
+ from honestcode.guards import EffectStore, SideEffectGuard, durable_scope
409
+
410
+ guard = SideEffectGuard(session_id="run-42", store_path=".agent/effects.db")
411
+
412
+ outcome = guard.run(
413
+ "charge",
414
+ target="cus_123",
415
+ payload={"cents": 4200},
416
+ execute=lambda: charge_card("cus_123", 4200),
417
+ approve_irreversible=True,
418
+ scope=durable_scope("order-4711", "charge", "cus_123"),
419
+ )
420
+
421
+ outcome.executed # False on the retry — a receipt, not a second charge
422
+ outcome.effect_id # stable across a restart, because the key is on disk
423
+ ```
424
+
425
+ The interesting part is what `scope` does. Pass it and the effect's identity
426
+ comes from the scope rather than the session — which is the difference between a
427
+ retry being recognised and a customer being charged twice, because a restart *is*
428
+ a new session id. Pass nothing and the guard stays session-scoped.
429
+
430
+ Refusing is the default direction: an unrecognised action is treated as
431
+ irreversible rather than idempotent, and irreversible effects need explicit
432
+ approval. A run that dies between the call and the record leaves the effect
433
+ `executing`, and the next attempt reports the outcome as unknown instead of
434
+ guessing — which is what stops the retry. Scoped effects are recovered with
435
+ `guard.recover(scope=durable_scope(...))`, and `guard.effects(scope=...)` /
436
+ `guard.summary(scope=...)` report them; the records live under a namespace
437
+ derived from the scope so a restart can still find them.
438
+
439
+ For a framework, `integrations/` binds the guard to pydantic-ai, where a
440
+ committed effect replaces the tool result instead of re-running the tool. See
441
+ [`honestcode/integrations/README.md`](honestcode/integrations/README.md).
442
+
443
+ ---
444
+
445
+ ## Benchmark
446
+
447
+ HonestCode includes four benchmark suites that run automatically in CI on every
448
+ push and pull request to `main`:
449
+
450
+ ### Agent-accuracy benchmark
451
+
452
+ `benchmarks/agent_accuracy/` is a deterministic, LLM-free benchmark that
453
+ measures how well HonestCode catches common agent hallucinations (**recall**).
454
+
455
+ Each task is a tiny agent episode: the agent writes a broken file, HonestCode
456
+ verifies it, then the file is replaced with the fix and verified again.
457
+
458
+ ```bash
459
+ cd benchmarks/agent_accuracy
460
+ python run.py
461
+ ```
462
+
463
+ Current results (6 tasks, deterministic verification):
464
+
465
+ | task | expected issue | broken detected | fixed clean | broken ms | fixed ms |
466
+ |---|---|---|---|---|---|
467
+ | invented_method | invented_api (`UserClient.refresh_token`) | yes | yes | 4.18 | 3.78 |
468
+ | invented_module_attr | invented_api (`Connection.query`) | yes | yes | 4.19 | 3.95 |
469
+ | undefined_import | undefined_symbol (`delete_user`) | yes | yes | 2.69 | 3.55 |
470
+ | wrong_signature | wrong_call (`add()`) | yes | yes | 3.56 | 3.35 |
471
+ | cross_file_new_symbol | invented_api (`fly`) | yes | yes | 3.85 | 2.45 |
472
+ | dep_alias_and_pypi_name | invented_api (`toll`) | yes | yes | 3.44 | 2.9 |
473
+
474
+ **Summary:** precision 1.0, recall 1.0, F1 1.0, false-positive rate 0.0, median
475
+ verify time 4–7 ms on a laptop.
476
+
477
+ The per-task milliseconds above are one run's output, not a guarantee — they
478
+ move a few tens of percent between runs on the same machine. What is stable, and
479
+ what the benchmark is gating on, is the first four columns plus the summary's
480
+ counts.
481
+
482
+ The summary line is `python benchmarks/agent_accuracy/run.py` verbatim rather
483
+ than a figure copied into this file, because a timing quoted here goes stale the
484
+ moment the code gets faster — as one did. Reproduce it with `--format markdown`.
485
+
486
+ The `cross_file_new_symbol` task pins the agent-session flow (write a module,
487
+ then a file importing it — the index must be fresh, never stale); the
488
+ `dep_alias_and_pypi_name` task pins dependency grounding through aliased
489
+ submodules and PyPI-named requirements (`PyYAML` → `yaml`).
490
+
491
+ ### Real-world false-positive benchmark
492
+
493
+ `benchmarks/realworld_fp/` measures **precision on real code**: it scans a
494
+ vendored snapshot of `psf/requests` (19 files, ~6.4k LOC, Apache-2.0) and
495
+ asserts HonestCode reports **zero** findings. Real code exercises the
496
+ constructs synthetic tasks never do — stdlib imports, keyword-call style,
497
+ `@staticmethod` members, classes inheriting stdlib ABCs, compat re-export
498
+ modules — and any finding on this corpus is, by construction, a false
499
+ positive.
500
+
501
+ The same run also scans `benchmarks/realworld_fp/regressions/`: one tiny
502
+ project per false-positive class found in the wild (a receiver rebound across
503
+ types, `with ... as` bindings, monorepo sub-project imports, local binding
504
+ forms). Each scenario fails the benchmark by name if the regression returns.
505
+
506
+ ```bash
507
+ python benchmarks/realworld_fp/run.py
508
+ # files: 25 findings: 0 elapsed: 0.7s (~26 ms/file)
509
+ ```
510
+
511
+ The finding count is the claim; the elapsed time is indicative, and varies with
512
+ the machine.
513
+
514
+ ### Value-drift benchmark
515
+
516
+ `benchmarks/value_drift/` asks a different question: when a formatting-tolerant
517
+ edit matcher resolves a search string, did the edit land on the region the
518
+ caller named, or silently on a near-miss sibling that carries a different value?
519
+ It runs ten scenarios across a guarded matcher cascade and an unguarded one, and
520
+ classifies each edit as `correct`, `silent_drift`, `no_match`, or `wrong_span`.
521
+ The guarded matcher must produce the intended edit on every scenario; a
522
+ `no_match` is a refusal, not a pass.
523
+
524
+ ```bash
525
+ python benchmarks/value_drift/run.py # human-readable
526
+ python benchmarks/value_drift/run.py --json # machine-readable (status on stderr)
527
+ ```
528
+
529
+ ### Performance benchmark
530
+
531
+ `scripts/benchmark.py` measures the latency of core operations (index, scan,
532
+ graph, dead code detection, similarity search) on the repository itself.
533
+
534
+ ```bash
535
+ python scripts/benchmark.py # text output
536
+ python scripts/benchmark.py --format markdown # table for README
537
+ python scripts/benchmark.py --repo psf/requests # benchmark a real-world repo
538
+ ```
539
+
540
+ All four benchmarks run in CI (`.github/workflows/ci.yml`) as separate jobs:
541
+ `benchmark-accuracy`, `benchmark-realworld`, `benchmark-value-drift`, and
542
+ `benchmark-performance`. The accuracy job gates on precision and recall staying
543
+ at 1.0, the real-world job on any finding or failed scan, and the value-drift
544
+ job on every guarded scenario landing correctly. The performance job reports
545
+ numbers rather than gating on them.
546
+
547
+ See `benchmarks/agent_accuracy/README.md` for the dataset format and how to
548
+ add new accuracy tasks.
549
+
550
+ ---
551
+
552
+ ## Architecture
553
+
554
+ ```
555
+ honestcode/
556
+ ├── verify/ # Evidence protocol + repository-grounded verification
557
+ ├── mcp/ # MCP server layer
558
+ ├── honest/ # Symbol index + project binding
559
+ ├── graph/ # Persistent call graph (SQLite)
560
+ ├── structure/ # AST extraction
561
+ ├── sandbox/ # Sandboxed execution
562
+ ├── guards/ # Crash-safe side effects (stdlib-only, extractable)
563
+ ├── integrations/ # Framework bindings — pydantic-ai today
564
+ └── cli.py # Command-line interface
565
+ ```
566
+
567
+ `verify_file` is the agent interface. `scan_file` is exposed alongside it by
568
+ default and returns both the new evidence shape and the legacy `issues` list.
569
+
570
+ Two of those directories answer a different question from the rest. Everything
571
+ above `guards/` asks *is this code consistent with the repository*. `guards/`
572
+ asks *did this effect already happen* — an agent that retries a tool call after
573
+ a crash can charge a card twice, and durable-execution engines leave exactly-once
574
+ external effects to the application. `integrations/` binds the guard to a
575
+ framework; pydantic-ai's `wrap_tool_execute` is the first, and a tool result is
576
+ replaced rather than re-run when the effect has already been committed.
577
+
578
+ `guards/` is deliberately stdlib-only and does not depend on this package's
579
+ protocols, so it can be extracted into its own distribution. `tests/test_guards.py`
580
+ enforces that with an AST import check.
581
+
582
+ ---
583
+
584
+ ## Roadmap
585
+
586
+ ### v0.1 — Repository grounding (done)
587
+ - symbol / API / function / class / method / import / call verification
588
+ - invented API detection with structured evidence
589
+ - auto-index on `scan_file`
590
+
591
+ ### Beyond the roadmap — crash-safe side effects (now)
592
+ Not a verification layer, so it has no place in the v0.2/v0.3 sequence below:
593
+ `guards/` makes an effect idempotent across a crash, and `integrations/` binds
594
+ it to pydantic-ai. It answers "did this already happen", which is a runtime
595
+ question rather than a static one.
596
+
597
+ ### v0.2 — Semantic contract verification
598
+ - function expects `UserID`, agent passes `User` → suspicious
599
+
600
+ ### v0.3 — Execution verification
601
+ - static verification → tests → runtime evidence
602
+
603
+ ### v1.0 — Verification Runtime for Coding Agents
604
+ ```text
605
+ Coding Agent
606
+ │
607
+ ┌───────▼───────┐
608
+ │ HonestCode │
609
+ │ Verification │
610
+ │ Runtime │
611
+ └───────┬───────┘
612
+ │
613
+ ┌──────────────┼──────────────┐
614
+ ↓ ↓ ↓
615
+ Static Semantic Runtime
616
+ Verification Verification Verification
617
+ │ │ │
618
+ └──────────────┼──────────────┘
619
+ ↓
620
+ Evidence
621
+ ↓
622
+ Agent
623
+ ```
624
+
625
+ ---
626
+
627
+ ## Development
628
+
629
+ ```bash
630
+ git clone https://github.com/Fengrru/honestcode.git
631
+ cd honestcode
632
+ python -m venv .venv
633
+ .venv\Scripts\activate # Windows
634
+ # source .venv/bin/activate # macOS/Linux
635
+ pip install -e ".[dev]"
636
+ pytest
637
+ ruff check honestcode tests scripts
638
+ ```
639
+
640
+ Run benchmarks locally:
641
+
642
+ ```bash
643
+ # Accuracy benchmark
644
+ python benchmarks/agent_accuracy/run.py
645
+
646
+ # Performance benchmark
647
+ python scripts/benchmark.py
648
+ ```
649
+
650
+ See [CONTRIBUTING.md](CONTRIBUTING.md) for pull request guidelines.
651
+
652
+ ---
653
+
654
+ ## Security
655
+
656
+ See [SECURITY.md](SECURITY.md) for vulnerability reporting.
657
+
658
+ ---
659
+
660
+ ## Telemetry
661
+
662
+ **HonestCode collects no telemetry.** There are no analytics libraries,
663
+ no background services, and no phone-home endpoints.
664
+
665
+ ---
666
+
667
+ ## License
668
+
669
+ [MIT](LICENSE) - Copyright (c) 2026 HonestCode Team