codevanta 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codevanta-0.1.0/CHANGELOG.md +949 -0
- codevanta-0.1.0/LICENSE +21 -0
- codevanta-0.1.0/MANIFEST.in +15 -0
- codevanta-0.1.0/PKG-INFO +688 -0
- codevanta-0.1.0/README.md +639 -0
- codevanta-0.1.0/docs/api.md +1822 -0
- codevanta-0.1.0/docs/architecture.md +93 -0
- codevanta-0.1.0/docs/attribution.md +88 -0
- codevanta-0.1.0/docs/install.md +74 -0
- codevanta-0.1.0/docs/phase2.md +219 -0
- codevanta-0.1.0/docs/phase2b.md +315 -0
- codevanta-0.1.0/docs/phase2c.md +299 -0
- codevanta-0.1.0/docs/phase2d.md +302 -0
- codevanta-0.1.0/docs/phase2e.md +252 -0
- codevanta-0.1.0/docs/phase2f.md +441 -0
- codevanta-0.1.0/docs/phase2g.md +433 -0
- codevanta-0.1.0/docs/phase2h.md +387 -0
- codevanta-0.1.0/docs/phase2i.md +436 -0
- codevanta-0.1.0/docs/phase2j.md +332 -0
- codevanta-0.1.0/docs/phase2k.md +466 -0
- codevanta-0.1.0/docs/phase3.md +1058 -0
- codevanta-0.1.0/docs/phase6.md +353 -0
- codevanta-0.1.0/docs/release-checklist.md +119 -0
- codevanta-0.1.0/docs/repository-settings.md +81 -0
- codevanta-0.1.0/docs/samples.md +254 -0
- codevanta-0.1.0/pyproject.toml +118 -0
- codevanta-0.1.0/requirements.txt +9 -0
- codevanta-0.1.0/scripts/agent_commit.py +213 -0
- codevanta-0.1.0/scripts/bench.py +483 -0
- codevanta-0.1.0/scripts/bench_gate.py +777 -0
- codevanta-0.1.0/scripts/check_decompiler_backends.py +175 -0
- codevanta-0.1.0/scripts/fuzz_parsers.py +1112 -0
- codevanta-0.1.0/scripts/install.ps1 +90 -0
- codevanta-0.1.0/scripts/install.sh +79 -0
- codevanta-0.1.0/scripts/make_samples.py +500 -0
- codevanta-0.1.0/scripts/report_bundle.py +288 -0
- codevanta-0.1.0/setup.cfg +4 -0
- codevanta-0.1.0/src/codevanta/__init__.py +1 -0
- codevanta-0.1.0/src/codevanta/analysis/__init__.py +1 -0
- codevanta-0.1.0/src/codevanta/analysis/binary_diff.py +1168 -0
- codevanta-0.1.0/src/codevanta/analysis/callgraph.py +542 -0
- codevanta-0.1.0/src/codevanta/analysis/cfg.py +1201 -0
- codevanta-0.1.0/src/codevanta/analysis/dataflow.py +762 -0
- codevanta-0.1.0/src/codevanta/analysis/debugger.py +289 -0
- codevanta-0.1.0/src/codevanta/analysis/debuginfo.py +304 -0
- codevanta-0.1.0/src/codevanta/analysis/decompile.py +1094 -0
- codevanta-0.1.0/src/codevanta/analysis/decompilers/__init__.py +250 -0
- codevanta-0.1.0/src/codevanta/analysis/decompilers/angr_backend.py +251 -0
- codevanta-0.1.0/src/codevanta/analysis/decompilers/base.py +278 -0
- codevanta-0.1.0/src/codevanta/analysis/decompilers/ghidra_backend.py +316 -0
- codevanta-0.1.0/src/codevanta/analysis/dynamic.py +756 -0
- codevanta-0.1.0/src/codevanta/analysis/emulate.py +1349 -0
- codevanta-0.1.0/src/codevanta/analysis/evidence_bundle.py +736 -0
- codevanta-0.1.0/src/codevanta/analysis/function_hash.py +562 -0
- codevanta-0.1.0/src/codevanta/analysis/graph_export.py +441 -0
- codevanta-0.1.0/src/codevanta/analysis/html_view.py +695 -0
- codevanta-0.1.0/src/codevanta/analysis/jump_tables.py +546 -0
- codevanta-0.1.0/src/codevanta/analysis/solver.py +715 -0
- codevanta-0.1.0/src/codevanta/analysis/symbolic.py +1809 -0
- codevanta-0.1.0/src/codevanta/analysis/trace.py +646 -0
- codevanta-0.1.0/src/codevanta/analysis/type_propagation.py +1055 -0
- codevanta-0.1.0/src/codevanta/analysis/type_recovery.py +1214 -0
- codevanta-0.1.0/src/codevanta/analysis/vtables.py +572 -0
- codevanta-0.1.0/src/codevanta/analysis/xrefs.py +434 -0
- codevanta-0.1.0/src/codevanta/api/__init__.py +1 -0
- codevanta-0.1.0/src/codevanta/cli/__init__.py +1 -0
- codevanta-0.1.0/src/codevanta/cli/main.py +3038 -0
- codevanta-0.1.0/src/codevanta/core/__init__.py +31 -0
- codevanta-0.1.0/src/codevanta/core/disassembler.py +121 -0
- codevanta-0.1.0/src/codevanta/core/entropy.py +237 -0
- codevanta-0.1.0/src/codevanta/core/functions.py +176 -0
- codevanta-0.1.0/src/codevanta/core/loader.py +1351 -0
- codevanta-0.1.0/src/codevanta/core/report.py +306 -0
- codevanta-0.1.0/src/codevanta/core/strings.py +109 -0
- codevanta-0.1.0/src/codevanta/evidence/__init__.py +7 -0
- codevanta-0.1.0/src/codevanta/evidence/confidence.py +36 -0
- codevanta-0.1.0/src/codevanta/evidence/models.py +43 -0
- codevanta-0.1.0/src/codevanta/evidence/provenance.py +37 -0
- codevanta-0.1.0/src/codevanta/gui/__init__.py +46 -0
- codevanta-0.1.0/src/codevanta/gui/layout.py +1384 -0
- codevanta-0.1.0/src/codevanta/gui/server.py +1305 -0
- codevanta-0.1.0/src/codevanta/ir/__init__.py +6 -0
- codevanta-0.1.0/src/codevanta/ir/adapters/__init__.py +31 -0
- codevanta-0.1.0/src/codevanta/ir/base.py +31 -0
- codevanta-0.1.0/src/codevanta/ir/instruction.py +42 -0
- codevanta-0.1.0/src/codevanta/mcp_server/__init__.py +5 -0
- codevanta-0.1.0/src/codevanta/mcp_server/server.py +710 -0
- codevanta-0.1.0/src/codevanta/mcp_server/tools.py +1364 -0
- codevanta-0.1.0/src/codevanta/plugins/__init__.py +748 -0
- codevanta-0.1.0/src/codevanta/setup/__init__.py +28 -0
- codevanta-0.1.0/src/codevanta/setup/apply.py +22 -0
- codevanta-0.1.0/src/codevanta/setup/clients.py +285 -0
- codevanta-0.1.0/src/codevanta/setup/plan.py +79 -0
- codevanta-0.1.0/src/codevanta/setup/verify.py +43 -0
- codevanta-0.1.0/src/codevanta/storage/__init__.py +1 -0
- codevanta-0.1.0/src/codevanta/storage/session.py +514 -0
- codevanta-0.1.0/src/codevanta.egg-info/PKG-INFO +688 -0
- codevanta-0.1.0/src/codevanta.egg-info/SOURCES.txt +162 -0
- codevanta-0.1.0/src/codevanta.egg-info/dependency_links.txt +1 -0
- codevanta-0.1.0/src/codevanta.egg-info/entry_points.txt +2 -0
- codevanta-0.1.0/src/codevanta.egg-info/requires.txt +27 -0
- codevanta-0.1.0/src/codevanta.egg-info/top_level.txt +1 -0
- codevanta-0.1.0/tests/conftest.py +200 -0
- codevanta-0.1.0/tests/test_agent_commit.py +178 -0
- codevanta-0.1.0/tests/test_bench.py +133 -0
- codevanta-0.1.0/tests/test_bench_gate.py +362 -0
- codevanta-0.1.0/tests/test_binary_diff.py +950 -0
- codevanta-0.1.0/tests/test_callgraph.py +600 -0
- codevanta-0.1.0/tests/test_cfg.py +296 -0
- codevanta-0.1.0/tests/test_cfg_recursive.py +601 -0
- codevanta-0.1.0/tests/test_cli.py +72 -0
- codevanta-0.1.0/tests/test_cli_gui.py +137 -0
- codevanta-0.1.0/tests/test_cli_phase2.py +189 -0
- codevanta-0.1.0/tests/test_cli_phase2b.py +233 -0
- codevanta-0.1.0/tests/test_cli_phase2d.py +118 -0
- codevanta-0.1.0/tests/test_cli_phase2e.py +236 -0
- codevanta-0.1.0/tests/test_cli_phase2f.py +444 -0
- codevanta-0.1.0/tests/test_cli_phase2g.py +463 -0
- codevanta-0.1.0/tests/test_cli_phase2h.py +317 -0
- codevanta-0.1.0/tests/test_cli_phase2i.py +280 -0
- codevanta-0.1.0/tests/test_cli_phase2j.py +347 -0
- codevanta-0.1.0/tests/test_cli_phase2k.py +599 -0
- codevanta-0.1.0/tests/test_cli_phase3.py +703 -0
- codevanta-0.1.0/tests/test_cli_phase7.py +657 -0
- codevanta-0.1.0/tests/test_cli_trace.py +560 -0
- codevanta-0.1.0/tests/test_dataflow.py +400 -0
- codevanta-0.1.0/tests/test_debugger.py +354 -0
- codevanta-0.1.0/tests/test_debuginfo.py +108 -0
- codevanta-0.1.0/tests/test_decompile.py +513 -0
- codevanta-0.1.0/tests/test_decompile_backends.py +508 -0
- codevanta-0.1.0/tests/test_decompiler_parity.py +262 -0
- codevanta-0.1.0/tests/test_disassembler.py +50 -0
- codevanta-0.1.0/tests/test_dynamic.py +506 -0
- codevanta-0.1.0/tests/test_emulate.py +604 -0
- codevanta-0.1.0/tests/test_entropy.py +215 -0
- codevanta-0.1.0/tests/test_evidence.py +52 -0
- codevanta-0.1.0/tests/test_evidence_bundle.py +549 -0
- codevanta-0.1.0/tests/test_function_hash.py +498 -0
- codevanta-0.1.0/tests/test_functions.py +86 -0
- codevanta-0.1.0/tests/test_fuzz_parsers.py +373 -0
- codevanta-0.1.0/tests/test_graph_export.py +579 -0
- codevanta-0.1.0/tests/test_gui.py +1234 -0
- codevanta-0.1.0/tests/test_html_view.py +451 -0
- codevanta-0.1.0/tests/test_jump_tables.py +596 -0
- codevanta-0.1.0/tests/test_loader.py +583 -0
- codevanta-0.1.0/tests/test_mcp_tools.py +1701 -0
- codevanta-0.1.0/tests/test_plugins.py +539 -0
- codevanta-0.1.0/tests/test_report_bundle.py +229 -0
- codevanta-0.1.0/tests/test_report_callgraph.py +295 -0
- codevanta-0.1.0/tests/test_report_phase2.py +329 -0
- codevanta-0.1.0/tests/test_samples.py +167 -0
- codevanta-0.1.0/tests/test_session.py +437 -0
- codevanta-0.1.0/tests/test_setup_plan.py +181 -0
- codevanta-0.1.0/tests/test_solver.py +310 -0
- codevanta-0.1.0/tests/test_strings.py +93 -0
- codevanta-0.1.0/tests/test_symbolic.py +580 -0
- codevanta-0.1.0/tests/test_symbolic_parity.py +320 -0
- codevanta-0.1.0/tests/test_tooling_config.py +143 -0
- codevanta-0.1.0/tests/test_trace.py +481 -0
- codevanta-0.1.0/tests/test_trace_samples.py +267 -0
- codevanta-0.1.0/tests/test_type_propagation.py +536 -0
- codevanta-0.1.0/tests/test_type_recovery.py +866 -0
- codevanta-0.1.0/tests/test_vtables.py +199 -0
- codevanta-0.1.0/tests/test_xrefs.py +506 -0
|
@@ -0,0 +1,949 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to CODEVANTA are documented here. The format follows
|
|
4
|
+
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project uses
|
|
5
|
+
[Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
6
|
+
|
|
7
|
+
## [Unreleased]
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- Vtable discovery and thunk resolution (`codevanta.analysis.vtables` +
|
|
12
|
+
`codevanta vtable` + MCP tool `codevanta_vtable`): indirect cross-references
|
|
13
|
+
the xref scanner cannot see. Candidates are runs of consecutive code
|
|
14
|
+
pointers in non-executable data sections (`vtable.candidate` /
|
|
15
|
+
`vtable.slot_ref`, medium confidence — a run of code pointers is a
|
|
16
|
+
heuristic, not proof); PIE builds work because pointer slots that are zero
|
|
17
|
+
on disk are filled through a dynamic-relocation overlay. Slot targets that
|
|
18
|
+
are one-instruction tail-call thunks resolve to the final callee
|
|
19
|
+
(`resolved_target` + `vtable.thunk_chain`). Address mode decodes one
|
|
20
|
+
candidate at an exact address, skipping up to two leading Itanium-header
|
|
21
|
+
slots (offset-to-top, RTTI); `--scan` sweeps every data section.
|
|
22
|
+
`samples/shapes-vtable` (8.4 KB aarch64 ELF, two C++ classes built with the
|
|
23
|
+
NDK's clang++) backs the tests.
|
|
24
|
+
- DWARF debug-info extraction (`codevanta.analysis.debuginfo` + `codevanta
|
|
25
|
+
dwarf` + MCP tool `codevanta_dwarf`): when a build kept its debug info, the
|
|
26
|
+
strongest symbol evidence available is now first-class — producer-grade
|
|
27
|
+
function names with byte-exact extents (`dwarf.debug_info` /
|
|
28
|
+
`DIE_subprogram`, high confidence) and named type layouts (structs, unions,
|
|
29
|
+
enums, classes) with byte sizes. ELF only via the optional pyelftools extra
|
|
30
|
+
(`pip install -e ".[dwarf]"`, included in `dev`); missing extra and
|
|
31
|
+
non-ELF inputs fail with typed messages naming the limitation. PDB (PE)
|
|
32
|
+
stays open: the LIEF OSS build gates its PDB parser behind the Extended
|
|
33
|
+
distribution, so there is no OSS backend to wrap yet.
|
|
34
|
+
`samples/hello-dwarf` (7.7 KB aarch64 ELF built with the NDK's clang `-g`)
|
|
35
|
+
backs the test suite.
|
|
36
|
+
- DEX loader (`codevanta.core.loader`): Android DEX containers (magic
|
|
37
|
+
`dex\n`) now load through LIEF's dedicated DEX parser with format, version
|
|
38
|
+
endianness, and Dalvik method extraction. Defined methods become
|
|
39
|
+
`high`-confidence function symbols from `dex.method_ids` with
|
|
40
|
+
smali-style names (`LMain;->add`); referenced-but-undefined methods become
|
|
41
|
+
imports carrying the defining class as `library`. DEX has no virtual
|
|
42
|
+
address space, so symbol addresses are **file offsets** — stated in a
|
|
43
|
+
loader warning and in every finding's evidence. Disassembly stays
|
|
44
|
+
unsupported (`dalvik` is not a Capstone target) and fails with the usual
|
|
45
|
+
typed error. A real 952-byte `samples/hello.dex` (built with the Android
|
|
46
|
+
SDK's `d8`) backs the test suite.
|
|
47
|
+
|
|
48
|
+
- Phase 7 wiring (`feat/phase7-wiring`): the phase 6 engines are reachable
|
|
49
|
+
from every surface. New CLI commands `codevanta symbolic` (branch-decision
|
|
50
|
+
path exploration with `--set REG=VALUE` seeds and `--max-paths`/`--max-steps`
|
|
51
|
+
budgets), `codevanta dataflow` (def-use and provable constants of one
|
|
52
|
+
function), `codevanta evidence-bundle` (hash a binary plus `--entry
|
|
53
|
+
NAME=FILE` payloads into a tamper-evident bundle; refuses to overwrite the
|
|
54
|
+
subject) and `codevanta plugins list/run` (the live registry, and one plugin
|
|
55
|
+
run under the harness budget). MCP gains `codevanta_symbolic` and
|
|
56
|
+
`codevanta_dataflow` (the server now registers 22 tools) and the dashboard
|
|
57
|
+
gains `/api/symbolic` with its panel (14 routes; `build_server().list_tools()`
|
|
58
|
+
and `ROUTES` both verified). `scripts/report_bundle.py` ties a report and an
|
|
59
|
+
optional saved trace into one verified bundle; `bench/baseline.json` became
|
|
60
|
+
per-platform (`bench/baseline.windows.json`, `bench/baseline.linux.json`) so
|
|
61
|
+
the gate never compares timings across operating systems; saved traces
|
|
62
|
+
(`samples/gdb_trace.log`, `samples/frida_trace.jsonl`) let `codevanta trace`
|
|
63
|
+
be exercised offline, and `tests/test_cli_phase7.py`,
|
|
64
|
+
`test_trace_samples.py`, `test_symbolic_parity.py`, `test_report_bundle.py`
|
|
65
|
+
pin all of it.
|
|
66
|
+
- CI hardening: a `fuzz-smoke` job runs the parser fuzzer (30 seeded iterations
|
|
67
|
+
over the committed samples) plus `tests/test_fuzz_parsers.py` on every PR; an
|
|
68
|
+
informational `benchmark` job reports drift against the committed baseline
|
|
69
|
+
with a wide tolerance and `continue-on-error` (timings are machine-dependent,
|
|
70
|
+
so it reports, it does not gate); a non-blocking `test-platform` job runs the
|
|
71
|
+
suite on windows-latest and macos-latest (py3.13) as platform signal while
|
|
72
|
+
parity stays proven by the Linux job.
|
|
73
|
+
- Path exploration (`codevanta.analysis.symbolic` + `codevanta symbolic` + MCP
|
|
74
|
+
tool `codevanta_symbolic`): breadth-first replay of the branch decisions of
|
|
75
|
+
one function on a private unicorn CPU. The observed side of every branch is
|
|
76
|
+
the real emulation result; the other side is enqueued with its guarding flags
|
|
77
|
+
or loop counter rewritten (`seeded` — an assumption, never a solved
|
|
78
|
+
constraint, and each such branch says so with a per-path warning and drops
|
|
79
|
+
the summary to low confidence). No symbolic variables, no path predicate, no
|
|
80
|
+
SMT anywhere. Paths report decisions, taken targets, end registers, the
|
|
81
|
+
`rax`/`eax` value at a leaving `ret` (`None` otherwise), a stop reason
|
|
82
|
+
(`return`/`budget`/`blocked`/`unmapped`) and executed steps; `complete` is
|
|
83
|
+
true only when nothing was left unexplored and every reported path returned.
|
|
84
|
+
Optional angr bridge (`explore_with_angr`) reports the same shape when angr
|
|
85
|
+
is installed; x86/x86-64 only; missing unicorn, bad addresses and bad seeds
|
|
86
|
+
fail with typed errors before anything runs.
|
|
87
|
+
- Dataflow (`codevanta.analysis.dataflow` + `codevanta dataflow` + MCP tool
|
|
88
|
+
`codevanta_dataflow`): one forward pass over the recursive CFG recovers
|
|
89
|
+
register definitions, uses, provable constants (`mov` immediates, copies,
|
|
90
|
+
`add`/`sub`/`xor` of known values, rip-relative loads) and reaching
|
|
91
|
+
definitions, with alias normalisation and call-clobber handling. Honest
|
|
92
|
+
limits stated in the result: the pass is a linearization (joins take the
|
|
93
|
+
highest-addressed predecessor), undefined reads become warnings, and nothing
|
|
94
|
+
is guessed.
|
|
95
|
+
- Evidence bundles (`codevanta.analysis.evidence_bundle`): hash a subject
|
|
96
|
+
binary (streamed SHA-256) and any number of canonical-JSON payloads into one
|
|
97
|
+
frozen document; `verify_bundle` recomputes every digest and reports each
|
|
98
|
+
mismatch by name, catching edited payloads, edited digests, and
|
|
99
|
+
added/removed/renamed entries. Bounded (256 entries, 16 MiB per payload),
|
|
100
|
+
deterministic byte-for-byte across builds, with `build`/`verify`/`read`/
|
|
101
|
+
`write`/`dumps` helpers and the digest recipe documented for non-Python
|
|
102
|
+
verifiers.
|
|
103
|
+
- Plugin SDK (`codevanta.plugins`): a structural protocol (name, version,
|
|
104
|
+
description, `analyze(context) -> mapping`) so an outside analyser never
|
|
105
|
+
needs to import CODEVANTA; entry-point discovery (`codevanta.analyzers`,
|
|
106
|
+
kinds analyzer/exporter/provider) that reads metadata without importing; a
|
|
107
|
+
process-local registry with `register`/`unregister`/`builtin_plugins`;
|
|
108
|
+
`build_context` (loaded binary, discovered candidates, entry-point focus,
|
|
109
|
+
read-only budget) and `run_plugin` under a wall-clock and canonical-JSON byte
|
|
110
|
+
budget. The built-in `MetricsAnalyzer` proves the path end to end, and
|
|
111
|
+
`codevanta plugins list/run` drive it from the CLI.
|
|
112
|
+
- Trace and attach CLI (`codevanta trace`, `codevanta attach`): replay a saved
|
|
113
|
+
gdb/Frida session capture and map its hits onto function candidates
|
|
114
|
+
(`--provider auto` detects the format and states its choice; an unplaceable
|
|
115
|
+
capture exits 2 instead of guessing), and generate the provider script plus
|
|
116
|
+
the exact argv that runs it — without ever launching anything. MCP parity via
|
|
117
|
+
`codevanta_trace`, whose errors carry the real reason instead of a generic
|
|
118
|
+
tool failure.
|
|
119
|
+
- GUI session persistence: `/api/session_save` (the dashboard's one writing
|
|
120
|
+
route — one row in the same SQLite store `codevanta session save` uses) and
|
|
121
|
+
`/api/session_list`, plus a Sessions panel; the counts a save records come
|
|
122
|
+
from the same engines the page displays.
|
|
123
|
+
- Parser fuzz harness (`scripts/fuzz_parsers.py` +
|
|
124
|
+
`tests/test_fuzz_parsers.py`): seeded, replayable mutations (byteflip,
|
|
125
|
+
truncate, length-field, section-table, huge-counts, zero-sizes, splice) of
|
|
126
|
+
the committed samples and in-memory-packed ELFs, treating any undocumented
|
|
127
|
+
exception from the public API as a finding that names seed, iteration, file,
|
|
128
|
+
edit and mutant digest.
|
|
129
|
+
- Benchmark regression gate (`scripts/bench_gate.py` +
|
|
130
|
+
`tests/test_bench_gate.py`): imports `scripts/bench.py`'s measurement helpers
|
|
131
|
+
and compares every `best_ms` against a committed best-of-15 baseline (50%
|
|
132
|
+
default tolerance, missing rows fail, `--repeat` can raise precision but
|
|
133
|
+
never drop below the baseline's, one confirming re-measure keeps the better
|
|
134
|
+
pass, `--update-baseline` accepts new numbers, `--platform` picks the
|
|
135
|
+
baseline).
|
|
136
|
+
|
|
137
|
+
### Changed
|
|
138
|
+
|
|
139
|
+
- Dashboard usability pass (`codevanta gui`): the workflow is now guided
|
|
140
|
+
instead of expert-first. Step 1 is a single path field with one primary
|
|
141
|
+
**Analyze** button (Enter works too); its result renders as a plain-language
|
|
142
|
+
overview — format/arch chips, section/import/export/symbol/function/string
|
|
143
|
+
counts, entry point, human-readable file size, SHA-256 — instead of a raw
|
|
144
|
+
JSON table. Step 2 presents the six deeper views as cards with
|
|
145
|
+
plain-English titles and one-line questions they answer ("Who calls whom",
|
|
146
|
+
"Code flow", "Data layouts", ...), each carrying only its own contextual
|
|
147
|
+
options. Diff moved into a collapsed "Advanced" disclosure, the empty state
|
|
148
|
+
explains what to do first, and JSON stays available under "Raw JSON (for
|
|
149
|
+
scripts and agents)". Element ids stayed stable where tests pin them; all
|
|
150
|
+
rendering remains `textContent`-only with zero external references.
|
|
151
|
+
- Dashboard redesign (`codevanta gui`): the layout was rewritten around a
|
|
152
|
+
sticky top bar with trust badges (read-only, local), a hero path input with
|
|
153
|
+
Enter-to-run, a labeled options panel, a full-width seven-action toolbar,
|
|
154
|
+
and a result card with a sticky-header table, colored confidence badges
|
|
155
|
+
(certain/high green, medium amber, low red), a "HTTP 200 · N of M rows"
|
|
156
|
+
status chip, auto-scroll to results, and raw JSON tucked into a collapsible
|
|
157
|
+
disclosure. Typography now pairs a system sans for UI with monospace
|
|
158
|
+
reserved for data; surfaces are layered (page, card, inset inputs) with a
|
|
159
|
+
single teal accent. Still a fully self-contained page: no CDN, no fonts, no
|
|
160
|
+
external URL, and results remain `textContent`-only.
|
|
161
|
+
|
|
162
|
+
### Fixed
|
|
163
|
+
|
|
164
|
+
- Trace attribution (`codevanta.analysis.trace`): a hit above the last known
|
|
165
|
+
function was attributed to it because an unknown candidate size was treated
|
|
166
|
+
as unbounded. Candidate extents now come from the symbol size or, failing
|
|
167
|
+
that, the enclosing executable section, and an address outside every
|
|
168
|
+
candidate is reported unattributed instead of guessed onto the nearest
|
|
169
|
+
symbol.
|
|
170
|
+
|
|
171
|
+
### Added
|
|
172
|
+
|
|
173
|
+
- Dynamic session layer (`codevanta.analysis.dynamic`): gdb and Frida providers
|
|
174
|
+
with availability detection that never imports the tool, deterministic
|
|
175
|
+
non-interactive script builders (`build_gdb_script`, `build_frida_script`) that
|
|
176
|
+
bound hits, validate addresses against executable sections and record a header
|
|
177
|
+
with the target and image base, plus `attach_command` for the exact argv a user
|
|
178
|
+
should run. Nothing is launched: the scripts are generated for the user to run.
|
|
179
|
+
- Trace analysis (`codevanta.analysis.trace`): `parse_gdb_trace` and
|
|
180
|
+
`parse_frida_trace` turn a finished session into a `TraceReport` — hits mapped to
|
|
181
|
+
function candidates, hits per function, unique addresses, truncation and
|
|
182
|
+
evidence — so dynamic results feed the same evidence pipeline as static ones.
|
|
183
|
+
- Ghidra provider in CI (`ghidra-parity`, best effort): a pinned Ghidra release and
|
|
184
|
+
JDK are downloaded, `GHIDRA_INSTALL_DIR` is exported and the parity tests run
|
|
185
|
+
against the real Ghidra decompiler; when the download cannot happen the job
|
|
186
|
+
prints a notice instead of pretending it ran. `pyghidra` joined the
|
|
187
|
+
`decompilers` extra.
|
|
188
|
+
- Type propagation for the pseudo-C decompiler (`codevanta.analysis.type_propagation`):
|
|
189
|
+
per-argument inference from how each register is used (pointer dereference,
|
|
190
|
+
sub-width load, arithmetic, comparison against zero, unused), a return type from
|
|
191
|
+
the last write before `ret`, and every verdict carrying the observation that
|
|
192
|
+
produced it at a stated confidence. The decompiler signature and
|
|
193
|
+
`evidence.extra` now say `uint64_t * a0` / `int32_t a1` / `uint32_t` instead of
|
|
194
|
+
`undefined64` for everything.
|
|
195
|
+
- Decompiler parity job (`decompile-parity` in CI): installs the new
|
|
196
|
+
`decompilers` extra, prints `scripts/check_decompiler_backends.py`, and runs
|
|
197
|
+
`tests/test_decompiler_parity.py`, which decompiles a fixture through a real
|
|
198
|
+
angr backend and asserts the provider layer picked it instead of the native
|
|
199
|
+
fallback. `scripts/check_decompiler_backends.py` gives the same availability
|
|
200
|
+
report locally, with `--json` for tooling.
|
|
201
|
+
- MCP: `codevanta_entropy` exposes the entropy triage over MCP (same bounded
|
|
202
|
+
`threshold` / `max_sample_bytes` validation as the CLI), and
|
|
203
|
+
`codevanta_analyze` gains `include_xrefs` / `include_cfg` /
|
|
204
|
+
`include_callgraph` flags mirroring the CLI so a single call can carry the
|
|
205
|
+
phase 2 sections; defaults keep the phase 1 payload unchanged.
|
|
206
|
+
- Entropy triage (`codevanta.core.entropy` + `codevanta entropy`): per-section
|
|
207
|
+
Shannon entropy sampled from file bytes (capped per section, sample size
|
|
208
|
+
reported) with a packing heuristic — sections at or above the threshold
|
|
209
|
+
(default 7.2 bits/byte, `--threshold`) get a `packed_hint` labelled medium
|
|
210
|
+
confidence in evidence. Sections without file bytes (e.g. `S_ZEROFILL`,
|
|
211
|
+
`NOBITS`) carry `entropy: null` with a warning instead of a fabricated
|
|
212
|
+
value, and samples below the minimum floor suppress the packing judgment.
|
|
213
|
+
`--max-sample-bytes` bounds the read; `--json` emits the full report.
|
|
214
|
+
- Local web dashboard (`codevanta.gui`, `codevanta gui`): a standard-library-only
|
|
215
|
+
`ThreadingHTTPServer` bound to a loopback address serves one self-contained
|
|
216
|
+
dashboard page and nine read-only `GET` JSON routes (`/api/health`,
|
|
217
|
+
`/api/info`, `/api/functions`, `/api/strings`, `/api/xrefs`, `/api/cfg`,
|
|
218
|
+
`/api/struct`, `/api/callgraph`, `/api/diff`). Every route is a thin adapter
|
|
219
|
+
over the engine of the same name — no analysis is reimplemented — so each
|
|
220
|
+
payload keeps that engine's evidence record and limits. A host outside
|
|
221
|
+
`localhost`, `::1` or `127.0.0.0/8` is refused with `ValueError`, every numeric
|
|
222
|
+
query parameter is validated against the consuming engine's ceiling, and a
|
|
223
|
+
payload over `MAX_RESPONSE_BYTES` (4 MiB) has its top-level lists halved and is
|
|
224
|
+
flagged `response_truncated`. `create_server(port=0)` returns a bound,
|
|
225
|
+
unstarted server for tests and `create_handler(paths_allowed=False)` rejects
|
|
226
|
+
the file-taking routes outright.
|
|
227
|
+
- Fat (universal) Mach-O loading (`codevanta.core.loader`): the four container
|
|
228
|
+
magics (`FAT_MAGIC`, `FAT_CIGAM`, `FAT_MAGIC_64`, `FAT_CIGAM_64`, 32- and
|
|
229
|
+
64-bit `fat_arch` tables) are parsed before LIEF runs, and exactly one slice is
|
|
230
|
+
then parsed. Selection is deterministic and stated in the evidence: a 64-bit
|
|
231
|
+
slice wins over a 32-bit one (`prefer_64bit`), otherwise the lowest file offset
|
|
232
|
+
wins (`first_by_offset`); every other slice is reported, never parsed. Section
|
|
233
|
+
offsets are rebased by the slice offset, so `va_to_offset()` maps into the
|
|
234
|
+
outer container, and `BinaryInfo.evidence` carries `source="macho.fat_header"`
|
|
235
|
+
with the structured `extra` keys (`fat`, `fat_magic`, `fat_magic_name`,
|
|
236
|
+
`fat_arch_64`, `slice_index`, `slice_count`, `slice_arch`, `slice_cputype`,
|
|
237
|
+
`slice_offset`, `slice_size`, `selection_rule`, `selection_reason`,
|
|
238
|
+
`available_arches`, `available_slices`) at `certain` confidence. Malformed
|
|
239
|
+
containers raise `BinaryLoadError` naming the defect — zero `nfat_arch`, a
|
|
240
|
+
truncated `fat_arch` table, a zero slice size, or an offset/size past the end
|
|
241
|
+
of the file.
|
|
242
|
+
- Committed PE and Mach-O samples (`samples/pe64_text.bin`,
|
|
243
|
+
`samples/macho64_text.bin`): a PE32+ image with one executable `.text` and a
|
|
244
|
+
one-entry export table, and a thin `MH_EXECUTE` Mach-O exporting
|
|
245
|
+
`_codevanta_sample_main`, sharing one hand-assembled x86_64 code body and ASCII
|
|
246
|
+
payload so cross-format behaviour can be compared directly.
|
|
247
|
+
`scripts/make_samples.py` (standard library only) regenerates them byte for
|
|
248
|
+
byte from zeroed timestamps and fixed padding, `--check` exits non-zero when a
|
|
249
|
+
committed sample is stale, `--output-dir` writes elsewhere, and
|
|
250
|
+
`tests/test_samples.py` enforces that in CI.
|
|
251
|
+
- MCP HTTP transports (`codevanta.mcp_server.server.run`, `codevanta mcp serve`):
|
|
252
|
+
`transport` accepts `stdio` (default, unchanged: JSON-RPC over stdin/stdout),
|
|
253
|
+
`sse` and `streamable-http`, which bind `host:port` instead
|
|
254
|
+
(`port=0` picks a free port). Because these tools read arbitrary local files, a
|
|
255
|
+
non-loopback host is refused unless `allow_remote=True`, and an unknown
|
|
256
|
+
transport raises `AnalysisError` naming the supported set. The CLI forwards
|
|
257
|
+
every option verbatim (`--transport`, `--host`, `--port`, `--allow-remote`) and
|
|
258
|
+
exits `2` on a rejected transport or bind, so existing stdio configurations
|
|
259
|
+
keep their meaning.
|
|
260
|
+
- CI coverage and type checks (`.github/workflows/ci.yml`): the test job runs
|
|
261
|
+
`pytest --cov=codevanta --cov-report=term-missing`, and a separate `type-check`
|
|
262
|
+
job runs `mypy src/codevanta` on Python 3.13.
|
|
263
|
+
- Benchmark script (`scripts/bench.py`): times the public analysis entry points
|
|
264
|
+
(`load`, `discover_functions`, `extract_strings`, disassembly, `find_xrefs`,
|
|
265
|
+
`build_cfg`, `build_call_graph`, `function_fingerprint`, `suggest_struct`, and
|
|
266
|
+
`diff_binaries` when two inputs are given) with `time.perf_counter` only, so a
|
|
267
|
+
performance regression shows up as a number. It defaults to the committed
|
|
268
|
+
`samples/`, takes `--repeat`, `--max-instructions` and `--limit-functions`
|
|
269
|
+
bounds, reports the shortest run per benchmark as `best_ms`, prints a sorted
|
|
270
|
+
table or `--json`, and writes nothing — inputs are read, never mapped or
|
|
271
|
+
executed. A benchmark the analysis layer refuses shows `iterations 0` with the
|
|
272
|
+
reason on stderr instead of a fabricated time.
|
|
273
|
+
- API reference and sample documentation (`docs/api.md`, `docs/samples.md`):
|
|
274
|
+
every public entry point with the signature and defaults read from the source,
|
|
275
|
+
the evidence/confidence contract, the dashboard routes, the fifteen MCP tools,
|
|
276
|
+
and six runnable recipes whose printed output is the real output; plus a page
|
|
277
|
+
per sample binary with its SHA-256 and one working command each.
|
|
278
|
+
|
|
279
|
+
- Big-endian UTF-16 string extraction (`codevanta.core.strings`): the UTF-16
|
|
280
|
+
scan now follows the binary's endianness — big-endian targets (e.g. MIPS,
|
|
281
|
+
s390x, PowerPC) yield `utf-16be` strings, little-endian targets keep
|
|
282
|
+
`utf-16le`. Without binary metadata the scan stays UTF-16LE rather than
|
|
283
|
+
guessing a byte swap, so shifted cross-byte-order matches are never
|
|
284
|
+
fabricated. Evidence labels follow the existing scheme
|
|
285
|
+
(`string_scanner` / `utf-16be_scan`).
|
|
286
|
+
- Mach-O loader support (`codevanta.core.loader`): Mach-O 32/64 binaries now
|
|
287
|
+
load with format, architecture (x86_64, aarch64, x86, arm, ppc), bitness,
|
|
288
|
+
endianness, image base, entry point, sections, imports (undefined symbols),
|
|
289
|
+
exports (dyld export information), and symbol-table entries — every finding
|
|
290
|
+
carrying evidence. Mach-O symbol entries do not encode kind, so
|
|
291
|
+
`is_function` is derived from executable-section membership, stated in the
|
|
292
|
+
evidence detail, and function candidates from the Mach-O symbol table are
|
|
293
|
+
reported at `medium` confidence. Sections without file bytes (e.g.
|
|
294
|
+
`S_ZEROFILL`) map to no file offset, keeping VA/offset translation honest.
|
|
295
|
+
- Cross-reference scanner (`codevanta.analysis.xrefs`): `call`, `jump`, and
|
|
296
|
+
`branch` references recovered from direct branch operands in executable
|
|
297
|
+
sections. Targets that are only reachable through registers or memory are
|
|
298
|
+
skipped rather than guessed. Every reference carries evidence
|
|
299
|
+
(`capstone.direct_call` / `direct_jump` / `conditional_branch`, confidence
|
|
300
|
+
`certain`), and the scan reports section coverage, scanned bytes, and whether
|
|
301
|
+
it was truncated by the `max_bytes` budget.
|
|
302
|
+
- Control-flow graph builder (`codevanta.analysis.cfg`): linear sweep of one
|
|
303
|
+
region into `BasicBlock`s with sorted, deduplicated successor edges derived
|
|
304
|
+
from leader and terminator rules. Indirect branches stay unresolved; the
|
|
305
|
+
sweep is bounded by `max_instructions` and `max_blocks` and is labelled
|
|
306
|
+
`medium` confidence because block boundaries are heuristic.
|
|
307
|
+
- SQLite session storage (`codevanta.storage.session`): additive analysis
|
|
308
|
+
records (fingerprint, format, architecture, bitness, entry point, function and
|
|
309
|
+
string counts, notes) with `save`, `get`, `list`, and `delete`. Standard
|
|
310
|
+
library only, parameterized SQL, bounded queries.
|
|
311
|
+
- CLI: `codevanta xrefs` (with `--limit`, `--max-bytes`, `--json`),
|
|
312
|
+
`codevanta cfg` (with `--address`/`--function`, `--max-instructions`,
|
|
313
|
+
`--max-blocks`, `--json`), and `codevanta session save|list|show` (with
|
|
314
|
+
`--db`, defaulting to `$CODEVANTA_DB` or `~/.codevanta/sessions.db`).
|
|
315
|
+
- Direct-call call graph (`codevanta.analysis.callgraph`): `build_call_graph`
|
|
316
|
+
aggregates the `call` cross references into `CallGraphNode` /
|
|
317
|
+
`CallGraphEdge` records plus a generic `{nodes, edges}` model. Every call
|
|
318
|
+
target that is not a known candidate becomes an `external` node, edges are
|
|
319
|
+
deduplicated by `(source, target, from_address)` and bounded by
|
|
320
|
+
`max_functions` / `max_edges`. Caller attribution is heuristic — the nearest
|
|
321
|
+
function candidate at or below the call site inside the same section — so the
|
|
322
|
+
graph reports `medium` confidence, uses the call address as the source for
|
|
323
|
+
unattributable sites, and says so in `warnings`.
|
|
324
|
+
- Graph export (`codevanta.analysis.graph_export`): dependency-free `to_dot` and
|
|
325
|
+
`to_graphml` render the shared graph model with deterministic ordering, strict
|
|
326
|
+
contract validation (`GraphExportError`), escaped/quoted DOT strings, and a
|
|
327
|
+
GraphML document with `label`/`kind`/edge-label keys. Output only — nothing is
|
|
328
|
+
written to disk.
|
|
329
|
+
- Session annotations (`codevanta.storage.session`): `Annotation` records plus
|
|
330
|
+
`annotate` (one upsertable note per `(session_id, address)`), `annotations`
|
|
331
|
+
(ordered by address), and `remove_annotation`. Notes are bounded to 4,000
|
|
332
|
+
characters, blank notes and unknown sessions raise `SessionError`.
|
|
333
|
+
- MCP tools: `codevanta_xrefs`, `codevanta_cfg`, `codevanta_session_save`, and
|
|
334
|
+
`codevanta_session_list` join the five phase 1 tools, for nine tools total.
|
|
335
|
+
Bounded arguments, shared database resolution (explicit `db`, then
|
|
336
|
+
`$CODEVANTA_DB`, then `~/.codevanta/sessions.db`), and the same evidence
|
|
337
|
+
semantics as the CLI.
|
|
338
|
+
- CLI: `codevanta callgraph` (with `--limit-functions`, `--limit-edges`,
|
|
339
|
+
`--format json|dot|graphml`, `--output`/`-o`) and
|
|
340
|
+
`codevanta session annotate SESSION_ID ADDRESS NOTE` /
|
|
341
|
+
`codevanta session notes SESSION_ID` (decimal or `0x` hex addresses, `--db`).
|
|
342
|
+
- Import and indirect call resolution (`codevanta.analysis.xrefs`): `call`
|
|
343
|
+
instructions through an absolute or RIP-relative pointer slot
|
|
344
|
+
(`call qword ptr [0x402000]`, `call qword ptr [rip + 0xa]`) now resolve to the
|
|
345
|
+
slot address — RIP-relative operands are biased by the instruction length, so
|
|
346
|
+
negative displacements work. A slot that matches an `info.imports` address
|
|
347
|
+
becomes `call_import` with `target_name` set to the import name and confidence
|
|
348
|
+
`certain`; any other resolved slot becomes `call_indirect` with confidence
|
|
349
|
+
`high` (the slot is proven, the callee is not). The reported `to_address` is
|
|
350
|
+
always the slot, never the code behind it. Register and indexed operands
|
|
351
|
+
(`call rax`, `call qword ptr [rax + 8]`) are skipped and counted in the new
|
|
352
|
+
`XrefResult.indirect_skipped` field, which `to_dict()` exposes; indirect jumps
|
|
353
|
+
and conditional branches are not counted. Direct literal operands still take
|
|
354
|
+
precedence, and the scan stays bounded by `max_bytes`.
|
|
355
|
+
- Call graph coverage (`codevanta.analysis.callgraph`): edges now come from
|
|
356
|
+
`call`, `call_import`, and `call_indirect` cross references. A resolved import
|
|
357
|
+
slot becomes an `import` node labelled from the import table (`high`
|
|
358
|
+
confidence, namespaced `import:0x<addr>` in the generic graph model), a
|
|
359
|
+
`call_indirect` slot stays an `external` node (`unknown` confidence), and
|
|
360
|
+
`jump`/`branch` references plus unresolved indirect calls still produce no
|
|
361
|
+
edges. Edge deduplication, ordering, limits, and the heuristic caller
|
|
362
|
+
attribution (nearest candidate at or below the call site in the same section,
|
|
363
|
+
with one aggregated warning for unattributable sites) are unchanged.
|
|
364
|
+
- `codevanta analyze --callgraph`: embeds `report["callgraph"]` when the flag is
|
|
365
|
+
passed and omits the key entirely when it is not. The embedded graph is bounded
|
|
366
|
+
to 1000 functions and 5000 edges
|
|
367
|
+
(`REPORT_MAX_CALLGRAPH_FUNCTIONS` / `REPORT_MAX_CALLGRAPH_EDGES`), reuses the
|
|
368
|
+
pre-computed candidates, sets `truncated` when a cap cuts it short, and prints
|
|
369
|
+
a one-line node/edge/truncation summary alongside `--output`. The analysis
|
|
370
|
+
layer is imported lazily, so `codevanta.core.report` gains no import-time
|
|
371
|
+
dependency on `codevanta.analysis`.
|
|
372
|
+
- MCP tool `codevanta_callgraph` (`limit_functions`, `limit_edges`, defaulting to
|
|
373
|
+
1000 and 5000, accepted range 1..200000) joins the server, bringing it to ten
|
|
374
|
+
tools. It returns `CallGraph.to_dict()` with nodes, edges, `truncated`,
|
|
375
|
+
`warnings`, and evidence, and rejects non-integer or out-of-range limits with
|
|
376
|
+
`AnalysisError`.
|
|
377
|
+
- Documentation: `docs/phase2c.md` covers import/indirect resolution, import
|
|
378
|
+
nodes, the embedded report graph, the ten MCP tools, CLI examples, remaining
|
|
379
|
+
limits, and next steps; the README now links `CONTRIBUTORS.md` from its
|
|
380
|
+
Community section.
|
|
381
|
+
- `validate_graph` (`codevanta.analysis.graph_export`) is now public: the DOT and
|
|
382
|
+
GraphML checks return a normalized, deterministically ordered copy, so other
|
|
383
|
+
renderers reuse one contract instead of duplicating it.
|
|
384
|
+
- Offline HTML/SVG graph viewer (`codevanta.analysis.html_view`): `to_html`
|
|
385
|
+
renders the shared graph model as one self-contained HTML5 document — inline
|
|
386
|
+
stylesheet, inline SVG in three `kind`-based columns
|
|
387
|
+
(`function`/`entry_point`, `call_site`, everything else), header node/edge
|
|
388
|
+
counts, arrowed straight-line edges, a collapsible `<details>` table listing
|
|
389
|
+
every edge, and an `application/json` payload of the normalized graph. It
|
|
390
|
+
references no `http(s)` URL, CDN, web font or external file and shipped without
|
|
391
|
+
JavaScript logic in its phase 2d form (phase 2f adds one inline script; see
|
|
392
|
+
below), so it opens offline; the `<svg>` omits `xmlns` and is
|
|
393
|
+
inline-only. Output is byte-identical for identical content.
|
|
394
|
+
- CLI: `codevanta callgraph --format html` (with `--output`/`-o`) joins
|
|
395
|
+
`json|dot|graphml`. Every serializer now receives size-derived caps
|
|
396
|
+
(`max(default, len(...))`), so a graph the analysis already accepted is never
|
|
397
|
+
rejected downstream by an exporter default.
|
|
398
|
+
- Documentation: `docs/phase2d.md` covers the viewer, the three columns, document
|
|
399
|
+
anatomy, caps and sizing, offline guarantees, limits, and next steps.
|
|
400
|
+
- Recursive CFG traversal (`codevanta.analysis.cfg`): new
|
|
401
|
+
`build_recursive_cfg(path, info, start, max_instructions=2000, max_blocks=500)`
|
|
402
|
+
returns the same `FunctionCFG` as `build_cfg`, but follows control flow from
|
|
403
|
+
`start` with a worklist of block starts and a visited set instead of decoding a
|
|
404
|
+
fixed window. Successors follow the block-terminating instruction: a
|
|
405
|
+
conditional jump yields fallthrough plus its direct target, an unconditional
|
|
406
|
+
`jmp` yields the target only, `ret`/`iret` yield none, and every other
|
|
407
|
+
instruction (including `call`) yields fallthrough only. Call targets are not
|
|
408
|
+
followed and indirect jumps are not resolved; direct targets outside executable
|
|
409
|
+
sections are dropped and reported in `warnings`. Blocks and successors are
|
|
410
|
+
sorted, the graph is bounded by `max_instructions` / `max_blocks` (with the
|
|
411
|
+
existing `truncated` flag), and evidence uses `cfg.recursive_traversal` at
|
|
412
|
+
`medium` confidence because indirect targets remain unresolved. Unreachable
|
|
413
|
+
padding after a `ret` or `jmp` is no longer decoded.
|
|
414
|
+
- CLI: `codevanta cfg --traversal linear|recursive`. The default stays `linear`,
|
|
415
|
+
so existing invocations, tables, and JSON output are unchanged; `recursive`
|
|
416
|
+
selects `build_recursive_cfg` and reports the same payload with the
|
|
417
|
+
`cfg.recursive_traversal` evidence source.
|
|
418
|
+
- Documentation: `docs/phase2e.md` covers the new traversal, its successor rules,
|
|
419
|
+
how it differs from the linear sweep, the CLI flag, evidence and confidence,
|
|
420
|
+
bounds, limitations, and next steps.
|
|
421
|
+
- Jump-table resolver (`codevanta.analysis.jump_tables`): `resolve_jump_table`
|
|
422
|
+
decodes a bounded array of absolute pointers starting at a caller-supplied
|
|
423
|
+
address — 8-byte slots on a 64-bit binary, 4-byte on a 32-bit one, decoded with
|
|
424
|
+
the binary's endianness. The scan stops at the first slot that cannot be a
|
|
425
|
+
target (zero terminator, target outside `min_target`/`max_target`, target no
|
|
426
|
+
section maps, or — with the default `executable_only` — a target in a
|
|
427
|
+
non-executable section) and records why in `warnings`. Work is bounded by
|
|
428
|
+
`max_entries` (default 256, hard ceiling 4096) plus one pointer-sized peek that
|
|
429
|
+
decides `truncated`. Slot and summary evidence use
|
|
430
|
+
`jump_table.pointer_array` / `absolute_pointers` at `medium` confidence: a
|
|
431
|
+
pointer array found by shape is evidence of a dispatch table, not proof that a
|
|
432
|
+
dispatcher indexes it.
|
|
433
|
+
- Opt-in indirect resolution in the recursive CFG
|
|
434
|
+
(`codevanta.analysis.cfg.build_recursive_cfg`): `resolve_indirect=False`
|
|
435
|
+
(default) keeps phase 2e behaviour byte-for-byte; when enabled, a
|
|
436
|
+
block-terminating `jmp qword ptr [rip + disp]` / `jmp qword ptr [abs]` (and
|
|
437
|
+
the `dword` 32-bit forms) is turned into a table address and decoded with the
|
|
438
|
+
jump-table resolver, and its executable, file-backed targets become sorted,
|
|
439
|
+
deduplicated successors that the traversal follows. Bounded by
|
|
440
|
+
`max_jump_tables` (default 8) and `max_table_entries` (default 256, ceiling
|
|
441
|
+
4096); register/indexed operands, unreadable tables, and tables past the budget
|
|
442
|
+
leave the edge out and are reported in `FunctionCFG.warnings`, prefixed with
|
|
443
|
+
the block start. Evidence stays `cfg.recursive_traversal` at `medium` and its
|
|
444
|
+
detail names how many tables and slots were resolved when the option is on.
|
|
445
|
+
- CLI: `codevanta jump-table PATH --address ADDR [--entries N]
|
|
446
|
+
[--executable-only/--no-executable-only] [--json]` prints an index/target
|
|
447
|
+
table with a pointer-size, entry-count, and truncated footer plus one warning
|
|
448
|
+
per stop reason, and exits `2` on an invalid address, a rejected budget, or a
|
|
449
|
+
table that cannot be read.
|
|
450
|
+
- CLI: `codevanta cfg --resolve-indirect`, accepted only with
|
|
451
|
+
`--traversal recursive` (the linear sweep exits `2` instead of silently
|
|
452
|
+
ignoring it). The other cfg output and options are unchanged, and the
|
|
453
|
+
jump-table budgets stay at their library defaults on the command line.
|
|
454
|
+
- Interactive offline viewer (`codevanta.analysis.html_view`): the HTML export
|
|
455
|
+
now carries one inline, dependency-free script adding drag-to-pan (via
|
|
456
|
+
`viewBox`), wheel zoom anchored at the pointer and clamped to 0.2x–8x, a label
|
|
457
|
+
search that highlights hits and dims the rest (Esc clears it), and per-kind
|
|
458
|
+
filter checkboxes (function/entry_point, call_site, import/external). Still no
|
|
459
|
+
CDN, no web font, and no `http(s)` reference anywhere; the static view renders
|
|
460
|
+
in full with scripting disabled, and output remains deterministic and escaped.
|
|
461
|
+
- Structural binary diff (`codevanta.analysis.binary_diff`): `diff_infos` /
|
|
462
|
+
`diff_binaries` compare two binaries as metadata only — functions matched by
|
|
463
|
+
address (with `name`/`size` compared) then by name (so a relocated function is
|
|
464
|
+
one `changed` entry, not a removal plus an addition), sections and exports by
|
|
465
|
+
name, imports by `(library, name)`. `max_functions` defaults to 5000, is
|
|
466
|
+
clamped to 1..20000, keeps the lowest addresses when it truncates, and records
|
|
467
|
+
the clamp. Evidence uses `binary_diff.structural` / `metadata_compare` at
|
|
468
|
+
`medium` confidence with the explicit caveat that no instruction bytes were
|
|
469
|
+
read, compared, or executed, so `identical` means "no structural delta was
|
|
470
|
+
observed", never "the code is equivalent".
|
|
471
|
+
- CLI: `codevanta diff LEFT RIGHT [--max-functions N] [--json]` prints an
|
|
472
|
+
`Identical` header, per-kind delta counts, the named section/import/export
|
|
473
|
+
deltas, and a function-delta table sorted by address. A difference exits `0`;
|
|
474
|
+
only load or input errors exit `2`.
|
|
475
|
+
- Documentation: `docs/phase2f.md` covers the jump-table resolver and its stop
|
|
476
|
+
rules, CFG opt-in resolution and its warnings, the interactive viewer, the
|
|
477
|
+
structural diff and its matching rules, the CLI surface, confidence, limits,
|
|
478
|
+
and next steps.
|
|
479
|
+
- Relative jump tables (`codevanta.analysis.jump_tables`): `resolve_jump_table`
|
|
480
|
+
gained `mode="absolute"|"relative"` (case-insensitive, `MODE_RELATIVE`) plus
|
|
481
|
+
`base_address`. Relative mode reads each 4-byte slot as a *signed* 32-bit
|
|
482
|
+
offset and resolves `target = base_address + offset`, so position-independent
|
|
483
|
+
switch tables work; an explicit `pointer_size` other than 4 is rejected, and a
|
|
484
|
+
missing base raises `AnalysisError` instead of guessing one. A zero offset is a
|
|
485
|
+
real entry pointing at the base and no longer terminates anything — only the
|
|
486
|
+
shared stop rules (target outside `min_target`/`max_target`, unmapped target,
|
|
487
|
+
or a non-executable target under `executable_only`) end the scan, and each is
|
|
488
|
+
reported in `warnings` exactly as in absolute mode. `truncated` still requires
|
|
489
|
+
the entry ceiling plus a further usable slot. Evidence uses
|
|
490
|
+
`jump_table.relative_offsets` / `signed_offsets` at `medium` confidence and
|
|
491
|
+
records the base on every record; absolute mode keeps
|
|
492
|
+
`jump_table.pointer_array` / `absolute_pointers` and byte-identical output.
|
|
493
|
+
- Function fingerprints (`codevanta.analysis.function_hash`, new):
|
|
494
|
+
`normalize_instruction` renders one instruction as `"<mnemonic> <operands>"`
|
|
495
|
+
with a lower-cased mnemonic and every numeric literal masked to `#` (hex before
|
|
496
|
+
bare decimals, digit-bearing register/operand names preserved, consecutive
|
|
497
|
+
masks collapsed, whitespace normalized). `function_fingerprint` disassembles a
|
|
498
|
+
bounded linear sweep from an address and returns a `FunctionFingerprint` whose
|
|
499
|
+
`digest` is the SHA-256 of the newline-joined normalized stream. The budget is
|
|
500
|
+
`max_instructions` (default 2000, accepted 1..100000) and the read window is
|
|
501
|
+
`max_instructions * 16` bytes clipped to the section, so `truncated` reports a
|
|
502
|
+
prefix-only digest. Evidence uses `function_hash.normalized` /
|
|
503
|
+
`normalized_linear_disassembly` at `medium` confidence and states the caveat:
|
|
504
|
+
this is a shape fingerprint, so equal digests are not byte equality and
|
|
505
|
+
different digests are not a semantic difference.
|
|
506
|
+
- Deep diff (`codevanta.analysis.binary_diff`): `deep=False` is now an explicit
|
|
507
|
+
argument on `diff_infos` and `diff_binaries`. When true, every
|
|
508
|
+
address-matched function pair is fingerprinted on both sides
|
|
509
|
+
(`DEEP_MAX_INSTRUCTIONS` = 2000, equal to the standalone default); a differing
|
|
510
|
+
digest makes the entry `changed` with a content note and both digests, a
|
|
511
|
+
metadata-only change whose digests match says so, and `FunctionDelta` gained
|
|
512
|
+
`left_digest` / `right_digest` (always both set or both `None`, serialized by
|
|
513
|
+
`to_dict()` when present). Matching rules, structural-only output, and the
|
|
514
|
+
`added`/`removed`/`changed` semantics are unchanged; relocated (name-matched)
|
|
515
|
+
and one-sided functions are not fingerprinted. Deep mode is bounded by
|
|
516
|
+
`MAX_DEEP_FINGERPRINTS=1000` address-matched pairs per side (lowest addresses)
|
|
517
|
+
and reports truncated, capped, and failed fingerprints through `warnings`
|
|
518
|
+
instead of raising; its top-level evidence switches to the content caveat and
|
|
519
|
+
records `deep`, `deep_instructions`, and `max_deep_fingerprints`.
|
|
520
|
+
- CLI: `codevanta jump-table --mode absolute|relative --base ADDR`.
|
|
521
|
+
`--mode` is optional, case-insensitive, and validated before the file loads
|
|
522
|
+
(an unknown layout exits `2`); omitting it keeps the phase 2f absolute
|
|
523
|
+
behaviour and the unchanged footer. `--base` uses the same decimal/`0x` parser
|
|
524
|
+
as `--address` and rejects unparsable or negative values with `invalid base`.
|
|
525
|
+
`--mode relative` without `--base` exits `2` instead of inventing an origin,
|
|
526
|
+
and the footer gains `mode: <layout>` only when `--mode` was given plus
|
|
527
|
+
`base: 0x…` only for relative tables. `--json` output is the resolver's
|
|
528
|
+
`to_dict()` unchanged.
|
|
529
|
+
- CLI: `codevanta diff --deep` passes `deep=True` to `diff_binaries`. The
|
|
530
|
+
function-delta table gains a `Digest (left -> right)` column only when a delta
|
|
531
|
+
carries digests, showing the first 12 hex characters of each side's SHA-256
|
|
532
|
+
joined by ` -> ` and `-` for `added`/`removed` or over-ceiling pairs; a dim
|
|
533
|
+
footer states that equal digests are not byte equality. `--json` keeps the
|
|
534
|
+
existing document shape with `left_digest`/`right_digest` inside
|
|
535
|
+
`functions_changed` and `evidence.extra.deep`, and exit codes are unchanged.
|
|
536
|
+
- Documentation: `docs/phase2g.md` covers the relative table layout and its
|
|
537
|
+
zero-offset rule, the shared stop rules, fingerprint normalization, digests,
|
|
538
|
+
bounds and caveats, deep-diff behaviour and bounds, the `--mode`/`--base` and
|
|
539
|
+
`--deep` CLI reference, confidence, limits, and next steps.
|
|
540
|
+
- Struct-field suggestions (`codevanta.analysis.type_recovery`):
|
|
541
|
+
`suggest_struct(path, info, start, max_instructions=2000, max_fields=64,
|
|
542
|
+
include_stack=False)` disassembles a bounded window and reads the memory
|
|
543
|
+
operands of every decoded instruction. Only `[reg]`, `[reg + 0xNN]` and
|
|
544
|
+
`[reg - 0xNN]` are understood — a decimal displacement is accepted like a hex
|
|
545
|
+
one — while an index register, a scale or a segment prefix is skipped rather
|
|
546
|
+
than guessed, and `lea`, `nop` and `prefetch*` are skipped because they do not
|
|
547
|
+
access memory. Widths come from the `byte`/`word`/`dword`/`qword` keyword
|
|
548
|
+
(1/2/4/8), falling back to the pointer width of the binary. Direction is a
|
|
549
|
+
write when the memory operand is first and a read otherwise, except
|
|
550
|
+
`cmp`/`cmps`/`cmpsb`/`cmpsw`/`cmpsd`/`cmpsq`/`test`, which are always reads.
|
|
551
|
+
Accesses aggregate per offset into `SuggestedField` records (dominant width,
|
|
552
|
+
ties to the wider access; read/write totals across widths; ascending offsets),
|
|
553
|
+
stack registers (`sp`/`esp`/`rsp`/`bp`/`ebp`/`rbp`) are dropped unless
|
|
554
|
+
`include_stack=True`, and `max_fields` sets `truncated` with lower offsets
|
|
555
|
+
winning. Every record is `medium` confidence
|
|
556
|
+
(`type_recovery.memory_operand`/`operand_scan`,
|
|
557
|
+
`type_recovery.suggestion`/`operand_aggregation`) and states that fields are
|
|
558
|
+
heuristic from operand shapes, not verified types.
|
|
559
|
+
- MCP tools: `codevanta_diff` (structural binary diff with optional `deep`
|
|
560
|
+
fingerprinting, `max_functions` default 5000), `codevanta_jump_table`
|
|
561
|
+
(absolute by default, `mode="relative"` requires `base`, `entries` default 256
|
|
562
|
+
and values above 4096 are rejected), and `codevanta_function_hash` (normalized-shape SHA-256
|
|
563
|
+
with `max_instructions` default 2000) bring the server to thirteen tools.
|
|
564
|
+
Arguments are validated the same way as the CLI (limits `1 .. 20000`,
|
|
565
|
+
non-negative addresses, booleans rejected for flags) and analysis failures
|
|
566
|
+
raise `AnalysisError`; struct suggestions were not exposed over MCP until
|
|
567
|
+
`codevanta_struct` in phase 2i.
|
|
568
|
+
- CLI: `codevanta struct PATH` with `--address`/`-a`, `--function`/`-f`,
|
|
569
|
+
`--max-instructions`, `--max-fields`, `--include-stack`, and `--json`. The
|
|
570
|
+
start address resolves as address, then function name, then entry point. The
|
|
571
|
+
table lists `Offset`, `Width`, `Reads`, and `Writes` per suggested field with a
|
|
572
|
+
footer for the base registers the sweep saw, the field and instruction counts,
|
|
573
|
+
the applied field cap, and the truncation flag, followed by one `warning:` line
|
|
574
|
+
when the cap was reached or no field was decoded (the latter also points at
|
|
575
|
+
`--include-stack`). Base registers come from a second bounded pass over the
|
|
576
|
+
same window, so they are the registers in use, not a per-field attribution.
|
|
577
|
+
`--json` prints the suggestion's `to_dict()`, and load or input errors (missing
|
|
578
|
+
file, invalid address or limits, unknown function, non-executable address)
|
|
579
|
+
exit `2`.
|
|
580
|
+
- CI: the test matrix in `.github/workflows/ci.yml` is now `"3.11"`, `"3.13"`,
|
|
581
|
+
and `"3.14"` (`fail-fast: false`), with the wheel build and clean-venv smoke
|
|
582
|
+
test still on 3.13. Python 3.14 was previously held back because it needed
|
|
583
|
+
published `lief` wheels; `lief` now ships `cp314` wheels (`lief 1.0.0`), so the
|
|
584
|
+
`lief>=0.14` requirement resolves to a 3.14-capable release.
|
|
585
|
+
`requires-python` stays `>=3.10`, so 3.10 and 3.12 remain supported without
|
|
586
|
+
being exercised by CI, and `docs/repository-settings.md` was updated to match.
|
|
587
|
+
- Documentation: `docs/phase2h.md` covers the operand shapes that are and are not
|
|
588
|
+
recovered, the stack-register rule, width and direction rules, per-offset
|
|
589
|
+
aggregation (including that offsets from different base registers share one
|
|
590
|
+
bucket), the `medium` evidence labels, bounds and determinism, the thirteen
|
|
591
|
+
MCP tools, the `codevanta struct` CLI reference, the new CI matrix, confidence,
|
|
592
|
+
limitations, and next steps.
|
|
593
|
+
- Struct layout hints (`codevanta.analysis.type_recovery`): `StructSuggestion`
|
|
594
|
+
gains `alignment` and `array_groups`, both derived from the fields the
|
|
595
|
+
suggestion actually reports, so the `max_fields` cap can change them.
|
|
596
|
+
`alignment` is the greatest common divisor of the absolute non-zero offsets —
|
|
597
|
+
offset `0` is ignored because it is a multiple of every candidate — and is
|
|
598
|
+
`None` when fewer than two distinct non-zero offsets were seen. `array_groups`
|
|
599
|
+
holds `ArrayGroup(offset, stride, count, width, evidence)` records for maximal
|
|
600
|
+
runs of at least three consecutive fields (in ascending offset order) that
|
|
601
|
+
share one width and one stride; a run stops at the first width or stride
|
|
602
|
+
change, the scan then restarts at the next field so no field belongs to two
|
|
603
|
+
groups, and `span` is `stride * (count - 1) + width`. Every group is `medium`
|
|
604
|
+
confidence (`type_recovery.array_run` / `stride_run_detection`) and its detail
|
|
605
|
+
states that the heuristic cannot tell an array from repeated struct members.
|
|
606
|
+
- Function similarity (`codevanta.analysis.function_hash`):
|
|
607
|
+
`normalized_tokens(path, info, address, max_instructions=2000)` returns the
|
|
608
|
+
tokens the fingerprint hashes — one shared sweep, so the digest and the tokens
|
|
609
|
+
cannot disagree — and `function_similarity(...)` compares two regions as
|
|
610
|
+
multisets of those tokens (`shared = sum(min)`, `union = sum(max)`,
|
|
611
|
+
`score = shared / union`, in `0.0 .. 1.0`), returning
|
|
612
|
+
`SimilarityResult(score, shared, left_total, right_total, evidence, warnings)`
|
|
613
|
+
with the score rounded to four decimals in `to_dict()`. Evidence is
|
|
614
|
+
`function_hash.similarity` / `token_multiset` at `medium` confidence with a
|
|
615
|
+
shape-not-semantics caveat; a truncated side adds a warning, and a side with no
|
|
616
|
+
tokens raises `AnalysisError` instead of scoring `0.0`. Work is bounded to
|
|
617
|
+
`2 * max_instructions` instructions and `2 * max_instructions * 16` bytes.
|
|
618
|
+
- MCP: `codevanta_struct` exposes `suggest_struct` with `address`/`function`/
|
|
619
|
+
`max_instructions`/`max_fields`/`include_stack` and returns the suggestion's
|
|
620
|
+
`to_dict()` unchanged, including `alignment` and `array_groups` — fourteen
|
|
621
|
+
tools total. Bounded arguments, `AnalysisError` on bad input, and the same
|
|
622
|
+
evidence semantics as the CLI. Function similarity is not exposed over MCP.
|
|
623
|
+
- CLI: `codevanta struct` prints `alignment: <n>` (or `alignment: unknown` when
|
|
624
|
+
none could be derived) plus one
|
|
625
|
+
`array at 0x10: 3 x 8-byte, stride 8` line per array candidate — or
|
|
626
|
+
`arrays: none` when no run was found — between the counts line and the
|
|
627
|
+
warning lines. `--json` output carries `alignment` and `array_groups`
|
|
628
|
+
unchanged.
|
|
629
|
+
- Documentation: `docs/phase2i.md` covers the gcd alignment rule and its `None`
|
|
630
|
+
cases, the maximal-run array heuristic and its evidence, the Ruzicka multiset
|
|
631
|
+
similarity and its normalization, the fourteen MCP tools including
|
|
632
|
+
`codevanta_struct`, the `codevanta struct` footer, the `medium` labels, bounds
|
|
633
|
+
and determinism, limitations, and next steps.
|
|
634
|
+
- Semantic diff pairing (`codevanta.analysis.binary_diff`): `diff_infos` and
|
|
635
|
+
`diff_binaries` gain `semantic=False` and
|
|
636
|
+
`threshold=DEFAULT_SEMANTIC_THRESHOLD` (0.9). With `semantic=True` the
|
|
637
|
+
candidates that address and name matching left unpaired are scored with
|
|
638
|
+
`function_similarity`, sorted by descending score (ties broken by left then
|
|
639
|
+
right address) and taken greedily while the score is at least `threshold` and
|
|
640
|
+
neither side has been used. Each accepted pair becomes one `changed` delta at
|
|
641
|
+
the right-hand address with `FunctionDelta.similarity` rounded to four decimals
|
|
642
|
+
and an evidence detail naming the score, the threshold, both addresses, and the
|
|
643
|
+
fact that a shape overlap is not proof of equal code; structurally matched
|
|
644
|
+
deltas keep `similarity=None` and omit the key from `to_dict()`. A pairing can
|
|
645
|
+
only replace a remove-plus-add pair, never an address or name match, so
|
|
646
|
+
structural mode and the `identical` flag semantics are unchanged. The pass is
|
|
647
|
+
bounded by `MAX_SEMANTIC_PAIRS` (4096): above that leftover cross product the
|
|
648
|
+
whole pass is skipped with a warning and every leftover keeps its
|
|
649
|
+
`added`/`removed` verdict, and each score decodes at most
|
|
650
|
+
`SEMANTIC_MAX_INSTRUCTIONS` (2000) instructions per side. A pair that cannot be
|
|
651
|
+
scored is skipped and summarised in a warning, `threshold` is validated in every
|
|
652
|
+
mode (finite number in `0.0 .. 1.0`, else `AnalysisError`), and the comparison
|
|
653
|
+
evidence gains `semantic`, `semantic_threshold`, `semantic_instructions`, and
|
|
654
|
+
`max_semantic_pairs`.
|
|
655
|
+
- Signedness hints (`codevanta.analysis.type_recovery`): `SuggestedField` gains
|
|
656
|
+
`signed: bool | None`, read from the mnemonic of each access —
|
|
657
|
+
`movsx`/`movsxd`/`sar`/`idiv`/`imul` mark signed,
|
|
658
|
+
`movzx`/`shr`/`div`/`mul` mark unsigned, every other mnemonic carries no hint.
|
|
659
|
+
Hints are aggregated per offset: one kind gives `True`/`False`, no hint gives
|
|
660
|
+
`None`, and a contradiction gives `None` plus one
|
|
661
|
+
`SignednessConflictWarning` (a `UserWarning`) naming the offset. `signed` is
|
|
662
|
+
omitted from `SuggestedField.to_dict()` when it is `None`, so an unclassified
|
|
663
|
+
field's JSON is unchanged, while the field evidence detail now names the hint
|
|
664
|
+
(`signedness signed|unsigned|unknown`) and `evidence.extra["signed"]` carries
|
|
665
|
+
the same value. `None` means "no evidence", never "safe".
|
|
666
|
+
- MCP: `codevanta_similarity` takes `path_a`, `address_a`, `path_b`, `address_b`,
|
|
667
|
+
and `max_instructions=2000`, exposes `function_similarity`, and returns its
|
|
668
|
+
`to_dict()` (`score`, `shared`, `left_total`, `right_total`, `union`,
|
|
669
|
+
`warnings`, evidence at `function_hash.similarity` / `token_multiset`,
|
|
670
|
+
`medium`). Addresses must be non-negative integers and are validated per side,
|
|
671
|
+
`max_instructions` stays in `1 .. 20000`, and bad input raises
|
|
672
|
+
`AnalysisError` — fifteen tools total. `codevanta_diff` keeps only
|
|
673
|
+
`max_functions` and `deep`, so the semantic pass has no MCP argument yet.
|
|
674
|
+
- CLI: `codevanta diff --semantic` / `--threshold`. The threshold defaults to
|
|
675
|
+
`DEFAULT_SEMANTIC_THRESHOLD` (0.9), is validated before either binary is loaded
|
|
676
|
+
(a value outside `0.0 .. 1.0`, `nan`, or `inf` exits 2 with
|
|
677
|
+
`invalid threshold ...`), and is checked even without `--semantic`.
|
|
678
|
+
`--semantic` passes `semantic=True` / `threshold` to `diff_binaries`, so a
|
|
679
|
+
relocated function becomes one `changed` row instead of a removal plus an
|
|
680
|
+
addition, and each scored pair prints one dim
|
|
681
|
+
`semantic pair 0x401000 -> 0x401010 (<name>): similarity 1.0000` line after the
|
|
682
|
+
otherwise unchanged function table (no `Similarity` column), followed by a dim
|
|
683
|
+
caveat that a pairing is not proof of equal code. `AnalysisError` from the
|
|
684
|
+
analysis layer now also maps to exit 2, and `--json` carries `similarity` plus
|
|
685
|
+
the `semantic` evidence keys.
|
|
686
|
+
- CLI: `codevanta struct` closes its footer with a `signed:` line listing
|
|
687
|
+
`<offset> signed|unsigned` for every field whose mnemonics carried a hint, in
|
|
688
|
+
table order — for example `signed: 0x10 signed, 0x18 unsigned` — or
|
|
689
|
+
`signed: none` when nothing hinted (`none` means "no evidence", never "safe").
|
|
690
|
+
The four `Offset`/`Width`/`Reads`/`Writes` columns and the phase 2h/2i footer
|
|
691
|
+
lines are unchanged, and contradictory offsets stay absent from the line because
|
|
692
|
+
the analysis layer already collapsed them to `signed=None`.
|
|
693
|
+
- Documentation: `docs/phase2j.md` covers the semantic pairing rule, its two
|
|
694
|
+
bounds and warnings, threshold validation, the semantic caveat and evidence
|
|
695
|
+
keys, the fifteen MCP tools including `codevanta_similarity`, the mnemonic
|
|
696
|
+
signedness sets and their `None`/conflict semantics, the `diff --semantic` /
|
|
697
|
+
`--threshold` switches and the `struct` `signed:` footer, limitations, and next
|
|
698
|
+
steps.
|
|
699
|
+
- MCP `codevanta_diff` (`codevanta.mcp_server.tools` / `server`): gains
|
|
700
|
+
`deep=False`, `semantic=False`, and `threshold=DEFAULT_SEMANTIC_THRESHOLD`
|
|
701
|
+
(0.9) next to `max_functions`, so the phase 2j pairing pass is reachable from an
|
|
702
|
+
agent in one call instead of a `codevanta_similarity` call per address pair.
|
|
703
|
+
`threshold` is a finite number in `0.0 .. 1.0` and is validated in every mode —
|
|
704
|
+
a boolean, a non-numeric value, `NaN`, `inf`, or an out-of-range value raises
|
|
705
|
+
`AnalysisError` even with `semantic=False`, and only the two paths are checked
|
|
706
|
+
first, so a bad value never costs a binary load. `deep`/`semantic` must be real
|
|
707
|
+
booleans. The returned payload is unchanged except that semantic mode adds
|
|
708
|
+
`semantic`, `semantic_threshold`, `semantic_instructions`, and
|
|
709
|
+
`max_semantic_pairs` to the comparison evidence `extra` and `similarity` to each
|
|
710
|
+
paired `changed` entry. No tool was added or removed: the server stays at
|
|
711
|
+
fifteen tools.
|
|
712
|
+
- Cross-architecture guard (`codevanta.analysis.binary_diff`): the semantic pass
|
|
713
|
+
now requires both sides to describe the same instruction set. `_semantic_pairs`
|
|
714
|
+
compares `BinaryInfo.architecture` through the new `_comparable_architecture`
|
|
715
|
+
helper, which **trims surrounding whitespace and lower-cases the token only** and
|
|
716
|
+
maps the loader's literal `"unknown"` placeholder (`_UNKNOWN_ARCHITECTURE`) to an
|
|
717
|
+
empty token, so "not determined" is never treated as "a different target". There
|
|
718
|
+
is no alias table: a hand-built `"AMD64"` becomes `"amd64"` and a hand-built
|
|
719
|
+
`"x86-64"` stays `"x86-64"`, so both count as *different* from `"x86_64"` and
|
|
720
|
+
skip the pass. The token itself comes from the loader's per-format substring
|
|
721
|
+
helper, which does not cover every machine value: PE `MIPS`/`R4000` and ELF
|
|
722
|
+
`IA_64`/RISC-V/PowerPC/s390 headers all fall through to `"unknown"` (the PE
|
|
723
|
+
helper has no MIPS branch, the ELF helper no IA-64 branch), and since `unknown`
|
|
724
|
+
means "no claim", two `unknown` sides bypass the guard and are scored as if they
|
|
725
|
+
shared an instruction set. Two different identified tokens skip the pass
|
|
726
|
+
entirely with one warning naming both sides
|
|
727
|
+
(`semantic matching skipped: the two sides target different architectures (left
|
|
728
|
+
x86, right x86_64), ... every leftover keeps its added/removed verdict`); nothing
|
|
729
|
+
is decoded or scored, address and name matching plus every structural delta and
|
|
730
|
+
`deep=True` are untouched, and the gate runs before the `MAX_SEMANTIC_PAIRS`
|
|
731
|
+
ceiling so one explanation is reported instead of two. It runs *after* the
|
|
732
|
+
"nothing left to pair" early return, so a cross-architecture diff with an empty
|
|
733
|
+
leftover on one side emits no warning at all.
|
|
734
|
+
- Merged struct windows (`codevanta.analysis.type_recovery`): `suggest_structs`
|
|
735
|
+
folds several windows of one binary into one `StructSuggestion`. Each address is
|
|
736
|
+
decoded once with the shared `max_instructions` budget and `include_stack`
|
|
737
|
+
policy, and the per-window fields are merged into one field per offset — read
|
|
738
|
+
and write counters summed, `first_access` the earliest any window recorded,
|
|
739
|
+
width the **widest** per-window width with a `WidthConflictWarning` naming the
|
|
740
|
+
offset and every width seen when the windows disagree, and signedness folded
|
|
741
|
+
with the single-window rule — except that a contradiction seen *inside* one
|
|
742
|
+
window is not lost to a window that only hinted one way: each per-window field
|
|
743
|
+
records `signed_conflict` in its evidence, the merge checks it, and such an
|
|
744
|
+
offset keeps `signed=None` with a `SignednessConflictWarning` raised by the merge
|
|
745
|
+
too (whenever at least one window left a hint to compare against).
|
|
746
|
+
`alignment`, `array_groups`, and `truncated` are recomputed from the merged
|
|
747
|
+
fields, so a stride or alignment that only becomes visible across windows is
|
|
748
|
+
still reported; `truncated` is the **OR** of the merge cap and every window's own
|
|
749
|
+
`DEFAULT_MAX_FIELDS` cap, because a window that hit its own cap could not
|
|
750
|
+
contribute the offsets it dropped, and `window_truncated` lists the addresses of
|
|
751
|
+
the windows that hit theirs. Work is bounded by `max_windows`
|
|
752
|
+
(`MIN_MERGED_WINDOWS` 1 .. `MAX_MERGED_WINDOWS` 16, default 16), one decode per
|
|
753
|
+
window, `DEFAULT_MAX_FIELDS` per window, and `DEFAULT_MAX_FIELDS` merged offsets
|
|
754
|
+
— at most `max_windows * max_instructions` decoded instructions. Addresses must
|
|
755
|
+
be a non-empty collection of non-negative integers; duplicates
|
|
756
|
+
(`window address 0x... appears more than once`), empty lists, non-integer
|
|
757
|
+
entries, negative values, and over-limit counts raise `AnalysisError` before
|
|
758
|
+
anything is decoded. A *partial* overlap is allowed instead of rejected — looking
|
|
759
|
+
further into the same function is a legitimate second window — and announced with
|
|
760
|
+
`WindowOverlapWarning`, which notes that the shared instructions are then counted
|
|
761
|
+
by both windows, so the summed per-offset counters and `memory_access_count` are
|
|
762
|
+
upper bounds rather than totals. Addresses are sorted, so the result and the
|
|
763
|
+
warning order are identical for any permutation, `address` is the lowest merged
|
|
764
|
+
address, `instruction_count` is the sum over windows, `max_fields` is always
|
|
765
|
+
`DEFAULT_MAX_FIELDS`, and the merge is identifiable from
|
|
766
|
+
`type_recovery.merged_suggestion` / `type_recovery.merged_field` evidence
|
|
767
|
+
(`window_merge` method) carrying `window_count`, `windows`, `max_windows`,
|
|
768
|
+
`max_instructions`, `memory_access_count`, `window_truncated`, and per-field
|
|
769
|
+
`widths`/`signed_conflict`.
|
|
770
|
+
- CLI: `codevanta struct --address` / `-a` is repeatable. One address keeps the
|
|
771
|
+
phase 2i/2j single-window output, zero addresses keep the entry-point /
|
|
772
|
+
`--function` behaviour, and two or more addresses are merged through
|
|
773
|
+
`suggest_structs` — bounded by `STRUCT_MAX_WINDOWS` (`MAX_MERGED_WINDOWS`, 16),
|
|
774
|
+
sorted before decoding, so the output is identical for any order of the same
|
|
775
|
+
address set. A merged run prints the usual table plus one
|
|
776
|
+
`windows: N (0x.., 0x..)` footer line between the `instructions`/`max fields`/
|
|
777
|
+
`truncated` line and `alignment:`. Its `base registers:` value is **not** a
|
|
778
|
+
second sweep: the merge does not decode a window twice, the CLI reads the
|
|
779
|
+
registers out of the evidence the merge already produced
|
|
780
|
+
(`_struct_evidence_base_registers`), and because the phase 2k analysis layer
|
|
781
|
+
records no `base_registers` key there the footer prints
|
|
782
|
+
`base registers: not recorded by the merge` instead of a list (a single window
|
|
783
|
+
still re-sweeps itself through `_struct_base_registers` and does name them). A
|
|
784
|
+
truncated merge is told to split the windows — `field cap reached: the merged
|
|
785
|
+
sweep keeps at most 64 offsets, so split the windows into separate runs to see
|
|
786
|
+
the remaining offsets` — because the same command exits 2 on a non-default
|
|
787
|
+
`--max-fields` next to a merge, so its hint must not send the user to a rejected
|
|
788
|
+
flag. Because `--function` and `--max-fields` describe one window, more than one
|
|
789
|
+
`--address` together with either a `--function` or a non-default `--max-fields`
|
|
790
|
+
exits 2 with an explicit message instead of ignoring the option; duplicated
|
|
791
|
+
addresses, more than 16 addresses, and addresses outside an executable section
|
|
792
|
+
also exit 2 through the analysis layer's `AnalysisError`. `--json` stays
|
|
793
|
+
`StructSuggestion.to_dict()`, with `window_count`/`windows` in the evidence
|
|
794
|
+
`extra`.
|
|
795
|
+
- Documentation: `docs/phase2k.md` covers the new `codevanta_diff` arguments and
|
|
796
|
+
their validation, the architecture guard's token rules and warning, the
|
|
797
|
+
`suggest_structs` merge rules, address validation, bounds and evidence keys, the
|
|
798
|
+
repeatable `struct --address` switch with its `windows:` footer and exit-2
|
|
799
|
+
combinations, limitations, and next steps.
|
|
800
|
+
- `codevanta analyze` can now embed the phase 2 results in its JSON report under
|
|
801
|
+
two additive top-level keys, opt-in per section exactly like `--callgraph`:
|
|
802
|
+
`xrefs` (count, returned, truncated, items with per-item evidence, scanned
|
|
803
|
+
sections/bytes, scan-level truncation, skipped indirect calls, warnings) and
|
|
804
|
+
`cfg` (entry-point linear-sweep CFG with blocks, successors, and warnings).
|
|
805
|
+
Both degrade to warning-carrying empty results on binaries with no executable
|
|
806
|
+
content, and the provenance step chain gains `find_xrefs` and `build_cfg`.
|
|
807
|
+
New options: `--xrefs`, `--cfg`, `--xref-limit` (default 2000, capped at
|
|
808
|
+
200000) and `--json` (echo the payload when `--output` is used). Without the
|
|
809
|
+
new flags the report is exactly the phase 1 payload, so existing JSON
|
|
810
|
+
consumers are unaffected.
|
|
811
|
+
- Pseudo-C decompilation (`codevanta.analysis.decompile`, `codevanta decompile`):
|
|
812
|
+
`decompile_function(path, info, address, max_instructions=2000,
|
|
813
|
+
max_blocks=500)` renders one function as readable C from the recursive CFG's
|
|
814
|
+
blocks, so nothing is decoded twice and nothing is executed. Argument count
|
|
815
|
+
comes from the ABI argument registers read before the first `call` (capped at
|
|
816
|
+
six, `mov rax, rdi` reads `rdi` while `mov rdi, rax` does not); argument
|
|
817
|
+
registers become `a0`..`a5`, frame slots become `local_0x8`/`arg_0x8`, other
|
|
818
|
+
memory operands become `*(u64*)(...)` with the binary's pointer width. Only
|
|
819
|
+
mnemonic-justified statements are emitted (`mov`/`lea`, compound and unary
|
|
820
|
+
assignments) so nothing is invented, and every emitted line keeps its raw
|
|
821
|
+
instruction text as a trailing comment. Control flow is structured into
|
|
822
|
+
`if`/`else`, `while (1)` with `continue;`/`break`, and `goto LABEL_<addr>;`
|
|
823
|
+
for whatever cannot be nested; conditions come from `cmp`/`test` plus the
|
|
824
|
+
branch mnemonic, with `(unsigned)` forms for `jb`..`ja`. Every rendering is
|
|
825
|
+
`Confidence.LOW` with `source="decompile.pseudo_c"` /
|
|
826
|
+
`method="mnemonic_heuristic"`, repeated in a header comment block inside the
|
|
827
|
+
text. Budgets are `2000`/`500` by default and capped at
|
|
828
|
+
`MAX_MAX_INSTRUCTIONS`/`MAX_MAX_BLOCKS` (`100000`); a cut-short rendering sets
|
|
829
|
+
`truncated` and warns instead of failing, while a non-executable address, an
|
|
830
|
+
out-of-range budget and a region that decodes to nothing raise `AnalysisError`
|
|
831
|
+
(exit `2`).
|
|
832
|
+
- Pluggable decompiler backends (`codevanta.analysis.decompilers`): `ghidra`
|
|
833
|
+
(pyghidra + `GHIDRA_INSTALL_DIR`), `angr` and the always-available `native`
|
|
834
|
+
renderer sit behind one `decompile_function(...) -> BackendResult` entry point.
|
|
835
|
+
`BackendAvailability` reports `available`, a non-empty `reason` and the
|
|
836
|
+
`requires` tuple per provider and never raises or installs anything, so a
|
|
837
|
+
missing package is a probe result, never an `ImportError`. `backend="auto"`
|
|
838
|
+
(the default) prefers `ghidra` then `angr` and falls back to `native` with one
|
|
839
|
+
warning per skipped backend, while `backend="angr"`/`"ghidra"` are strict and
|
|
840
|
+
fail with the probe's reason and install hint. Budgets are validated once,
|
|
841
|
+
before any backend runs. `codevanta decompile --backend` and `--output` wrap
|
|
842
|
+
it, and neither Ghidra nor angr is a CODEVANTA dependency.
|
|
843
|
+
- Bounded offline emulation (`codevanta.analysis.emulate`, `codevanta emulate`):
|
|
844
|
+
`emulate_function` copies the file's bytes into a *private* unicorn CPU and
|
|
845
|
+
runs them there, so nothing is ever executed on the host and no host syscall,
|
|
846
|
+
process or library is invoked. Only ABI entry state is seeded (`rip` at the
|
|
847
|
+
address, `rsp`/`rbp` in a stack region at `STACK_BASE_64`/`STACK_BASE_32`);
|
|
848
|
+
inputs arrive through `initial_registers`/`initial_memory` (`--set`, `--mem`),
|
|
849
|
+
never by guessing. Every run ends with a stated `stopped_reason` — `return`,
|
|
850
|
+
`budget`, `unmapped`, `unsupported`, `error` — plus a warning, and a value
|
|
851
|
+
left in `rax` after a budget stop is reported as a register rather than a
|
|
852
|
+
return value. Work is bounded by `max_instructions` (2000, ceiling 100000),
|
|
853
|
+
`stack_size` (64 KiB, ceiling 16 MiB), `MAX_SECTION_BYTES`,
|
|
854
|
+
`MAX_TOTAL_MAPPED` (64 MiB), `MAX_INITIAL_BLOCKS`/`MAX_INITIAL_BLOCK_BYTES`
|
|
855
|
+
and `MAX_MEMORY_WRITES` (256, overflow counted and warned).
|
|
856
|
+
`resolve_indirect_target` stops at the first indirect jump/call and returns
|
|
857
|
+
its target, with `None` meaning "not resolved" rather than "no target";
|
|
858
|
+
`xor_bytes` and `decrypt_string_loop` (xor/add/sub, optional per-byte
|
|
859
|
+
`key_step`, C-style terminators) apply a recovered key without executing
|
|
860
|
+
anything. Emulation needs the optional `unicorn` engine and exits `2` with the
|
|
861
|
+
reason when it is missing; only x86 and x86-64 are emulated, other
|
|
862
|
+
architectures report `unsupported` instead of a guess.
|
|
863
|
+
- Phase 3 surfaces: the sixteenth MCP tool `codevanta_decompile` (same payload
|
|
864
|
+
as `BackendResult.to_dict()` plus the resolved `path`/`address` and the block
|
|
865
|
+
count, with an unavailable strict backend surfacing as a tool error carrying
|
|
866
|
+
the probe's reason), and the dashboard's tenth route `/api/decompile` with its
|
|
867
|
+
form on the page — `backend=auto` by default, `address` defaulting to the
|
|
868
|
+
entry point, and `400` naming the backend when a strict one is unavailable.
|
|
869
|
+
- Documentation: `docs/phase3.md` covers the rendering rules, the ABI argument
|
|
870
|
+
inference, the confidence/evidence contract and exact bounds of the native
|
|
871
|
+
renderer, every emulation stop reason and limit, the provider layer's
|
|
872
|
+
selection policy and the availability matrix observed in this environment,
|
|
873
|
+
how to install `angr`/`pyghidra`+Ghidra to unlock real C output, `docs/api.md`
|
|
874
|
+
gains the three new modules' reference sections, and the README usage block
|
|
875
|
+
and docs index list the two new commands and the phase 3 document.
|
|
876
|
+
|
|
877
|
+
### Notes
|
|
878
|
+
|
|
879
|
+
- Phase 2 is in progress: the CFG uses a linear sweep by default and an optional
|
|
880
|
+
bounded recursive traversal (`--traversal recursive`), with direct branch
|
|
881
|
+
targets plus absolute and RIP-relative pointer slots only; jump tables are
|
|
882
|
+
absolute 4/8-byte pointer arrays, or signed 32-bit offset tables once a base is
|
|
883
|
+
supplied through the API (CFG indirect resolution still uses the absolute
|
|
884
|
+
layout), so register and indexed dispatch stay unresolved; no call following,
|
|
885
|
+
no cross-function analysis, no slot dereferencing, and no debugger yet. The HTML
|
|
886
|
+
export is interactive but still a single offline file, and the binary diff is
|
|
887
|
+
structural (metadata) by default with an opt-in normalized-shape fingerprint
|
|
888
|
+
comparison for address-matched functions — still no byte-level or semantic
|
|
889
|
+
equivalence claim. The phase 2g switches are `jump-table --mode
|
|
890
|
+
absolute|relative --base ADDR`, the new `function_fingerprint` API, and
|
|
891
|
+
`diff --deep`; the CFG's indirect resolution still uses the absolute layout.
|
|
892
|
+
Phase 2h adds heuristic struct-field suggestions (`suggest_struct` and
|
|
893
|
+
`codevanta struct`), which report an offset, an access width, and read/write
|
|
894
|
+
counts only: there is no type name, and offsets
|
|
895
|
+
reached through different base registers share one bucket (cross-window merging
|
|
896
|
+
arrives in phase 2k as `suggest_structs`, and even then stays inside one
|
|
897
|
+
binary). Phase 2i adds
|
|
898
|
+
`alignment` (the gcd of the reported non-zero offsets, `None` when fewer than
|
|
899
|
+
two exist) and `array_groups` (maximal runs of at least three equally wide,
|
|
900
|
+
equally spaced fields) to those suggestions — both are `medium` heuristics over
|
|
901
|
+
one bounded window, so they cannot prove padding, a struct size or a declared
|
|
902
|
+
array — plus `function_similarity`, a multiset overlap of normalized tokens, and
|
|
903
|
+
the `codevanta_struct` MCP tool. Phase 2j adds the opt-in semantic pairing pass
|
|
904
|
+
on the binary diff (`semantic=True`, `threshold` defaulting to `0.9`, greedy,
|
|
905
|
+
capped at 4096 leftover pairs), the `codevanta_similarity` MCP tool (fifteen
|
|
906
|
+
tools), and the `signed` mnemonic hint on struct fields, reachable as
|
|
907
|
+
`diff --semantic --threshold` and as the `codevanta struct` footer `signed:`
|
|
908
|
+
line. Phase 2k adds `deep`/`semantic`/`threshold` to the `codevanta_diff` MCP
|
|
909
|
+
tool (still fifteen tools), the semantic pass's cross-architecture guard, and
|
|
910
|
+
the `suggest_structs` merged-window API over up to `MAX_MERGED_WINDOWS` (16)
|
|
911
|
+
addresses of one binary — one field per offset, widest width winning with a
|
|
912
|
+
`WidthConflictWarning`, layout hints recomputed, reachable from the CLI by
|
|
913
|
+
repeating `--address` — while `codevanta_struct` still takes a single address.
|
|
914
|
+
See `docs/phase2.md`, `docs/phase2b.md`, `docs/phase2c.md`,
|
|
915
|
+
`docs/phase2d.md`, `docs/phase2e.md`, `docs/phase2f.md`, `docs/phase2g.md`,
|
|
916
|
+
`docs/phase2h.md`, `docs/phase2i.md`, `docs/phase2j.md`, and
|
|
917
|
+
`docs/phase2k.md`.
|
|
918
|
+
- Node boundaries and caller attribution are heuristics, not proven function
|
|
919
|
+
extents. Sessions and annotations live in one local SQLite file with no schema
|
|
920
|
+
migrations yet.
|
|
921
|
+
|
|
922
|
+
## [0.1.0] - 2026-11-12
|
|
923
|
+
|
|
924
|
+
### Added
|
|
925
|
+
|
|
926
|
+
- PE and ELF loaders (format, architecture, bitness, endianness, entry point,
|
|
927
|
+
image base, sections, imports, exports, symbols) backed by LIEF.
|
|
928
|
+
- Capstone-backed x86/x64 disassembly of executable sections and the entry point.
|
|
929
|
+
- Function discovery from exports, function symbols, and the entry point, each
|
|
930
|
+
with a detection source and confidence level.
|
|
931
|
+
- ASCII and UTF-16LE string extraction with file offset and mapped virtual address.
|
|
932
|
+
- Evidence layer: every finding records source, method, and confidence.
|
|
933
|
+
- IR layer: backend-neutral `IRInstruction` plus an adapter registry for future
|
|
934
|
+
VEX/P-Code lifting.
|
|
935
|
+
- CLI: `info`, `functions`, `strings`, `disasm`, `analyze` with JSON export.
|
|
936
|
+
- MCP server (`codevanta mcp serve`, stdio) exposing five tools to coding agents.
|
|
937
|
+
- Guided setup for Claude Code, Cursor, Cline, and VS Code with plan preview,
|
|
938
|
+
unified diffs, `*.codevanta.bak` backups, idempotent writes, verification, and
|
|
939
|
+
`codevanta uninstall`.
|
|
940
|
+
- One-line bootstrap installers for Windows (PowerShell) and macOS/Linux (bash).
|
|
941
|
+
- 55 unit and integration tests; Ruff linting in CI.
|
|
942
|
+
|
|
943
|
+
### Notes
|
|
944
|
+
|
|
945
|
+
- Static analysis only: target binaries are never executed.
|
|
946
|
+
- PyVEX/angr, Z3, and Unicorn are intentionally deferred to a later phase.
|
|
947
|
+
|
|
948
|
+
[Unreleased]: https://github.com/kevindoni/CODEVANTA/compare/v0.1.0...HEAD
|
|
949
|
+
[0.1.0]: https://github.com/kevindoni/CODEVANTA/releases/tag/v0.1.0
|