komaru 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- komaru-0.1.0/.gitignore +8 -0
- komaru-0.1.0/NOTES.md +138 -0
- komaru-0.1.0/PKG-INFO +125 -0
- komaru-0.1.0/README.md +100 -0
- komaru-0.1.0/pyproject.toml +78 -0
- komaru-0.1.0/src/komaru/__init__.py +0 -0
- komaru-0.1.0/src/komaru/_probe_script.py +176 -0
- komaru-0.1.0/src/komaru/cli.py +269 -0
- komaru-0.1.0/src/komaru/discovery.py +94 -0
- komaru-0.1.0/src/komaru/extract/__init__.py +0 -0
- komaru-0.1.0/src/komaru/extract/handlers.py +225 -0
- komaru-0.1.0/src/komaru/extract/index.py +157 -0
- komaru-0.1.0/src/komaru/extract/objfile.py +135 -0
- komaru-0.1.0/src/komaru/extract/target.py +80 -0
- komaru-0.1.0/src/komaru/probe.py +123 -0
- komaru-0.1.0/src/komaru/tui/__init__.py +3 -0
- komaru-0.1.0/src/komaru/tui/app.py +655 -0
- komaru-0.1.0/tests/__init__.py +0 -0
- komaru-0.1.0/tests/conftest.py +5 -0
- komaru-0.1.0/tests/fixtures/fixture-stripped.elf +0 -0
- komaru-0.1.0/tests/fixtures/fixture.elf +0 -0
- komaru-0.1.0/tests/fixtures/fixture.s +48 -0
- komaru-0.1.0/tests/fixtures/make-fixture.sh +6 -0
- komaru-0.1.0/tests/test_cli.py +27 -0
- komaru-0.1.0/tests/test_cli_run.py +46 -0
- komaru-0.1.0/tests/test_discovery.py +97 -0
- komaru-0.1.0/tests/test_handlers.py +59 -0
- komaru-0.1.0/tests/test_index.py +55 -0
- komaru-0.1.0/tests/test_interactive.py +128 -0
- komaru-0.1.0/tests/test_objfile.py +47 -0
- komaru-0.1.0/tests/test_probe.py +100 -0
- komaru-0.1.0/tests/test_real_binary.py +74 -0
- komaru-0.1.0/tests/test_target.py +27 -0
- komaru-0.1.0/tests/test_tui.py +186 -0
- komaru-0.1.0/uv.lock +446 -0
komaru-0.1.0/.gitignore
ADDED
komaru-0.1.0/NOTES.md
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
# NOTES — assumptions about CPython internals
|
|
2
|
+
|
|
3
|
+
Every assumption komaru relies on, with the CPython source that justifies it.
|
|
4
|
+
Verified empirically (clang 21.1.8, `--with-tail-call-interp`, `OPT="-Og -g"`)
|
|
5
|
+
against:
|
|
6
|
+
|
|
7
|
+
| target | commit | handlers | helpers |
|
|
8
|
+
|---|---|---|---|
|
|
9
|
+
| 3.14.7 (branch `3.14`) | e7a45f431bf | 231 | `error`, `exception_unwind`, `start_frame` |
|
|
10
|
+
| 3.16.0a0 (`main`) | ea85e8faecc | 239 | + `pop_1_error`, `pop_2_error`, `exit_unwind`, `exit_unwind_notrace`, `UNKNOWN_OPCODE` |
|
|
11
|
+
|
|
12
|
+
The helper-symbol set differs between versions; nothing in komaru hardcodes
|
|
13
|
+
it — helpers are whatever `_TAIL_CALL_*` symbols do not match an opname.
|
|
14
|
+
The opcode↔handler bijection (below) holds exactly on both versions.
|
|
15
|
+
|
|
16
|
+
## Reference builds
|
|
17
|
+
|
|
18
|
+
```
|
|
19
|
+
mkdir -p build/tailcall-static && cd build/tailcall-static
|
|
20
|
+
CC=clang ../../configure --with-tail-call-interp OPT="-Og -g" && make -j
|
|
21
|
+
|
|
22
|
+
mkdir -p build/tailcall-shared && cd build/tailcall-shared
|
|
23
|
+
CC=clang ../../configure --with-tail-call-interp --enable-shared OPT="-Og -g" && make -j
|
|
24
|
+
|
|
25
|
+
# 3.14 (branch worktree):
|
|
26
|
+
git worktree add build/v3.14-src origin/3.14
|
|
27
|
+
mkdir -p build/tailcall-3.14 && cd build/tailcall-3.14
|
|
28
|
+
CC=clang ../v3.14-src/configure --with-tail-call-interp OPT="-Og -g" && make -j
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
## Symbol scheme
|
|
32
|
+
|
|
33
|
+
- **One function per opcode, named `_TAIL_CALL_<OPNAME>`.** The tail-call
|
|
34
|
+
interpreter generates a `static` function per opcode:
|
|
35
|
+
`Python/opcode_targets.h:525` (declarations, under `_Py_TAIL_CALL_INTERP`),
|
|
36
|
+
bodies in `Python/generated_tail_call_handlers.c.h`. Verified: 239 symbols
|
|
37
|
+
in the static build's `python` binary.
|
|
38
|
+
- **Symbols are `static` ⇒ LOCAL, `.symtab` only.** Zero appear in
|
|
39
|
+
`.dynsym`. A stripped binary is therefore unusable; the extractor fails
|
|
40
|
+
loudly (`objfile.py`, `StrippedBinaryError`).
|
|
41
|
+
- **Which object:** default build → the `python` executable;
|
|
42
|
+
`--enable-shared` → `libpython3.X.so.1.0` (the `python` stub has none).
|
|
43
|
+
- **No `.cold` / `.part.N` / `.constprop` fragments at `-Og`**, and every
|
|
44
|
+
symbol has a nonzero `st_size`, so byte ranges come straight from the
|
|
45
|
+
symbol table. **Revisit at `-O2`/LTO** — hot/cold splitting would break
|
|
46
|
+
the one-symbol-per-opcode assumption.
|
|
47
|
+
- **Specialized opcodes have their own handlers** (e.g.
|
|
48
|
+
`_TAIL_CALL_BINARY_OP_ADD_INT` distinct from `_TAIL_CALL_BINARY_OP`),
|
|
49
|
+
because the cases generator emits one function per uop-instruction
|
|
50
|
+
(`Tools/cases_generator/tier1_tail_call_generator.py`).
|
|
51
|
+
|
|
52
|
+
## Opcode ↔ handler accounting (verified bijection)
|
|
53
|
+
|
|
54
|
+
- The target's `opcode.opmap` names with value **> 255 are
|
|
55
|
+
pseudo-instructions** (`Include/opcode_ids.h`, `_PyOpcode_ID` pseudo range;
|
|
56
|
+
emitted only inside the compiler) and can never appear in a code object —
|
|
57
|
+
a one-byte opcode field cannot encode them. Exactly those 11 names have no
|
|
58
|
+
handler symbol; every encodable opcode (242 − 11 = 231) has exactly one.
|
|
59
|
+
- **8 extra symbols are not opcodes**: `_TAIL_CALL_UNKNOWN_OPCODE` plus the
|
|
60
|
+
lowercase helpers `start_frame`, `error`, `pop_1_error`, `pop_2_error`,
|
|
61
|
+
`exception_unwind`, `exit_unwind`, `exit_unwind_notrace`
|
|
62
|
+
(`Python/opcode_targets.h:525-531`). They are kept in the index (they are
|
|
63
|
+
common jump targets) but are distinguishable by case / UNKNOWN_OPCODE name.
|
|
64
|
+
|
|
65
|
+
## Dispatch tail
|
|
66
|
+
|
|
67
|
+
- `DISPATCH()` expands to NEXTOPARG-style decode plus
|
|
68
|
+
`Py_MUSTTAIL return table[opcode](...)` — `Python/ceval_macros.h:106-110`.
|
|
69
|
+
At `-Og` on x86-64 this codegens as a trailing contiguous sequence ending
|
|
70
|
+
in an indirect `jmp` (verified for `LOAD_FAST`:
|
|
71
|
+
`movzx ecx, [r15+2]; add r15,2; movzx esi,ch; movzx edx,cl;
|
|
72
|
+
mov rcx,[rdi+rdx*8]; jmp rcx`).
|
|
73
|
+
- **Tagging heuristic** (`handlers.py`, `dispatch_flags`): for each indirect
|
|
74
|
+
jump, walk backwards collecting the register-dependency slice (stop at
|
|
75
|
+
control flow or a memory store, budget 16 instructions), then mark the
|
|
76
|
+
contiguous range up to the jump. This is a heuristic over codegen, not
|
|
77
|
+
ground truth; it is exact at `-Og` for all 226 handlers that dispatch.
|
|
78
|
+
- **Direct tail calls are *not* tagged as dispatch.** `DISPATCH_GOTO`-style
|
|
79
|
+
same-target tail calls, deopt jumps from specialized handlers back to the
|
|
80
|
+
generic handler, and error-path jumps to `_TAIL_CALL_error`
|
|
81
|
+
(`Python/ceval_macros.h:114-124`) are semantically part of the opcode;
|
|
82
|
+
hiding them would misrepresent the handler.
|
|
83
|
+
- 13 handlers legitimately have no dispatch tail: `INTERPRETER_EXIT`,
|
|
84
|
+
`CACHE`, `RESERVED`, `UNKNOWN_OPCODE`, `RAISE_VARARGS`, `RERAISE`,
|
|
85
|
+
`ENTER_EXECUTOR`, `TRACE_RECORD`, and the five error/unwind helpers.
|
|
86
|
+
|
|
87
|
+
## Call-target resolution
|
|
88
|
+
|
|
89
|
+
- Direct `call`/`jmp` immediates resolve against `.symtab` function
|
|
90
|
+
addresses. In the static build every direct call resolves this way.
|
|
91
|
+
- In `--enable-shared` builds, cross-object calls go through the PLT; stubs
|
|
92
|
+
are resolved to `name@plt` by decoding the stub's RIP-relative `jmp` and
|
|
93
|
+
matching the GOT slot against `.rela.plt` / `.rela.dyn` relocations.
|
|
94
|
+
`.plt.got` stubs are 8 bytes and can sit at the very end of their section,
|
|
95
|
+
so the stub read backs off (16 → 8 → 6 bytes) instead of reading across
|
|
96
|
+
the section boundary.
|
|
97
|
+
- Indirect calls (`call rax`, `call [r14+0x48]` — type slots, function
|
|
98
|
+
pointers) are left with `target: null`. They are not statically
|
|
99
|
+
resolvable, and komaru never guesses.
|
|
100
|
+
- Intra-function branch targets (`jne 0x...` within a handler) are likewise
|
|
101
|
+
`null`; the address itself is shown.
|
|
102
|
+
|
|
103
|
+
## Target-interpreter metadata
|
|
104
|
+
|
|
105
|
+
- Opcode names/numbers come from shelling out to the *target* interpreter
|
|
106
|
+
(`target.py`), never the host's `opcode` module — numbering changes every
|
|
107
|
+
release. The index records the target's full `version_info`, and
|
|
108
|
+
`Index.check_interpreter()` refuses a mismatched consumer.
|
|
109
|
+
- Build-tree interpreters from `--enable-shared` builds cannot locate their
|
|
110
|
+
own `libpython` without `LD_LIBRARY_PATH`; `interrogate()` prepends the
|
|
111
|
+
binary's directory for the probe subprocess only.
|
|
112
|
+
- `platform.python_build()[0]` carries the git ref/commit
|
|
113
|
+
(e.g. `heads/main-dirty:ea85e8faecc`); `sysconfig` `CONFIG_ARGS` and
|
|
114
|
+
`HOST_GNU_TYPE` supply build flags and target triple.
|
|
115
|
+
|
|
116
|
+
## Warmup / adaptive view
|
|
117
|
+
|
|
118
|
+
- `--warmup N` compiles the script once and ``exec``s the same code object N
|
|
119
|
+
times, so tier-1 specialisation accumulates in the code objects being
|
|
120
|
+
displayed. Execution counts are observed via `sys.monitoring` PY_START
|
|
121
|
+
(`Python/instrumentation.c`) — never inferred from dis output.
|
|
122
|
+
- Instrumentation swaps in `INSTRUMENTED_*` opcodes and de-instruments
|
|
123
|
+
*lazily* on next execution. After the N monitored runs, one extra
|
|
124
|
+
unmonitored settling run lets executed code de-instrument; otherwise
|
|
125
|
+
komaru's own measurement would appear in the adaptive view. A code path
|
|
126
|
+
taken during monitored runs but not during the settling run can still show
|
|
127
|
+
a leftover `INSTRUMENTED_*` opcode — that is genuinely what is in the code
|
|
128
|
+
object, and it is shown as-is.
|
|
129
|
+
- `sys.monitoring` PY_START instrumentation does not inhibit specialisation
|
|
130
|
+
of other instruction families (verified empirically: `BINARY_OP_ADD_INT`
|
|
131
|
+
appears while monitored).
|
|
132
|
+
|
|
133
|
+
## Architecture support
|
|
134
|
+
|
|
135
|
+
- x86-64 is implemented and tested. The aarch64 code paths (capstone arch
|
|
136
|
+
selection, w→x register canonicalisation) are wired but **untested** — no
|
|
137
|
+
aarch64 machine was available. ELF only; `ObjectFile` is a protocol so
|
|
138
|
+
Mach-O can be added later.
|
komaru-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: komaru
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: TUI mapping CPython bytecode to the interpreter's native opcode handlers (tail-call builds)
|
|
5
|
+
Author-email: Kirill Podoprigora <kirill.bast@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Keywords: assembly,bytecode,cpython,disassembly,interpreter,tui
|
|
8
|
+
Classifier: Development Status :: 4 - Beta
|
|
9
|
+
Classifier: Environment :: Console
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
17
|
+
Classifier: Topic :: Software Development :: Debuggers
|
|
18
|
+
Classifier: Topic :: Software Development :: Interpreters
|
|
19
|
+
Requires-Python: >=3.11
|
|
20
|
+
Requires-Dist: capstone>=5.0
|
|
21
|
+
Requires-Dist: pyelftools>=0.31
|
|
22
|
+
Requires-Dist: rich>=13.0
|
|
23
|
+
Requires-Dist: textual>=0.80
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
|
|
26
|
+
# komaru
|
|
27
|
+
|
|
28
|
+
A terminal UI that shows, for a Python function or module, three synchronised
|
|
29
|
+
views: the source, the `dis` bytecode, and **the actual machine code the
|
|
30
|
+
CPython interpreter executes for each opcode** — extracted from a real
|
|
31
|
+
`--with-tail-call-interp` build, never synthesised.
|
|
32
|
+
|
|
33
|
+
Not a compiler, not a JIT: an inspection tool. Tail-call interpreter builds
|
|
34
|
+
(CPython 3.14+, clang ≥ 19 / GCC ≥ 15) emit one function per opcode
|
|
35
|
+
(`_TAIL_CALL_<OPNAME>`), giving a mechanical opcode → machine-code mapping.
|
|
36
|
+
|
|
37
|
+
## Quickstart
|
|
38
|
+
|
|
39
|
+
```sh
|
|
40
|
+
# 1. Build CPython with the tail-call interpreter (readable codegen at -Og):
|
|
41
|
+
cd cpython && mkdir build && cd build
|
|
42
|
+
CC=clang ../configure --with-tail-call-interp OPT="-Og -g" && make -j
|
|
43
|
+
|
|
44
|
+
# 2. Just run it:
|
|
45
|
+
komaru # opens the TUI; first run asks for the CPython binary,
|
|
46
|
+
# extracts the handlers and caches the index
|
|
47
|
+
komaru script.py # open a file directly (bare path implies `run`)
|
|
48
|
+
komaru module.pyc
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
On first run komaru asks for the path to the tail-call `python` (or
|
|
52
|
+
`libpython*.so` plus an interpreter) and caches the extracted index in
|
|
53
|
+
`~/.cache/komaru/`, keyed by binary path — a rebuilt CPython is detected
|
|
54
|
+
and re-extracted automatically. Inside the TUI, `f` opens files, `w` sets
|
|
55
|
+
warmup, `o` picks nested code objects, `b` switches builds.
|
|
56
|
+
|
|
57
|
+
Index discovery order: `--index` → `KOMARU_INDEX` → `./index.json` → cache.
|
|
58
|
+
|
|
59
|
+
Everything is still scriptable with explicit flags:
|
|
60
|
+
|
|
61
|
+
```sh
|
|
62
|
+
komaru extract build/python -o index.json
|
|
63
|
+
komaru run script.py --target-func myfunc --warmup 3 --index index.json
|
|
64
|
+
komaru run --code 'a + b' --index index.json
|
|
65
|
+
komaru run --target-func json:dumps --index index.json
|
|
66
|
+
komaru open module.pyc --index index.json
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
For `--enable-shared` builds, extract from the library and point `--python`
|
|
70
|
+
at the build-tree interpreter:
|
|
71
|
+
|
|
72
|
+
```sh
|
|
73
|
+
komaru extract build/libpython3.16.so.1.0 --python build/python -o index.json
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
## Adaptive (specialising) mode
|
|
77
|
+
|
|
78
|
+
Warmup (`w` in the TUI, or `--warmup N`) executes the script/module body N
|
|
79
|
+
times (plus one unmonitored settling run) so the interpreter specialises;
|
|
80
|
+
the script itself is responsible for calling the functions you care about.
|
|
81
|
+
Press `a` in the TUI to
|
|
82
|
+
flip between `BINARY_OP` and e.g. `BINARY_OP_ADD_INT` — specialised opcodes
|
|
83
|
+
have completely different handlers and the native-size column shows it.
|
|
84
|
+
Execution counts are measured with `sys.monitoring`, so the UI tells you
|
|
85
|
+
when a code object never actually ran.
|
|
86
|
+
|
|
87
|
+
## Keys
|
|
88
|
+
|
|
89
|
+
`Tab` cycle panes · `↑/↓` select instruction · `f` open file · `w` warmup ·
|
|
90
|
+
`a` adaptive · `t` hide dispatch tail · `o` pick nested code object ·
|
|
91
|
+
`b` switch CPython build · `/` search · `e` export markdown · `?` help ·
|
|
92
|
+
`q` quit
|
|
93
|
+
|
|
94
|
+
## Correctness
|
|
95
|
+
|
|
96
|
+
- Every native instruction shown comes from capstone over bytes read out of
|
|
97
|
+
the binary; missing data is reported as unavailable, never guessed.
|
|
98
|
+
- The index records the target interpreter's version; `komaru run` refuses a
|
|
99
|
+
mismatched index (opcode numbering changes every release).
|
|
100
|
+
- Opcode names come from the target interpreter, never the host's `opcode`
|
|
101
|
+
module; nothing is hardcoded.
|
|
102
|
+
|
|
103
|
+
See `NOTES.md` for every assumption about CPython internals and where in the
|
|
104
|
+
CPython source it is justified.
|
|
105
|
+
|
|
106
|
+
## Requirements & scope (v1)
|
|
107
|
+
|
|
108
|
+
- Target: CPython ≥ 3.14 built with `--with-tail-call-interp` (verified
|
|
109
|
+
against 3.14 and main); each index is pinned to the exact version it was
|
|
110
|
+
extracted from. Linux, ELF, x86-64 (aarch64 wired but untested). The
|
|
111
|
+
extractor and the TUI may run under different interpreters; only the
|
|
112
|
+
target interpreter must match the extracted binary.
|
|
113
|
+
- The binary must not be stripped: handler symbols are `.symtab`-local.
|
|
114
|
+
|
|
115
|
+
Future work (explicitly out of scope for v1): Tier 2 uop expansion, JIT
|
|
116
|
+
stencil inspection, `perf` integration, Mach-O/macOS, Windows.
|
|
117
|
+
|
|
118
|
+
## Development
|
|
119
|
+
|
|
120
|
+
```sh
|
|
121
|
+
uv sync
|
|
122
|
+
uv run pytest # integration tests auto-skip without ../build trees
|
|
123
|
+
uv run ruff check . && uv run ruff format --check .
|
|
124
|
+
uv run mypy src tests
|
|
125
|
+
```
|
komaru-0.1.0/README.md
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# komaru
|
|
2
|
+
|
|
3
|
+
A terminal UI that shows, for a Python function or module, three synchronised
|
|
4
|
+
views: the source, the `dis` bytecode, and **the actual machine code the
|
|
5
|
+
CPython interpreter executes for each opcode** — extracted from a real
|
|
6
|
+
`--with-tail-call-interp` build, never synthesised.
|
|
7
|
+
|
|
8
|
+
Not a compiler, not a JIT: an inspection tool. Tail-call interpreter builds
|
|
9
|
+
(CPython 3.14+, clang ≥ 19 / GCC ≥ 15) emit one function per opcode
|
|
10
|
+
(`_TAIL_CALL_<OPNAME>`), giving a mechanical opcode → machine-code mapping.
|
|
11
|
+
|
|
12
|
+
## Quickstart
|
|
13
|
+
|
|
14
|
+
```sh
|
|
15
|
+
# 1. Build CPython with the tail-call interpreter (readable codegen at -Og):
|
|
16
|
+
cd cpython && mkdir build && cd build
|
|
17
|
+
CC=clang ../configure --with-tail-call-interp OPT="-Og -g" && make -j
|
|
18
|
+
|
|
19
|
+
# 2. Just run it:
|
|
20
|
+
komaru # opens the TUI; first run asks for the CPython binary,
|
|
21
|
+
# extracts the handlers and caches the index
|
|
22
|
+
komaru script.py # open a file directly (bare path implies `run`)
|
|
23
|
+
komaru module.pyc
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
On first run komaru asks for the path to the tail-call `python` (or
|
|
27
|
+
`libpython*.so` plus an interpreter) and caches the extracted index in
|
|
28
|
+
`~/.cache/komaru/`, keyed by binary path — a rebuilt CPython is detected
|
|
29
|
+
and re-extracted automatically. Inside the TUI, `f` opens files, `w` sets
|
|
30
|
+
warmup, `o` picks nested code objects, `b` switches builds.
|
|
31
|
+
|
|
32
|
+
Index discovery order: `--index` → `KOMARU_INDEX` → `./index.json` → cache.
|
|
33
|
+
|
|
34
|
+
Everything is still scriptable with explicit flags:
|
|
35
|
+
|
|
36
|
+
```sh
|
|
37
|
+
komaru extract build/python -o index.json
|
|
38
|
+
komaru run script.py --target-func myfunc --warmup 3 --index index.json
|
|
39
|
+
komaru run --code 'a + b' --index index.json
|
|
40
|
+
komaru run --target-func json:dumps --index index.json
|
|
41
|
+
komaru open module.pyc --index index.json
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
For `--enable-shared` builds, extract from the library and point `--python`
|
|
45
|
+
at the build-tree interpreter:
|
|
46
|
+
|
|
47
|
+
```sh
|
|
48
|
+
komaru extract build/libpython3.16.so.1.0 --python build/python -o index.json
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
## Adaptive (specialising) mode
|
|
52
|
+
|
|
53
|
+
Warmup (`w` in the TUI, or `--warmup N`) executes the script/module body N
|
|
54
|
+
times (plus one unmonitored settling run) so the interpreter specialises;
|
|
55
|
+
the script itself is responsible for calling the functions you care about.
|
|
56
|
+
Press `a` in the TUI to
|
|
57
|
+
flip between `BINARY_OP` and e.g. `BINARY_OP_ADD_INT` — specialised opcodes
|
|
58
|
+
have completely different handlers and the native-size column shows it.
|
|
59
|
+
Execution counts are measured with `sys.monitoring`, so the UI tells you
|
|
60
|
+
when a code object never actually ran.
|
|
61
|
+
|
|
62
|
+
## Keys
|
|
63
|
+
|
|
64
|
+
`Tab` cycle panes · `↑/↓` select instruction · `f` open file · `w` warmup ·
|
|
65
|
+
`a` adaptive · `t` hide dispatch tail · `o` pick nested code object ·
|
|
66
|
+
`b` switch CPython build · `/` search · `e` export markdown · `?` help ·
|
|
67
|
+
`q` quit
|
|
68
|
+
|
|
69
|
+
## Correctness
|
|
70
|
+
|
|
71
|
+
- Every native instruction shown comes from capstone over bytes read out of
|
|
72
|
+
the binary; missing data is reported as unavailable, never guessed.
|
|
73
|
+
- The index records the target interpreter's version; `komaru run` refuses a
|
|
74
|
+
mismatched index (opcode numbering changes every release).
|
|
75
|
+
- Opcode names come from the target interpreter, never the host's `opcode`
|
|
76
|
+
module; nothing is hardcoded.
|
|
77
|
+
|
|
78
|
+
See `NOTES.md` for every assumption about CPython internals and where in the
|
|
79
|
+
CPython source it is justified.
|
|
80
|
+
|
|
81
|
+
## Requirements & scope (v1)
|
|
82
|
+
|
|
83
|
+
- Target: CPython ≥ 3.14 built with `--with-tail-call-interp` (verified
|
|
84
|
+
against 3.14 and main); each index is pinned to the exact version it was
|
|
85
|
+
extracted from. Linux, ELF, x86-64 (aarch64 wired but untested). The
|
|
86
|
+
extractor and the TUI may run under different interpreters; only the
|
|
87
|
+
target interpreter must match the extracted binary.
|
|
88
|
+
- The binary must not be stripped: handler symbols are `.symtab`-local.
|
|
89
|
+
|
|
90
|
+
Future work (explicitly out of scope for v1): Tier 2 uop expansion, JIT
|
|
91
|
+
stencil inspection, `perf` integration, Mach-O/macOS, Windows.
|
|
92
|
+
|
|
93
|
+
## Development
|
|
94
|
+
|
|
95
|
+
```sh
|
|
96
|
+
uv sync
|
|
97
|
+
uv run pytest # integration tests auto-skip without ../build trees
|
|
98
|
+
uv run ruff check . && uv run ruff format --check .
|
|
99
|
+
uv run mypy src tests
|
|
100
|
+
```
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "komaru"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "TUI mapping CPython bytecode to the interpreter's native opcode handlers (tail-call builds)"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
authors = [{ name = "Kirill Podoprigora", email = "kirill.bast@gmail.com" }]
|
|
8
|
+
keywords = ["cpython", "bytecode", "disassembly", "assembly", "interpreter", "tui"]
|
|
9
|
+
classifiers = [
|
|
10
|
+
"Development Status :: 4 - Beta",
|
|
11
|
+
"Environment :: Console",
|
|
12
|
+
"Intended Audience :: Developers",
|
|
13
|
+
"Operating System :: POSIX :: Linux",
|
|
14
|
+
"Programming Language :: Python :: 3",
|
|
15
|
+
"Programming Language :: Python :: 3.11",
|
|
16
|
+
"Programming Language :: Python :: 3.12",
|
|
17
|
+
"Programming Language :: Python :: 3.13",
|
|
18
|
+
"Programming Language :: Python :: 3.14",
|
|
19
|
+
"Topic :: Software Development :: Debuggers",
|
|
20
|
+
"Topic :: Software Development :: Interpreters",
|
|
21
|
+
]
|
|
22
|
+
requires-python = ">=3.11"
|
|
23
|
+
dependencies = [
|
|
24
|
+
"capstone>=5.0",
|
|
25
|
+
"pyelftools>=0.31",
|
|
26
|
+
"rich>=13.0",
|
|
27
|
+
"textual>=0.80",
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
[project.scripts]
|
|
31
|
+
komaru = "komaru.cli:main"
|
|
32
|
+
|
|
33
|
+
[dependency-groups]
|
|
34
|
+
dev = [
|
|
35
|
+
"mypy>=1.11",
|
|
36
|
+
"pytest>=8.0",
|
|
37
|
+
"ruff>=0.6",
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
[build-system]
|
|
41
|
+
requires = ["hatchling"]
|
|
42
|
+
build-backend = "hatchling.build"
|
|
43
|
+
|
|
44
|
+
[tool.hatch.build.targets.wheel]
|
|
45
|
+
packages = ["src/komaru"]
|
|
46
|
+
|
|
47
|
+
[tool.hatch.build.targets.sdist]
|
|
48
|
+
exclude = [
|
|
49
|
+
"example.py",
|
|
50
|
+
"index.json",
|
|
51
|
+
"docs/",
|
|
52
|
+
".gitignore",
|
|
53
|
+
]
|
|
54
|
+
|
|
55
|
+
[tool.ruff]
|
|
56
|
+
line-length = 100
|
|
57
|
+
target-version = "py311"
|
|
58
|
+
|
|
59
|
+
[tool.ruff.lint]
|
|
60
|
+
select = ["E", "F", "W", "I", "UP", "B", "C4", "SIM", "RUF"]
|
|
61
|
+
|
|
62
|
+
[tool.mypy]
|
|
63
|
+
strict = true
|
|
64
|
+
python_version = "3.11"
|
|
65
|
+
|
|
66
|
+
[[tool.mypy.overrides]]
|
|
67
|
+
module = ["capstone.*", "elftools.*"]
|
|
68
|
+
ignore_missing_imports = true
|
|
69
|
+
|
|
70
|
+
# The probe runs under the *target* interpreter (possibly a different
|
|
71
|
+
# CPython version); checking it against the host's stdlib stubs is wrong
|
|
72
|
+
# by construction. It is exercised for real by tests/test_probe.py.
|
|
73
|
+
[[tool.mypy.overrides]]
|
|
74
|
+
module = "komaru._probe_script"
|
|
75
|
+
ignore_errors = true
|
|
76
|
+
|
|
77
|
+
[tool.pytest.ini_options]
|
|
78
|
+
testpaths = ["tests"]
|
|
File without changes
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
import dis
|
|
2
|
+
import importlib.util
|
|
3
|
+
import json
|
|
4
|
+
import marshal
|
|
5
|
+
import sys
|
|
6
|
+
import traceback
|
|
7
|
+
|
|
8
|
+
_PYC_HEADER = 16 # PEP 552: magic + flags + two words
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def read(path, mode="r"):
|
|
12
|
+
with open(path, mode) as f:
|
|
13
|
+
return f.read()
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def fail(message):
|
|
17
|
+
print(json.dumps({"error": message}))
|
|
18
|
+
sys.exit(2)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def serializeinstructions(code, adaptive):
|
|
22
|
+
out = []
|
|
23
|
+
for i in dis.get_instructions(code, adaptive=adaptive):
|
|
24
|
+
pos = i.positions
|
|
25
|
+
if pos is None or pos.lineno is None:
|
|
26
|
+
positions = None
|
|
27
|
+
else:
|
|
28
|
+
positions = [pos.lineno, pos.end_lineno, pos.col_offset, pos.end_col_offset]
|
|
29
|
+
out.append(
|
|
30
|
+
{
|
|
31
|
+
"offset": i.offset,
|
|
32
|
+
"opname": i.opname,
|
|
33
|
+
"arg": i.arg,
|
|
34
|
+
"argrepr": i.argrepr,
|
|
35
|
+
"line": i.line_number,
|
|
36
|
+
"positions": positions,
|
|
37
|
+
"jump_target": getattr(i, "jump_target", None),
|
|
38
|
+
}
|
|
39
|
+
)
|
|
40
|
+
return out
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def childcodes(code):
|
|
44
|
+
return [c for c in code.co_consts if isinstance(c, type(code))]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def walk(code, counts, warmed):
|
|
48
|
+
return {
|
|
49
|
+
"name": code.co_name,
|
|
50
|
+
"qualname": code.co_qualname,
|
|
51
|
+
"filename": code.co_filename,
|
|
52
|
+
"firstlineno": code.co_firstlineno,
|
|
53
|
+
"instructions": serializeinstructions(code, adaptive=False),
|
|
54
|
+
"adaptive": serializeinstructions(code, adaptive=True) if warmed else None,
|
|
55
|
+
"exec_count": counts.get(code, 0),
|
|
56
|
+
"children": [walk(c, counts, warmed) for c in childcodes(code)],
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def findbyqualname(code, qualname):
|
|
61
|
+
if code.co_qualname == qualname:
|
|
62
|
+
return code
|
|
63
|
+
for child in childcodes(code):
|
|
64
|
+
found = findbyqualname(child, qualname)
|
|
65
|
+
if found is not None:
|
|
66
|
+
return found
|
|
67
|
+
return None
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def allqualnames(code):
|
|
71
|
+
result = [code.co_qualname]
|
|
72
|
+
for child in childcodes(code):
|
|
73
|
+
result.extend(allqualnames(child))
|
|
74
|
+
return result
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def load(request):
|
|
78
|
+
mode = request["mode"]
|
|
79
|
+
if mode == "script":
|
|
80
|
+
path = request["path"]
|
|
81
|
+
source = read(path)
|
|
82
|
+
return compile(source, path, "exec"), source
|
|
83
|
+
if mode == "code":
|
|
84
|
+
snippet = request["code"]
|
|
85
|
+
return compile(snippet, "<komaru>", "exec"), snippet
|
|
86
|
+
if mode == "module":
|
|
87
|
+
spec = importlib.util.find_spec(request["path"])
|
|
88
|
+
if spec is None or spec.origin is None or not spec.origin.endswith(".py"):
|
|
89
|
+
fail(f"cannot locate source for module {request['path']!r}")
|
|
90
|
+
source = read(spec.origin)
|
|
91
|
+
return compile(source, spec.origin, "exec"), source
|
|
92
|
+
if mode == "pyc":
|
|
93
|
+
data = read(request["path"], "rb")
|
|
94
|
+
magic = importlib.util.MAGIC_NUMBER
|
|
95
|
+
if data[:4] != magic:
|
|
96
|
+
fail(
|
|
97
|
+
f"{request['path']}: pyc magic {data[:4].hex()} does not match "
|
|
98
|
+
f"this interpreter's magic {magic.hex()} — compiled by a "
|
|
99
|
+
"different CPython version"
|
|
100
|
+
)
|
|
101
|
+
code = marshal.loads(data[_PYC_HEADER:])
|
|
102
|
+
try:
|
|
103
|
+
source = read(code.co_filename)
|
|
104
|
+
except OSError:
|
|
105
|
+
source = None
|
|
106
|
+
return code, source
|
|
107
|
+
fail(f"unknown mode {mode!r}")
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def runonce(code):
|
|
111
|
+
namespace = {"__name__": "__main__", "__file__": code.co_filename}
|
|
112
|
+
try:
|
|
113
|
+
exec(code, namespace)
|
|
114
|
+
except BaseException:
|
|
115
|
+
return "".join(traceback.format_exception_only(*sys.exc_info()[:2])).strip()
|
|
116
|
+
return None
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def warm(code, runs):
|
|
120
|
+
mon = sys.monitoring
|
|
121
|
+
tool = mon.PROFILER_ID
|
|
122
|
+
counts = {}
|
|
123
|
+
|
|
124
|
+
def onstart(started_code, _offset):
|
|
125
|
+
counts[started_code] = counts.get(started_code, 0) + 1
|
|
126
|
+
|
|
127
|
+
mon.use_tool_id(tool, "komaru")
|
|
128
|
+
mon.register_callback(tool, mon.events.PY_START, onstart)
|
|
129
|
+
mon.set_events(tool, mon.events.PY_START)
|
|
130
|
+
error = None
|
|
131
|
+
try:
|
|
132
|
+
for _ in range(runs):
|
|
133
|
+
error = runonce(code)
|
|
134
|
+
if error is not None:
|
|
135
|
+
break
|
|
136
|
+
finally:
|
|
137
|
+
mon.set_events(tool, 0)
|
|
138
|
+
mon.register_callback(tool, mon.events.PY_START, None)
|
|
139
|
+
mon.free_tool_id(tool)
|
|
140
|
+
if error is None:
|
|
141
|
+
error = runonce(code)
|
|
142
|
+
return counts, error
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def main():
|
|
146
|
+
request = json.load(sys.stdin)
|
|
147
|
+
code, source = load(request)
|
|
148
|
+
warmup = request.get("warmup") or 0
|
|
149
|
+
counts, warmup_error = ({}, None) if warmup == 0 else warm(code, warmup)
|
|
150
|
+
|
|
151
|
+
root = code
|
|
152
|
+
target = request.get("target_func")
|
|
153
|
+
if target:
|
|
154
|
+
found = findbyqualname(code, target)
|
|
155
|
+
if found is None:
|
|
156
|
+
fail(
|
|
157
|
+
f"no code object with qualname {target!r}; available: "
|
|
158
|
+
+ ", ".join(sorted(allqualnames(code)))
|
|
159
|
+
)
|
|
160
|
+
root = found
|
|
161
|
+
|
|
162
|
+
print(
|
|
163
|
+
json.dumps(
|
|
164
|
+
{
|
|
165
|
+
"version_info": list(sys.version_info),
|
|
166
|
+
"source": source,
|
|
167
|
+
"root": walk(root, counts, warmed=warmup > 0),
|
|
168
|
+
"warmup_runs": warmup,
|
|
169
|
+
"warmup_error": warmup_error,
|
|
170
|
+
}
|
|
171
|
+
)
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
if __name__ == "__main__":
|
|
176
|
+
main()
|