codeanalyzer-python 0.1.14__py3-none-any.whl → 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codeanalyzer/__main__.py +108 -6
- codeanalyzer/core.py +100 -18
- codeanalyzer/neo4j/__init__.py +46 -0
- codeanalyzer/neo4j/bolt.py +234 -0
- codeanalyzer/neo4j/catalog.py +245 -0
- codeanalyzer/neo4j/cypher.py +138 -0
- codeanalyzer/neo4j/emit.py +74 -0
- codeanalyzer/neo4j/project.py +330 -0
- codeanalyzer/neo4j/rows.py +181 -0
- codeanalyzer/neo4j/schema.py +39 -0
- codeanalyzer/options/__init__.py +2 -2
- codeanalyzer/options/options.py +21 -0
- codeanalyzer/schema/__init__.py +2 -0
- codeanalyzer/schema/py_schema.py +15 -0
- codeanalyzer/semantic_analysis/codeql/codeql_analysis.py +109 -27
- codeanalyzer_python-0.2.1.dist-info/METADATA +415 -0
- codeanalyzer_python-0.2.1.dist-info/RECORD +39 -0
- {codeanalyzer_python-0.1.14.dist-info → codeanalyzer_python-0.2.1.dist-info}/WHEEL +1 -1
- codeanalyzer_python-0.2.1.dist-info/entry_points.txt +3 -0
- codeanalyzer_python-0.1.14.dist-info/METADATA +0 -392
- codeanalyzer_python-0.1.14.dist-info/RECORD +0 -31
- codeanalyzer_python-0.1.14.dist-info/entry_points.txt +0 -2
- {codeanalyzer_python-0.1.14.dist-info → codeanalyzer_python-0.2.1.dist-info}/licenses/LICENSE +0 -0
- {codeanalyzer_python-0.1.14.dist-info → codeanalyzer_python-0.2.1.dist-info}/licenses/NOTICE +0 -0
|
@@ -32,6 +32,67 @@ from codeanalyzer.semantic_analysis.codeql.codeql_query_runner import CodeQLQuer
|
|
|
32
32
|
from codeanalyzer.utils import logger
|
|
33
33
|
|
|
34
34
|
|
|
35
|
+
class _CallableResolver:
|
|
36
|
+
"""Maps a CodeQL endpoint ``(file, start_line, name, arity)`` to a Jedi
|
|
37
|
+
``PyCallable``.
|
|
38
|
+
|
|
39
|
+
Resolution ladder:
|
|
40
|
+
1. exact ``(abs_path, start_line)`` — the precise join;
|
|
41
|
+
2. on miss, candidates sharing ``(abs_path, short_name)``: a single
|
|
42
|
+
candidate is taken directly; otherwise prefer those whose
|
|
43
|
+
parameter count equals the CodeQL positional arity, then the
|
|
44
|
+
nearest ``start_line``;
|
|
45
|
+
3. no name match -> ``None`` (caller row skipped / callee becomes
|
|
46
|
+
a ghost node).
|
|
47
|
+
|
|
48
|
+
Step 2 recovers edges the ``(file, line)`` join silently drops when
|
|
49
|
+
CodeQL and Jedi disagree on a definition's start line (e.g. decorator
|
|
50
|
+
handling). Jedi's ``parameters`` counts every declared slot (incl.
|
|
51
|
+
``*args``/``**kwargs``/keyword-only) whereas CodeQL's arity is
|
|
52
|
+
positional only, so the arity filter is exact for plain signatures
|
|
53
|
+
and otherwise yields to the nearest-line tiebreak.
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
def __init__(self) -> None:
|
|
57
|
+
self._by_loc: Dict[Tuple[str, int], Any] = {}
|
|
58
|
+
self._by_name: Dict[Tuple[str, str], List[Any]] = {}
|
|
59
|
+
|
|
60
|
+
@staticmethod
|
|
61
|
+
def _abs(path: str) -> str:
|
|
62
|
+
try:
|
|
63
|
+
return str(Path(path).resolve())
|
|
64
|
+
except (OSError, RuntimeError):
|
|
65
|
+
return path
|
|
66
|
+
|
|
67
|
+
@classmethod
|
|
68
|
+
def from_symbol_table(
|
|
69
|
+
cls, symbol_table: Dict[str, PyModule]
|
|
70
|
+
) -> "_CallableResolver":
|
|
71
|
+
resolver = cls()
|
|
72
|
+
for c in iter_callables_in_symbol_table(symbol_table):
|
|
73
|
+
abs_path = cls._abs(c.path)
|
|
74
|
+
resolver._by_loc[(abs_path, c.start_line)] = c
|
|
75
|
+
resolver._by_name.setdefault((abs_path, c.name), []).append(c)
|
|
76
|
+
return resolver
|
|
77
|
+
|
|
78
|
+
def resolve(
|
|
79
|
+
self, file: str, start_line: int, name: str, arity: int
|
|
80
|
+
) -> Any:
|
|
81
|
+
exact = self._by_loc.get((file, start_line))
|
|
82
|
+
if exact is not None:
|
|
83
|
+
return exact
|
|
84
|
+
if not name:
|
|
85
|
+
return None
|
|
86
|
+
candidates = self._by_name.get((file, name))
|
|
87
|
+
if not candidates:
|
|
88
|
+
return None
|
|
89
|
+
if len(candidates) == 1:
|
|
90
|
+
return candidates[0]
|
|
91
|
+
arity_matched = [c for c in candidates if len(c.parameters) == arity]
|
|
92
|
+
pool = arity_matched or candidates
|
|
93
|
+
return min(pool, key=lambda c: abs(c.start_line - start_line))
|
|
94
|
+
|
|
95
|
+
|
|
35
96
|
class CodeQL:
|
|
36
97
|
"""A class for building the application view of a Python application using CodeQL.
|
|
37
98
|
|
|
@@ -99,9 +160,14 @@ class CodeQL:
|
|
|
99
160
|
# codeql/python-all 7.x — it returns the ``CallNode`` (CFG)
|
|
100
161
|
# whose target was resolved to that ``Value``. Cleaner than
|
|
101
162
|
# poking at ``pointsTo`` directly.
|
|
102
|
-
|
|
163
|
+
# ``callee`` is bound to the FunctionValue's scope so the
|
|
164
|
+
# endpoint emits the same Function-level facts (name, arity,
|
|
165
|
+
# location) the post-processor needs for the name+arity
|
|
166
|
+
# fallback when the (file, start_line) join misses.
|
|
167
|
+
"from CallNode call, Function caller, FunctionValue calleeVal, Function callee",
|
|
103
168
|
"where",
|
|
104
169
|
" call.getScope() = caller and",
|
|
170
|
+
" callee = calleeVal.getScope() and",
|
|
105
171
|
" (",
|
|
106
172
|
# Direct function / bound-method call: foo() or obj.foo()
|
|
107
173
|
" call = calleeVal.getACall()",
|
|
@@ -115,15 +181,20 @@ class CodeQL:
|
|
|
115
181
|
" )",
|
|
116
182
|
" )",
|
|
117
183
|
"select",
|
|
118
|
-
# --- Caller endpoint --- (joins to PyCallable
|
|
184
|
+
# --- Caller endpoint --- (joins to PyCallable: exact by
|
|
185
|
+
# (file, start_line), else by (file, name) + arity)
|
|
119
186
|
" caller.getLocation().getFile().getAbsolutePath(),",
|
|
120
187
|
" caller.getLocation().getStartLine(),",
|
|
121
188
|
" caller.getQualifiedName(),",
|
|
189
|
+
" caller.getName(),",
|
|
190
|
+
" count(caller.getArg(_)),",
|
|
122
191
|
# --- Callee endpoint --- (file/line may live in a library stub;
|
|
123
192
|
# post-processor classifies as in-source or ghost)
|
|
124
|
-
"
|
|
125
|
-
"
|
|
193
|
+
" callee.getLocation().getFile().getAbsolutePath(),",
|
|
194
|
+
" callee.getLocation().getStartLine(),",
|
|
126
195
|
" calleeVal.getQualifiedName(),",
|
|
196
|
+
" callee.getName(),",
|
|
197
|
+
" count(callee.getArg(_)),",
|
|
127
198
|
# --- Call-site location --- (for PyCallsite augmentation)
|
|
128
199
|
" call.getLocation().getStartLine(),",
|
|
129
200
|
" call.getLocation().getStartColumn(),",
|
|
@@ -149,9 +220,13 @@ class CodeQL:
|
|
|
149
220
|
"caller_file",
|
|
150
221
|
"caller_start_line",
|
|
151
222
|
"caller_qname",
|
|
223
|
+
"caller_name",
|
|
224
|
+
"caller_arity",
|
|
152
225
|
"callee_file",
|
|
153
226
|
"callee_start_line",
|
|
154
227
|
"callee_qname",
|
|
228
|
+
"callee_name",
|
|
229
|
+
"callee_arity",
|
|
155
230
|
"call_start_line",
|
|
156
231
|
"call_start_column",
|
|
157
232
|
"call_end_line",
|
|
@@ -162,24 +237,15 @@ class CodeQL:
|
|
|
162
237
|
return df
|
|
163
238
|
|
|
164
239
|
@staticmethod
|
|
165
|
-
def
|
|
240
|
+
def _build_callable_resolver(
|
|
166
241
|
symbol_table: Dict[str, PyModule],
|
|
167
|
-
) ->
|
|
168
|
-
"""Build
|
|
242
|
+
) -> _CallableResolver:
|
|
243
|
+
"""Build the endpoint -> ``PyCallable`` resolver from Jedi.
|
|
169
244
|
|
|
170
245
|
Paths are resolved so they match CodeQL's ``getAbsolutePath()``
|
|
171
246
|
regardless of symlinks or the current working directory.
|
|
172
247
|
"""
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
index: Dict[Tuple[str, int], PyCallable] = {}
|
|
176
|
-
for c in iter_callables_in_symbol_table(symbol_table):
|
|
177
|
-
try:
|
|
178
|
-
abs_path = str(Path(c.path).resolve())
|
|
179
|
-
except (OSError, RuntimeError):
|
|
180
|
-
abs_path = c.path
|
|
181
|
-
index[(abs_path, c.start_line)] = c
|
|
182
|
-
return index
|
|
248
|
+
return _CallableResolver.from_symbol_table(symbol_table)
|
|
183
249
|
|
|
184
250
|
def _iter_resolved_rows(
|
|
185
251
|
self, symbol_table: Dict[str, PyModule]
|
|
@@ -194,19 +260,27 @@ class CodeQL:
|
|
|
194
260
|
df = self._query_call_edges()
|
|
195
261
|
if df.empty:
|
|
196
262
|
return
|
|
197
|
-
|
|
263
|
+
resolver = self._build_callable_resolver(symbol_table)
|
|
198
264
|
|
|
199
265
|
skipped_unknown_caller = 0
|
|
200
266
|
ghost_callees = 0
|
|
201
267
|
for row in df.itertuples(index=False):
|
|
202
|
-
|
|
203
|
-
|
|
268
|
+
caller = resolver.resolve(
|
|
269
|
+
row.caller_file,
|
|
270
|
+
int(row.caller_start_line),
|
|
271
|
+
row.caller_name,
|
|
272
|
+
int(row.caller_arity),
|
|
273
|
+
)
|
|
204
274
|
if caller is None:
|
|
205
275
|
skipped_unknown_caller += 1
|
|
206
276
|
continue
|
|
207
277
|
|
|
208
|
-
|
|
209
|
-
|
|
278
|
+
callee = resolver.resolve(
|
|
279
|
+
row.callee_file,
|
|
280
|
+
int(row.callee_start_line),
|
|
281
|
+
row.callee_name,
|
|
282
|
+
int(row.callee_arity),
|
|
283
|
+
)
|
|
210
284
|
if callee is not None:
|
|
211
285
|
target_sig = callee.signature
|
|
212
286
|
else:
|
|
@@ -267,20 +341,28 @@ class CodeQL:
|
|
|
267
341
|
Returns:
|
|
268
342
|
Number of ``PyCallsite`` entries augmented.
|
|
269
343
|
"""
|
|
270
|
-
|
|
344
|
+
resolver = self._build_callable_resolver(symbol_table)
|
|
271
345
|
df = self._query_call_edges()
|
|
272
346
|
if df.empty:
|
|
273
347
|
return 0
|
|
274
348
|
|
|
275
349
|
augmented = 0
|
|
276
350
|
for row in df.itertuples(index=False):
|
|
277
|
-
|
|
278
|
-
|
|
351
|
+
caller = resolver.resolve(
|
|
352
|
+
row.caller_file,
|
|
353
|
+
int(row.caller_start_line),
|
|
354
|
+
row.caller_name,
|
|
355
|
+
int(row.caller_arity),
|
|
356
|
+
)
|
|
279
357
|
if caller is None:
|
|
280
358
|
continue
|
|
281
359
|
|
|
282
|
-
|
|
283
|
-
|
|
360
|
+
callee = resolver.resolve(
|
|
361
|
+
row.callee_file,
|
|
362
|
+
int(row.callee_start_line),
|
|
363
|
+
row.callee_name,
|
|
364
|
+
int(row.callee_arity),
|
|
365
|
+
)
|
|
284
366
|
resolved_sig = callee.signature if callee is not None else row.callee_qname
|
|
285
367
|
|
|
286
368
|
call_start = int(row.call_start_line)
|
|
@@ -0,0 +1,415 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: codeanalyzer-python
|
|
3
|
+
Version: 0.2.1
|
|
4
|
+
Summary: Static Analysis on Python source code using Jedi, CodeQL and Treesitter — emits analysis.json or a Neo4j property graph.
|
|
5
|
+
Author-email: Rahul Krishna <i.m.ralk@gmail.com>
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
License-File: NOTICE
|
|
8
|
+
Requires-Python: >=3.9
|
|
9
|
+
Requires-Dist: jedi<0.20.0,>=0.18.0; python_version < '3.11'
|
|
10
|
+
Requires-Dist: jedi<=0.19.2; python_version >= '3.11'
|
|
11
|
+
Requires-Dist: msgpack<1.0.7,>=1.0.0; python_version < '3.11'
|
|
12
|
+
Requires-Dist: msgpack<2.0.0,>=1.0.7; python_version >= '3.11'
|
|
13
|
+
Requires-Dist: networkx<3.2.0,>=2.6.0; python_version < '3.11'
|
|
14
|
+
Requires-Dist: networkx<4.0.0,>=3.0.0; python_version >= '3.11'
|
|
15
|
+
Requires-Dist: numpy<1.24.0,>=1.21.0; python_version < '3.11'
|
|
16
|
+
Requires-Dist: numpy<2.0.0,>=1.24.0; python_version >= '3.11' and python_version < '3.12'
|
|
17
|
+
Requires-Dist: numpy<2.0.0,>=1.26.0; python_version >= '3.12'
|
|
18
|
+
Requires-Dist: packaging>=25.0
|
|
19
|
+
Requires-Dist: pandas<2.0.0,>=1.3.0; python_version < '3.11'
|
|
20
|
+
Requires-Dist: pandas<3.0.0,>=2.0.0; python_version >= '3.11'
|
|
21
|
+
Requires-Dist: pydantic<2.0.0,>=1.8.0; python_version < '3.11'
|
|
22
|
+
Requires-Dist: pydantic<3.0.0,>=2.0.0; python_version >= '3.11'
|
|
23
|
+
Requires-Dist: ray<3.0.0,>=2.10.0; python_version >= '3.11'
|
|
24
|
+
Requires-Dist: ray==2.0.0; python_version < '3.11'
|
|
25
|
+
Requires-Dist: requests<3.0.0,>=2.20.0; python_version >= '3.11'
|
|
26
|
+
Requires-Dist: rich<14.0.0,>=12.6.0; python_version < '3.11'
|
|
27
|
+
Requires-Dist: rich<15.0.0,>=14.0.0; python_version >= '3.11'
|
|
28
|
+
Requires-Dist: typer<1.0.0,>=0.9.0; python_version < '3.11'
|
|
29
|
+
Requires-Dist: typer<2.0.0,>=0.9.0; python_version >= '3.11'
|
|
30
|
+
Requires-Dist: typing-extensions<5.0.0,>=4.0.0; python_version < '3.11'
|
|
31
|
+
Requires-Dist: typing-extensions<6.0.0,>=4.5.0; python_version >= '3.11'
|
|
32
|
+
Requires-Dist: uv>=0.5.0
|
|
33
|
+
Provides-Extra: neo4j
|
|
34
|
+
Requires-Dist: neo4j<6.0.0,>=5.0.0; extra == 'neo4j'
|
|
35
|
+
Description-Content-Type: text/markdown
|
|
36
|
+
|
|
37
|
+
<div align="center">
|
|
38
|
+
|
|
39
|
+
<img src="https://github.com/codellm-devkit/codeanalyzer-python/blob/main/docs/assets/logo.png?raw=true" alt="CodeLLM-DevKit" />
|
|
40
|
+
|
|
41
|
+
# codeanalyzer-python (`canpy`)
|
|
42
|
+
|
|
43
|
+
**A Python static-analysis toolkit — the CLDK backend that emits a canonical symbol table and call graph, as `analysis.json` or a Neo4j property graph.**
|
|
44
|
+
|
|
45
|
+
[](https://pypi.org/project/codeanalyzer-python/)
|
|
46
|
+
[](https://pypi.org/project/codeanalyzer-python/)
|
|
47
|
+
[](https://github.com/codellm-devkit/codeanalyzer-python/releases/latest)
|
|
48
|
+
[](https://github.com/codellm-devkit/codeanalyzer-python/actions/workflows/release.yml)
|
|
49
|
+
[](./LICENSE)
|
|
50
|
+
|
|
51
|
+
</div>
|
|
52
|
+
|
|
53
|
+
---
|
|
54
|
+
|
|
55
|
+
`canpy` is a static analyzer for Python built on [Jedi](https://jedi.readthedocs.io/), with optional
|
|
56
|
+
[CodeQL](https://codeql.github.com/)-resolved call edges and
|
|
57
|
+
[Tree-sitter](https://tree-sitter.github.io/) parsing. It produces the canonical CodeLLM-DevKit
|
|
58
|
+
(CLDK) `analysis.json` — a symbol table plus a call graph — and can project that same analysis into a
|
|
59
|
+
**Neo4j property graph**. It is the Python backend behind
|
|
60
|
+
[CLDK](https://github.com/codellm-devkit/python-sdk), mirroring its
|
|
61
|
+
[TypeScript](https://github.com/codellm-devkit/codeanalyzer-typescript) (`cants`) and
|
|
62
|
+
[Java](https://github.com/codellm-devkit/codeanalyzer-java) siblings.
|
|
63
|
+
|
|
64
|
+
Every run produces a symbol table **and** a call graph. Edges come from Jedi's lexical resolution by
|
|
65
|
+
default; `--codeql` resolves additional edges (RPC / third-party / dynamically-dispatched targets)
|
|
66
|
+
and merges them with the Jedi-derived edges, also backfilling callees Jedi could not resolve.
|
|
67
|
+
|
|
68
|
+
## Table of Contents
|
|
69
|
+
|
|
70
|
+
- [Features](#features)
|
|
71
|
+
- [Installation](#installation)
|
|
72
|
+
- [Prerequisites](#prerequisites)
|
|
73
|
+
- [Install via pip (PyPI)](#install-via-pip-pypi)
|
|
74
|
+
- [Install via shell script](#install-via-shell-script)
|
|
75
|
+
- [Install via Homebrew](#install-via-homebrew)
|
|
76
|
+
- [Build from source](#build-from-source)
|
|
77
|
+
- [Usage](#usage)
|
|
78
|
+
- [Options](#options)
|
|
79
|
+
- [Examples](#examples)
|
|
80
|
+
- [Output targets](#output-targets)
|
|
81
|
+
- [`analysis.json` (default)](#analysisjson-default)
|
|
82
|
+
- [Neo4j graph](#neo4j-graph)
|
|
83
|
+
- [Schema contract](#schema-contract)
|
|
84
|
+
- [Development](#development)
|
|
85
|
+
- [License](#license)
|
|
86
|
+
|
|
87
|
+
## Features
|
|
88
|
+
|
|
89
|
+
- **Symbol table** — modules, classes, functions, methods, variables, decorators, imports, and
|
|
90
|
+
docstrings, with precise source spans.
|
|
91
|
+
- **Call graph** — Jedi's lexical resolver by default, with optional **CodeQL**-resolved edges
|
|
92
|
+
(`--codeql`) for RPC / third-party / dynamically-dispatched targets, merged with the Jedi edges;
|
|
93
|
+
CodeQL also backfills callees Jedi could not resolve.
|
|
94
|
+
- **Neo4j output** — project the analysis into a labeled property graph: a self-contained
|
|
95
|
+
`graph.cypher` snapshot, or an **incremental** push to a live database over Bolt.
|
|
96
|
+
- **Versioned schema** — a machine-readable, version-stamped Neo4j schema contract (`--emit schema`),
|
|
97
|
+
checked in as `schema.neo4j.json` and shipped with every release.
|
|
98
|
+
- **Incremental cache** — per-file results are cached under `.codeanalyzer`; `--lazy` (default)
|
|
99
|
+
reuses them, `--eager` forces a clean rebuild. `--ray` distributes the work across cores.
|
|
100
|
+
- **Compact output** — canonical `analysis.json`, or binary `analysis.msgpack` for smaller artifacts.
|
|
101
|
+
|
|
102
|
+
## Installation
|
|
103
|
+
|
|
104
|
+
### Prerequisites
|
|
105
|
+
|
|
106
|
+
- **Python 3.10 or newer.**
|
|
107
|
+
- A C toolchain and the `venv` / development headers — the analyzer builds an isolated virtual
|
|
108
|
+
environment per project (via Python's `venv`) so Jedi can resolve types and imports:
|
|
109
|
+
|
|
110
|
+
```sh
|
|
111
|
+
# Ubuntu / Debian
|
|
112
|
+
sudo apt install python3-venv python3-dev build-essential
|
|
113
|
+
|
|
114
|
+
# Fedora / RHEL / CentOS
|
|
115
|
+
sudo dnf group install "Development Tools" && sudo dnf install python3-venv python3-devel
|
|
116
|
+
|
|
117
|
+
# macOS
|
|
118
|
+
xcode-select --install
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
### Install via pip (PyPI)
|
|
122
|
+
|
|
123
|
+
```sh
|
|
124
|
+
pip install codeanalyzer-python
|
|
125
|
+
canpy --help
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
For the optional **live Neo4j push** (`--emit neo4j --neo4j-uri …`), install the `neo4j` extra:
|
|
129
|
+
|
|
130
|
+
```sh
|
|
131
|
+
pip install 'codeanalyzer-python[neo4j]'
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
### Install via shell script
|
|
135
|
+
|
|
136
|
+
Install the CLI as an isolated tool with the one-line installer (provisions via uv / pipx / pip):
|
|
137
|
+
|
|
138
|
+
```sh
|
|
139
|
+
curl --proto '=https' --tlsv1.2 -LsSf https://github.com/codellm-devkit/codeanalyzer-python/releases/latest/download/canpy-installer.sh | sh
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
### Install via Homebrew
|
|
143
|
+
|
|
144
|
+
```sh
|
|
145
|
+
brew install codellm-devkit/tap/codeanalyzer-python
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
The formula depends on [uv](https://docs.astral.sh/uv/) and installs `canpy` as an isolated,
|
|
149
|
+
version-pinned uv tool (the package and its dependencies are resolved and cached on first run).
|
|
150
|
+
|
|
151
|
+
### Build from source
|
|
152
|
+
|
|
153
|
+
This project uses [uv](https://docs.astral.sh/uv/) for dependency management.
|
|
154
|
+
|
|
155
|
+
```sh
|
|
156
|
+
git clone https://github.com/codellm-devkit/codeanalyzer-python
|
|
157
|
+
cd codeanalyzer-python
|
|
158
|
+
uv sync --all-groups
|
|
159
|
+
uv run canpy --help
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
## Usage
|
|
163
|
+
|
|
164
|
+
```sh
|
|
165
|
+
canpy --input /path/to/python/project
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
With no `--output`, the analysis is printed to stdout as compact JSON; with `--output <dir>` it is
|
|
169
|
+
written to `analysis.json` (or `graph.cypher` for `--emit neo4j`, or `analysis.msgpack` with
|
|
170
|
+
`--format msgpack`) in that directory.
|
|
171
|
+
|
|
172
|
+
### Options
|
|
173
|
+
|
|
174
|
+
<!-- BEGIN canpy-help -->
|
|
175
|
+
|
|
176
|
+
```text
|
|
177
|
+
$ canpy --help
|
|
178
|
+
|
|
179
|
+
Usage: canpy [OPTIONS] COMMAND [ARGS]...
|
|
180
|
+
|
|
181
|
+
Static Analysis on Python source code using Jedi, CodeQL and Tree sitter.
|
|
182
|
+
|
|
183
|
+
╭─ Options ────────────────────────────────────────────────────────────────────╮
|
|
184
|
+
│ --input -i PATH Path to the │
|
|
185
|
+
│ project root │
|
|
186
|
+
│ directory (not │
|
|
187
|
+
│ required for │
|
|
188
|
+
│ --emit schema). │
|
|
189
|
+
│ --output -o PATH Output directory │
|
|
190
|
+
│ for artifacts. │
|
|
191
|
+
│ --format -f [json|msgpack] Output format for │
|
|
192
|
+
│ --emit json: json │
|
|
193
|
+
│ or msgpack. │
|
|
194
|
+
│ [default: json] │
|
|
195
|
+
│ --emit [json|neo4j|sche Output target: │
|
|
196
|
+
│ ma] json │
|
|
197
|
+
│ (analysis.json, │
|
|
198
|
+
│ default) | neo4j │
|
|
199
|
+
│ (graph.cypher or │
|
|
200
|
+
│ live Bolt push) | │
|
|
201
|
+
│ schema (the Neo4j │
|
|
202
|
+
│ schema.json │
|
|
203
|
+
│ contract). │
|
|
204
|
+
│ [default: json] │
|
|
205
|
+
│ --app-name TEXT Logical │
|
|
206
|
+
│ application name │
|
|
207
|
+
│ for the graph │
|
|
208
|
+
│ :PyApplication │
|
|
209
|
+
│ anchor (default: │
|
|
210
|
+
│ input dir name). │
|
|
211
|
+
│ --neo4j-uri TEXT Push the graph to │
|
|
212
|
+
│ a live Neo4j over │
|
|
213
|
+
│ Bolt │
|
|
214
|
+
│ (incremental); │
|
|
215
|
+
│ omit to write │
|
|
216
|
+
│ graph.cypher. │
|
|
217
|
+
│ [env var: │
|
|
218
|
+
│ NEO4J_URI] │
|
|
219
|
+
│ --neo4j-user TEXT Neo4j username. │
|
|
220
|
+
│ [env var: │
|
|
221
|
+
│ NEO4J_USERNAME] │
|
|
222
|
+
│ [default: neo4j] │
|
|
223
|
+
│ --neo4j-password TEXT Neo4j password. │
|
|
224
|
+
│ Prefer the env │
|
|
225
|
+
│ var over the flag │
|
|
226
|
+
│ (the flag is │
|
|
227
|
+
│ visible in shell │
|
|
228
|
+
│ history / process │
|
|
229
|
+
│ list). │
|
|
230
|
+
│ [env var: │
|
|
231
|
+
│ NEO4J_PASSWORD] │
|
|
232
|
+
│ [default: neo4j] │
|
|
233
|
+
│ --neo4j-database TEXT Neo4j database │
|
|
234
|
+
│ name (default: │
|
|
235
|
+
│ server default). │
|
|
236
|
+
│ [env var: │
|
|
237
|
+
│ NEO4J_DATABASE] │
|
|
238
|
+
│ --codeql --no-codeql Enable │
|
|
239
|
+
│ CodeQL-based │
|
|
240
|
+
│ analysis. │
|
|
241
|
+
│ [default: │
|
|
242
|
+
│ no-codeql] │
|
|
243
|
+
│ --ray --no-ray Enable Ray for │
|
|
244
|
+
│ distributed │
|
|
245
|
+
│ analysis. │
|
|
246
|
+
│ [default: no-ray] │
|
|
247
|
+
│ --eager --lazy Enable eager or │
|
|
248
|
+
│ lazy analysis. │
|
|
249
|
+
│ Defaults to lazy. │
|
|
250
|
+
│ [default: lazy] │
|
|
251
|
+
│ --skip-tests --include-tests Skip test files │
|
|
252
|
+
│ in analysis. │
|
|
253
|
+
│ [default: │
|
|
254
|
+
│ skip-tests] │
|
|
255
|
+
│ --no-venv --venv Skip virtualenv │
|
|
256
|
+
│ creation and │
|
|
257
|
+
│ dependency │
|
|
258
|
+
│ installation; │
|
|
259
|
+
│ resolve imports │
|
|
260
|
+
│ against the │
|
|
261
|
+
│ ambient Python │
|
|
262
|
+
│ environment │
|
|
263
|
+
│ instead. │
|
|
264
|
+
│ [default: venv] │
|
|
265
|
+
│ --file-name PATH Analyze only the │
|
|
266
|
+
│ specified file │
|
|
267
|
+
│ (relative to │
|
|
268
|
+
│ input directory). │
|
|
269
|
+
│ --cache-dir -c PATH Directory to │
|
|
270
|
+
│ store analysis │
|
|
271
|
+
│ cache. Defaults │
|
|
272
|
+
│ to │
|
|
273
|
+
│ '.codeanalyzer' │
|
|
274
|
+
│ in the input │
|
|
275
|
+
│ directory. │
|
|
276
|
+
│ --clear-cache --keep-cache Clear cache after │
|
|
277
|
+
│ analysis. By │
|
|
278
|
+
│ default, cache is │
|
|
279
|
+
│ retained. │
|
|
280
|
+
│ [default: │
|
|
281
|
+
│ keep-cache] │
|
|
282
|
+
│ -v INTEGER Increase │
|
|
283
|
+
│ verbosity: -v, │
|
|
284
|
+
│ -vv, -vvv │
|
|
285
|
+
│ [default: 0] │
|
|
286
|
+
│ --help Show this message │
|
|
287
|
+
│ and exit. │
|
|
288
|
+
╰──────────────────────────────────────────────────────────────────────────────╯
|
|
289
|
+
```
|
|
290
|
+
|
|
291
|
+
<!-- END canpy-help -->
|
|
292
|
+
|
|
293
|
+
### Examples
|
|
294
|
+
|
|
295
|
+
1. **Basic analysis to stdout, or to a file:**
|
|
296
|
+
```sh
|
|
297
|
+
canpy --input ./my-python-project # compact JSON on stdout
|
|
298
|
+
canpy --input ./my-python-project --output ./out # → ./out/analysis.json
|
|
299
|
+
```
|
|
300
|
+
|
|
301
|
+
2. **Binary output (msgpack):**
|
|
302
|
+
```sh
|
|
303
|
+
canpy --input ./my-python-project --output ./out --format msgpack # → ./out/analysis.msgpack
|
|
304
|
+
```
|
|
305
|
+
|
|
306
|
+
3. **Resolve extra call edges with CodeQL:**
|
|
307
|
+
```sh
|
|
308
|
+
canpy --input ./my-python-project --codeql
|
|
309
|
+
```
|
|
310
|
+
By default, edges come from Jedi's lexical analysis. Adding `--codeql` resolves additional edges
|
|
311
|
+
(including RPC / third-party / dynamically-dispatched targets) and merges them with the
|
|
312
|
+
Jedi-derived edges; CodeQL also backfills resolved callees Jedi could not resolve. CodeQL
|
|
313
|
+
integration is experimental; the CLI is downloaded into `<cache_dir>/codeql/` on first use.
|
|
314
|
+
|
|
315
|
+
4. **Emit a Neo4j snapshot, or push to a live database:**
|
|
316
|
+
```sh
|
|
317
|
+
canpy --input ./my-python-project --emit neo4j --output ./out # → ./out/graph.cypher
|
|
318
|
+
canpy --input ./my-python-project --emit neo4j \
|
|
319
|
+
--neo4j-uri bolt://localhost:7687 --neo4j-user neo4j --neo4j-password secret
|
|
320
|
+
```
|
|
321
|
+
|
|
322
|
+
5. **Emit the Neo4j schema contract:**
|
|
323
|
+
```sh
|
|
324
|
+
canpy --emit schema # print schema.json to stdout (no project needed)
|
|
325
|
+
canpy --emit schema --output ./out # → ./out/schema.json
|
|
326
|
+
```
|
|
327
|
+
|
|
328
|
+
6. **Force a clean rebuild with a custom cache directory:**
|
|
329
|
+
```sh
|
|
330
|
+
canpy --input ./my-python-project --eager --cache-dir /path/to/custom-cache
|
|
331
|
+
```
|
|
332
|
+
|
|
333
|
+
## Output targets
|
|
334
|
+
|
|
335
|
+
`canpy` builds one analysis in memory and can emit it three ways (`--emit`):
|
|
336
|
+
|
|
337
|
+
### `analysis.json` (default)
|
|
338
|
+
|
|
339
|
+
A `PyApplication` document — the canonical CLDK contract:
|
|
340
|
+
|
|
341
|
+
```jsonc
|
|
342
|
+
{
|
|
343
|
+
"symbol_table": { /* file path → module (classes, functions, variables, imports, …) */ },
|
|
344
|
+
"call_graph": [ /* CALL_DEP edges: { source, target, weight, provenance } keyed by callable signature */ ]
|
|
345
|
+
}
|
|
346
|
+
```
|
|
347
|
+
|
|
348
|
+
By default this is printed to stdout in JSON; with `--output` it is written to `analysis.json` (or
|
|
349
|
+
`analysis.msgpack` with `--format msgpack`, a more compact binary format).
|
|
350
|
+
|
|
351
|
+
### Neo4j graph
|
|
352
|
+
|
|
353
|
+
`--emit neo4j` projects the same analysis into a labeled property graph. Every node label is
|
|
354
|
+
`Py`-prefixed and every relationship type is `PY_`-prefixed (e.g. `:PyClass`, `PY_CALLS`) so multiple
|
|
355
|
+
language analyzers can share one database without label or relationship-type collisions. Declarations
|
|
356
|
+
are keyed by their signature under a shared `:PySymbol` label; calls, imports, inheritance,
|
|
357
|
+
decorators, and call sites are relationships:
|
|
358
|
+
|
|
359
|
+
- **Without `--neo4j-uri`** — writes a self-contained `graph.cypher` (constraints + indexes, a scoped
|
|
360
|
+
wipe, then batched `MERGE`s). Load it with `cypher-shell < graph.cypher`. Needs no extra
|
|
361
|
+
dependencies.
|
|
362
|
+
- **With `--neo4j-uri`** — pushes to a live Neo4j over Bolt **incrementally**: only modules whose
|
|
363
|
+
content hash changed are rewritten, and on a full run modules whose source file vanished are
|
|
364
|
+
pruned. Requires the `neo4j` extra. Every graph carries a `schema_version` on its `:PyApplication`
|
|
365
|
+
node.
|
|
366
|
+
|
|
367
|
+
Call-graph endpoints that aren't present in the symbol table (third-party / framework / RPC targets)
|
|
368
|
+
are materialized as `:PyExternal` ghost nodes, mirroring the analyzer's own ghost-node behaviour.
|
|
369
|
+
|
|
370
|
+
The connection options also read from the standard Neo4j environment variables — `NEO4J_URI`,
|
|
371
|
+
`NEO4J_USERNAME`, `NEO4J_PASSWORD`, `NEO4J_DATABASE` — when the corresponding flag is omitted (an
|
|
372
|
+
explicit flag wins). Prefer the env var for the password so it doesn't land in shell history or the
|
|
373
|
+
process list:
|
|
374
|
+
|
|
375
|
+
```sh
|
|
376
|
+
export NEO4J_URI=bolt://localhost:7687
|
|
377
|
+
export NEO4J_PASSWORD=secret
|
|
378
|
+
canpy -i ./my-project --emit neo4j # credentials picked up from the environment
|
|
379
|
+
```
|
|
380
|
+
|
|
381
|
+
### Schema contract
|
|
382
|
+
|
|
383
|
+
`--emit schema` writes the machine-readable, version-stamped Neo4j schema (`schema.json`: node labels,
|
|
384
|
+
relationships, properties, constraints, and indexes). It needs no project and is checked into the repo
|
|
385
|
+
as `schema.neo4j.json` and bundled in every release as a GitHub Release asset, so a consumer can
|
|
386
|
+
validate producer/consumer compatibility without invoking the tool. The shape of the contract matches
|
|
387
|
+
the [`codeanalyzer-typescript`](https://github.com/codellm-devkit/codeanalyzer-typescript) backend.
|
|
388
|
+
|
|
389
|
+
A UML of the `analysis.json` schema (the `PyApplication` containment tree) is checked in as
|
|
390
|
+
[`schema-uml.drawio`](./schema-uml.drawio), and the property-graph schema as
|
|
391
|
+
[`neo4j-schema.drawio`](./neo4j-schema.drawio).
|
|
392
|
+
|
|
393
|
+
## Development
|
|
394
|
+
|
|
395
|
+
This project uses [uv](https://docs.astral.sh/uv/).
|
|
396
|
+
|
|
397
|
+
```sh
|
|
398
|
+
uv sync --all-groups
|
|
399
|
+
uv run canpy --input /path/to/project # run from source
|
|
400
|
+
uv run canpy --emit schema > schema.neo4j.json # regenerate the checked-in schema contract
|
|
401
|
+
uv run python scripts/update_readme.py # regenerate the canpy --help block above
|
|
402
|
+
uv run pytest # run the test suite
|
|
403
|
+
```
|
|
404
|
+
|
|
405
|
+
The Neo4j schema-conformance test always runs. The Neo4j **bolt** integration test spins up a real
|
|
406
|
+
Neo4j via [Testcontainers](https://testcontainers.com/) and is **opt-in** — it needs a container
|
|
407
|
+
runtime (Docker or Podman) and is enabled with an environment variable:
|
|
408
|
+
|
|
409
|
+
```sh
|
|
410
|
+
RUN_CONTAINER_TESTS=1 uv run pytest test/test_neo4j_bolt.py -s
|
|
411
|
+
```
|
|
412
|
+
|
|
413
|
+
## License
|
|
414
|
+
|
|
415
|
+
Apache 2.0 — see [LICENSE](./LICENSE).
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
codeanalyzer/__init__.py,sha256=BZ3Kuwl-F_F-8H8cepLnVJ4Ku4NNUjjqg0Y6ujPQSsI,108
|
|
2
|
+
codeanalyzer/__main__.py,sha256=UHSBihqrJVQrx-ldp4HBMJQAUouDsiuEakyYt6oWVTw,8444
|
|
3
|
+
codeanalyzer/core.py,sha256=O7hW2Y65hzdxR3Rk1Gt2Mwb63pBGZ-zCk3gyAxb3CRM,33976
|
|
4
|
+
codeanalyzer/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
|
+
codeanalyzer/config/__init__.py,sha256=9XBxAn1oWGRuhg3bEBUuVGs3hFNXEAKrr-Ce7tq9a2k,61
|
|
6
|
+
codeanalyzer/config/config.py,sha256=ZiKzc5uEUCIvih58-6BDtLLI1hPij41wGQjBcj9KNQM,188
|
|
7
|
+
codeanalyzer/jedi/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
codeanalyzer/jedi/jedi.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
|
+
codeanalyzer/neo4j/__init__.py,sha256=AcbFNAMuXwkMFWH4h_HCmla6PCKTF3Xe5yNqg8F_kYk,1575
|
|
10
|
+
codeanalyzer/neo4j/bolt.py,sha256=-IFDb_d67IkGwxvmmbLNI6AvSQHY-XsutI3szibWidw,9635
|
|
11
|
+
codeanalyzer/neo4j/catalog.py,sha256=VQWtwcueck2Ug_cu8sitvAoRwbVZ-6KRiexRN8q1Wak,7418
|
|
12
|
+
codeanalyzer/neo4j/cypher.py,sha256=2zIWXA1AADrwCMhSTeqKjEXRgBjbob6o3bme_cwLu0s,5024
|
|
13
|
+
codeanalyzer/neo4j/emit.py,sha256=WtCndN6mA6PIzfzdgv9Xc5S5WP4rHUXCtB_r3G16rkg,3101
|
|
14
|
+
codeanalyzer/neo4j/project.py,sha256=2GDHkFjmWibrpMaOiDMo21K6Lnj3aefl2QPCM7ANA7g,12485
|
|
15
|
+
codeanalyzer/neo4j/rows.py,sha256=5xI3X-l-vwPe_gmKYUg7VuUQARcOlVAZnGmiQr9QyRk,7326
|
|
16
|
+
codeanalyzer/neo4j/schema.py,sha256=5xz8cZVuL73GAF-vs9QtN2TuKxIty0rHEe68nwQmLTY,2136
|
|
17
|
+
codeanalyzer/options/__init__.py,sha256=FNBGxdnESayUb0wEs395MKIJxoWIX7bp-FaLDP6qYUw,123
|
|
18
|
+
codeanalyzer/options/options.py,sha256=JfHGU1NwXL_XCkQQsxMM94iuRL96J8kOYAa0pV5bmBM,1275
|
|
19
|
+
codeanalyzer/schema/__init__.py,sha256=cLPjvowrnz8xzi7tZAsKQeIOjdOKRGHy4I7wbG0jHk8,2024
|
|
20
|
+
codeanalyzer/schema/py_schema.py,sha256=sWJ-hWTYpd02Am1548zNEsp5XKeK3B5MbWTqsXAMC_8,12316
|
|
21
|
+
codeanalyzer/semantic_analysis/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
22
|
+
codeanalyzer/semantic_analysis/call_graph.py,sha256=3hgLA1sL1YFQa4fzUz_ifVbLs1I9V2QFe7ldwN18mcg,10323
|
|
23
|
+
codeanalyzer/semantic_analysis/codeql/__init__.py,sha256=ODMkdGvs3ebJdfIZle8T4VcHoCBhH_ZehWuWFpNh3NI,1022
|
|
24
|
+
codeanalyzer/semantic_analysis/codeql/codeql_analysis.py,sha256=Z5E7Pj__eYsYSWFteOG1B4RTlH4huhEaPAOmTquzCBQ,15434
|
|
25
|
+
codeanalyzer/semantic_analysis/codeql/codeql_exceptions.py,sha256=PnJOasW9rP68SEX158jSqQFdqjW_Q_Fx3vbH6vNiCQs,474
|
|
26
|
+
codeanalyzer/semantic_analysis/codeql/codeql_loader.py,sha256=XFjTx6ERRipzTguwHzSS5BTI6Nzf95fcdK3nLxqHARs,3599
|
|
27
|
+
codeanalyzer/semantic_analysis/codeql/codeql_query_runner.py,sha256=Zr5NPIpQjm8W-up-js8f3Q7Y9YYfSfUalm6yxtdTC90,7768
|
|
28
|
+
codeanalyzer/syntactic_analysis/__init__.py,sha256=EUQkJEh6wHjWx2qTTKbTbUgwSbfKeNieKHNy7RknVXA,476
|
|
29
|
+
codeanalyzer/syntactic_analysis/exceptions.py,sha256=whs_n0vIu655Jkk1a7iOoXY6iIca4pZqJnU40V9Ejaw,537
|
|
30
|
+
codeanalyzer/syntactic_analysis/symbol_table_builder.py,sha256=zmHFt8pN50jG-Ex4fnisvbLmn1XaW05jwbV_xSG4qfU,38177
|
|
31
|
+
codeanalyzer/utils/__init__.py,sha256=hC6VWdR5rerSqBxzu9KQHTASWqwrrYJv-CMDwrTlzkc,137
|
|
32
|
+
codeanalyzer/utils/logging.py,sha256=0vTkGSl5EZN8yhhWa_5Mrn1n_twRCSW53rNwjzQ9RbI,601
|
|
33
|
+
codeanalyzer/utils/progress_bar.py,sha256=ZHJzGiCo5q4dyXq4CtsrJeq9Ip7sD84T3yZjNX7TBys,2443
|
|
34
|
+
codeanalyzer_python-0.2.1.dist-info/METADATA,sha256=UN2Bnyh-t1B3EzvQk708_n86Cp5kAOvyxcLTrk26g5k,22430
|
|
35
|
+
codeanalyzer_python-0.2.1.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
|
|
36
|
+
codeanalyzer_python-0.2.1.dist-info/entry_points.txt,sha256=v4Vux0Nnx7sOntVk_CH7W9RX6SkIkvR1FQYq73oVlCQ,105
|
|
37
|
+
codeanalyzer_python-0.2.1.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
38
|
+
codeanalyzer_python-0.2.1.dist-info/licenses/NOTICE,sha256=YU0Z9NDWqKY-2jfFcbxeZ6fbnzz0oZeKmnUcO8a-bcQ,901
|
|
39
|
+
codeanalyzer_python-0.2.1.dist-info/RECORD,,
|