codeanalyzer-python 0.2.1__py3-none-any.whl → 0.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codeanalyzer/__main__.py +116 -6
- codeanalyzer/core.py +139 -213
- codeanalyzer/neo4j/__init__.py +1 -1
- codeanalyzer/neo4j/emit.py +2 -2
- codeanalyzer/neo4j/project.py +84 -18
- codeanalyzer/neo4j/schema.py +261 -15
- codeanalyzer/options/__init__.py +2 -2
- codeanalyzer/options/options.py +20 -1
- codeanalyzer/provenance.py +61 -0
- codeanalyzer/schema/py_schema.py +39 -1
- codeanalyzer/semantic_analysis/call_graph.py +24 -3
- codeanalyzer/semantic_analysis/pycg/__init__.py +20 -0
- codeanalyzer/semantic_analysis/pycg/pycg_analysis.py +1054 -0
- codeanalyzer/semantic_analysis/{codeql/__init__.py → pycg/pycg_exceptions.py} +5 -8
- codeanalyzer/semantic_analysis/pycg/shard_planner.py +401 -0
- codeanalyzer/syntactic_analysis/import_resolver.py +67 -0
- codeanalyzer/syntactic_analysis/symbol_table_builder.py +74 -10
- codeanalyzer/utils/logging.py +5 -2
- codeanalyzer/utils/progress_bar.py +15 -7
- codeanalyzer_python-0.3.1.dist-info/METADATA +553 -0
- codeanalyzer_python-0.3.1.dist-info/RECORD +39 -0
- {codeanalyzer_python-0.2.1.dist-info → codeanalyzer_python-0.3.1.dist-info}/WHEEL +1 -1
- codeanalyzer/neo4j/catalog.py +0 -245
- codeanalyzer/semantic_analysis/codeql/codeql_analysis.py +0 -382
- codeanalyzer/semantic_analysis/codeql/codeql_exceptions.py +0 -12
- codeanalyzer/semantic_analysis/codeql/codeql_loader.py +0 -91
- codeanalyzer/semantic_analysis/codeql/codeql_query_runner.py +0 -185
- codeanalyzer_python-0.2.1.dist-info/METADATA +0 -415
- codeanalyzer_python-0.2.1.dist-info/RECORD +0 -39
- {codeanalyzer_python-0.2.1.dist-info → codeanalyzer_python-0.3.1.dist-info}/entry_points.txt +0 -0
- {codeanalyzer_python-0.2.1.dist-info → codeanalyzer_python-0.3.1.dist-info}/licenses/LICENSE +0 -0
- {codeanalyzer_python-0.2.1.dist-info → codeanalyzer_python-0.3.1.dist-info}/licenses/NOTICE +0 -0
|
@@ -14,13 +14,10 @@
|
|
|
14
14
|
# limitations under the License.
|
|
15
15
|
################################################################################
|
|
16
16
|
|
|
17
|
-
"""
|
|
18
|
-
CodeQL package
|
|
19
|
-
"""
|
|
20
17
|
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
from .codeql_query_runner import CodeQLQueryRunner
|
|
18
|
+
class PyCGExceptions:
|
|
19
|
+
class PyCGAnalysisError(Exception):
|
|
20
|
+
"""Raised when PyCG fails to analyze a project."""
|
|
25
21
|
|
|
26
|
-
|
|
22
|
+
class PyCGImportError(ImportError):
|
|
23
|
+
"""Raised when the pycg package is not installed."""
|
|
@@ -0,0 +1,401 @@
|
|
|
1
|
+
################################################################################
|
|
2
|
+
# Copyright IBM Corporation 2025
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
################################################################################
|
|
16
|
+
|
|
17
|
+
"""Coupling-aware shard planning for PyCG call-graph construction.
|
|
18
|
+
|
|
19
|
+
Sharding splits a project so PyCG can be run on each part independently; any
|
|
20
|
+
call edge whose caller and callee land in *different* shards becomes a ghost
|
|
21
|
+
node in both, so PyCG never resolves it. The directory + file-count heuristic
|
|
22
|
+
is blind to coupling and can sever heavily-interacting modules. This planner
|
|
23
|
+
instead partitions the **module dependency graph derived from the Jedi call
|
|
24
|
+
graph** (already computed at analysis level 1), so the cuts fall on the weakest
|
|
25
|
+
seams.
|
|
26
|
+
|
|
27
|
+
Pipeline:
|
|
28
|
+
|
|
29
|
+
1. **Module graph** — project the Jedi ``PyCallEdge`` list (callable→callable)
|
|
30
|
+
down to a weighted directed graph over *modules*. ``weight(A, B)`` is the
|
|
31
|
+
number of Jedi call sites from a callable in module ``A`` to one in ``B``.
|
|
32
|
+
2. **SCC condensation** — modules in an import/call cycle must be co-computed
|
|
33
|
+
(splitting them breaks both shards), so each strongly-connected component is
|
|
34
|
+
collapsed into one indivisible unit via Tarjan's algorithm.
|
|
35
|
+
3. **Community detection** — Louvain modularity over the undirected weighted
|
|
36
|
+
condensation groups tightly-coupled units; each community is a candidate
|
|
37
|
+
shard.
|
|
38
|
+
4. **Budget enforcement** — communities over the per-shard file budget are
|
|
39
|
+
re-partitioned (higher Louvain resolution, then a greedy first-fit fallback
|
|
40
|
+
that guarantees termination); communities under budget are agglomeratively
|
|
41
|
+
merged by inter-shard weight to recover cut edges and reduce shard count.
|
|
42
|
+
|
|
43
|
+
The result is a list of shards (each a list of file paths) plus a metrics
|
|
44
|
+
report whose headline figure is ``cut_ratio`` — the fraction of inter-module
|
|
45
|
+
Jedi edge weight severed by the partition. Lower is better; it is the
|
|
46
|
+
estimated upper bound on PyCG edges lost to sharding.
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
from __future__ import annotations
|
|
50
|
+
|
|
51
|
+
import logging
|
|
52
|
+
from collections import defaultdict
|
|
53
|
+
from dataclasses import dataclass, field
|
|
54
|
+
from typing import Dict, Iterator, List, Optional, Set, Tuple
|
|
55
|
+
|
|
56
|
+
import networkx as nx
|
|
57
|
+
|
|
58
|
+
from codeanalyzer.schema.py_schema import PyCallable, PyCallEdge, PyClass, PyModule
|
|
59
|
+
|
|
60
|
+
logger = logging.getLogger(__name__)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
# ----------------------------------------------------------------------------
|
|
64
|
+
# Symbol-table walking: callable / class signature -> defining file
|
|
65
|
+
# ----------------------------------------------------------------------------
|
|
66
|
+
|
|
67
|
+
def _walk_callable_sigs(c: PyCallable) -> Iterator[str]:
|
|
68
|
+
yield c.signature
|
|
69
|
+
for inner in c.inner_callables.values():
|
|
70
|
+
yield from _walk_callable_sigs(inner)
|
|
71
|
+
for inner_cls in c.inner_classes.values():
|
|
72
|
+
yield from _walk_class_sigs(inner_cls)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _walk_class_sigs(cls: PyClass) -> Iterator[str]:
|
|
76
|
+
yield cls.signature
|
|
77
|
+
for method in cls.methods.values():
|
|
78
|
+
yield from _walk_callable_sigs(method)
|
|
79
|
+
for inner in cls.inner_classes.values():
|
|
80
|
+
yield from _walk_class_sigs(inner)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _signature_to_file(symbol_table: Dict[str, PyModule]) -> Dict[str, str]:
|
|
84
|
+
"""Map every callable/class signature in the project to its defining file.
|
|
85
|
+
|
|
86
|
+
Built by walking each module's full nesting tree, recording the file that
|
|
87
|
+
actually defines each callable. Files are the partition unit because
|
|
88
|
+
``PyModule.module_name`` is only the file *stem* (``py_file.stem``), which
|
|
89
|
+
collides heavily across a real project (every ``__init__.py``, ``models.py``
|
|
90
|
+
…) — keying the module graph by name would collapse unrelated files into
|
|
91
|
+
one node and silently drop files from shards. ``file_path`` is unique.
|
|
92
|
+
"""
|
|
93
|
+
sig_to_file: Dict[str, str] = {}
|
|
94
|
+
for module in symbol_table.values():
|
|
95
|
+
for fn in module.functions.values():
|
|
96
|
+
for sig in _walk_callable_sigs(fn):
|
|
97
|
+
sig_to_file[sig] = module.file_path
|
|
98
|
+
for cls in module.classes.values():
|
|
99
|
+
for sig in _walk_class_sigs(cls):
|
|
100
|
+
sig_to_file[sig] = module.file_path
|
|
101
|
+
return sig_to_file
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
# ----------------------------------------------------------------------------
|
|
105
|
+
# Result type
|
|
106
|
+
# ----------------------------------------------------------------------------
|
|
107
|
+
|
|
108
|
+
@dataclass
|
|
109
|
+
class ShardPlan:
|
|
110
|
+
"""Output of :func:`plan_shards`.
|
|
111
|
+
|
|
112
|
+
``shards`` is what the PyCG executor consumes: each inner list is the set of
|
|
113
|
+
project file paths to analyse together as one shard. ``module_shards`` is
|
|
114
|
+
the same partition expressed in module names (handy for logging/tests).
|
|
115
|
+
"""
|
|
116
|
+
|
|
117
|
+
shards: List[List[str]] = field(default_factory=list)
|
|
118
|
+
module_shards: List[List[str]] = field(default_factory=list)
|
|
119
|
+
metrics: Dict[str, float] = field(default_factory=dict)
|
|
120
|
+
|
|
121
|
+
def __str__(self) -> str:
|
|
122
|
+
m = self.metrics
|
|
123
|
+
return (
|
|
124
|
+
f"ShardPlan({len(self.shards)} shards, "
|
|
125
|
+
f"cut_ratio={m.get('cut_ratio', 0):.3f}, "
|
|
126
|
+
f"max_shard_files={int(m.get('max_shard_files', 0))}, "
|
|
127
|
+
f"oversized={int(m.get('oversized_shards', 0))})"
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
# ----------------------------------------------------------------------------
|
|
132
|
+
# Module dependency graph
|
|
133
|
+
# ----------------------------------------------------------------------------
|
|
134
|
+
|
|
135
|
+
def build_module_graph(
|
|
136
|
+
symbol_table: Dict[str, PyModule],
|
|
137
|
+
jedi_edges: List[PyCallEdge],
|
|
138
|
+
) -> nx.DiGraph:
|
|
139
|
+
"""Project Jedi callable→callable edges onto a weighted file DiGraph.
|
|
140
|
+
|
|
141
|
+
Nodes are file paths (the unique, collision-free partition unit — see
|
|
142
|
+
:func:`_signature_to_file`); each carries a ``module_name`` attribute for
|
|
143
|
+
readable reporting. Every project file is a node (isolated files
|
|
144
|
+
included). Edge weight is the summed Jedi weight of cross-file call sites;
|
|
145
|
+
intra-file edges and edges touching external/library symbols (no
|
|
146
|
+
symbol-table entry) are dropped — they cannot influence the partition.
|
|
147
|
+
"""
|
|
148
|
+
sig_to_file = _signature_to_file(symbol_table)
|
|
149
|
+
|
|
150
|
+
g = nx.DiGraph()
|
|
151
|
+
for module in symbol_table.values():
|
|
152
|
+
g.add_node(module.file_path, module_name=module.module_name)
|
|
153
|
+
|
|
154
|
+
for edge in jedi_edges:
|
|
155
|
+
src = sig_to_file.get(edge.source)
|
|
156
|
+
dst = sig_to_file.get(edge.target)
|
|
157
|
+
if src is None or dst is None or src == dst:
|
|
158
|
+
continue
|
|
159
|
+
if g.has_edge(src, dst):
|
|
160
|
+
g[src][dst]["weight"] += edge.weight
|
|
161
|
+
else:
|
|
162
|
+
g.add_edge(src, dst, weight=edge.weight)
|
|
163
|
+
return g
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
# ----------------------------------------------------------------------------
|
|
167
|
+
# Partitioning
|
|
168
|
+
# ----------------------------------------------------------------------------
|
|
169
|
+
|
|
170
|
+
def _undirected_weighted(g: nx.DiGraph) -> nx.Graph:
|
|
171
|
+
"""Collapse a directed weighted graph to undirected, summing both directions."""
|
|
172
|
+
h = nx.Graph()
|
|
173
|
+
h.add_nodes_from(g.nodes())
|
|
174
|
+
for u, v, w in g.edges(data="weight", default=1):
|
|
175
|
+
if h.has_edge(u, v):
|
|
176
|
+
h[u][v]["weight"] += w
|
|
177
|
+
else:
|
|
178
|
+
h.add_edge(u, v, weight=w)
|
|
179
|
+
return h
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _communities(h: nx.Graph, resolution: float, seed: int) -> List[Set[str]]:
|
|
183
|
+
"""Weighted community detection, preferring Louvain with a graceful fallback."""
|
|
184
|
+
if h.number_of_nodes() == 0:
|
|
185
|
+
return []
|
|
186
|
+
if hasattr(nx.community, "louvain_communities"):
|
|
187
|
+
return [
|
|
188
|
+
set(c)
|
|
189
|
+
for c in nx.community.louvain_communities(
|
|
190
|
+
h, weight="weight", resolution=resolution, seed=seed
|
|
191
|
+
)
|
|
192
|
+
]
|
|
193
|
+
# networkx < 3.0 (Python 3.9/3.10 floor): greedy modularity has no seed.
|
|
194
|
+
return [set(c) for c in nx.community.greedy_modularity_communities(h, weight="weight")]
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _greedy_bin_pack(
|
|
198
|
+
units: List[Tuple[frozenset, int]], budget: int
|
|
199
|
+
) -> List[Set[str]]:
|
|
200
|
+
"""First-fit-decreasing pack (unit -> modules, size) into <= budget bins.
|
|
201
|
+
|
|
202
|
+
Termination guarantee for the recursive splitter: a community that Louvain
|
|
203
|
+
refuses to cut still gets divided here. A single unit larger than the
|
|
204
|
+
budget (an atomic SCC — a real import cycle) is emitted on its own; that is
|
|
205
|
+
unavoidable without breaking edges.
|
|
206
|
+
"""
|
|
207
|
+
bins: List[Tuple[Set[str], int]] = []
|
|
208
|
+
for members, size in sorted(units, key=lambda u: u[1], reverse=True):
|
|
209
|
+
placed = False
|
|
210
|
+
for b in bins:
|
|
211
|
+
if b[1] + size <= budget:
|
|
212
|
+
b[0].update(members)
|
|
213
|
+
bins[bins.index(b)] = (b[0], b[1] + size)
|
|
214
|
+
placed = True
|
|
215
|
+
break
|
|
216
|
+
if not placed:
|
|
217
|
+
bins.append((set(members), size))
|
|
218
|
+
return [b[0] for b in bins]
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _split_oversized(
|
|
222
|
+
h: nx.Graph,
|
|
223
|
+
community: Set[str],
|
|
224
|
+
unit_size: Dict[str, int],
|
|
225
|
+
budget: int,
|
|
226
|
+
seed: int,
|
|
227
|
+
depth: int = 0,
|
|
228
|
+
) -> List[Set[str]]:
|
|
229
|
+
"""Recursively partition a community whose file count exceeds the budget."""
|
|
230
|
+
size = sum(unit_size[n] for n in community)
|
|
231
|
+
if size <= budget or len(community) == 1:
|
|
232
|
+
# Fits, or is a single atomic SCC unit that cannot be split further.
|
|
233
|
+
return [community]
|
|
234
|
+
|
|
235
|
+
sub = h.subgraph(community)
|
|
236
|
+
# Escalate resolution so Louvain cuts more aggressively each level.
|
|
237
|
+
parts = _communities(sub, resolution=2.0 * (depth + 1), seed=seed)
|
|
238
|
+
parts = [p for p in parts if p]
|
|
239
|
+
|
|
240
|
+
if len(parts) <= 1:
|
|
241
|
+
# Louvain could not divide it (one dense blob) — fall back to bin packing
|
|
242
|
+
# so the budget is still honoured.
|
|
243
|
+
units = [(frozenset([n]), unit_size[n]) for n in community]
|
|
244
|
+
return _greedy_bin_pack(units, budget)
|
|
245
|
+
|
|
246
|
+
out: List[Set[str]] = []
|
|
247
|
+
for p in parts:
|
|
248
|
+
out.extend(_split_oversized(h, p, unit_size, budget, seed, depth + 1))
|
|
249
|
+
return out
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _merge_small(
|
|
253
|
+
shards: List[Set[str]],
|
|
254
|
+
h: nx.Graph,
|
|
255
|
+
unit_size: Dict[str, int],
|
|
256
|
+
budget: int,
|
|
257
|
+
) -> List[Set[str]]:
|
|
258
|
+
"""Agglomeratively merge under-budget shards by inter-shard weight.
|
|
259
|
+
|
|
260
|
+
Recovers cut edges: merging two shards turns the edges between them back
|
|
261
|
+
into intra-shard edges. Greedy — repeatedly merge the heaviest-coupled
|
|
262
|
+
pair that still fits the budget — which is enough to mop up the small
|
|
263
|
+
fragments Louvain leaves behind without reintroducing oversized shards.
|
|
264
|
+
"""
|
|
265
|
+
shards = [set(s) for s in shards]
|
|
266
|
+
sizes = [sum(unit_size[n] for n in s) for s in shards]
|
|
267
|
+
|
|
268
|
+
def pair_weight(a: Set[str], b: Set[str]) -> float:
|
|
269
|
+
w = 0.0
|
|
270
|
+
for u in a:
|
|
271
|
+
for v in h.adj[u]:
|
|
272
|
+
if v in b:
|
|
273
|
+
w += h[u][v]["weight"]
|
|
274
|
+
return w
|
|
275
|
+
|
|
276
|
+
while True:
|
|
277
|
+
best: Optional[Tuple[int, int]] = None
|
|
278
|
+
best_w = 0.0
|
|
279
|
+
for i in range(len(shards)):
|
|
280
|
+
for j in range(i + 1, len(shards)):
|
|
281
|
+
if sizes[i] + sizes[j] > budget:
|
|
282
|
+
continue
|
|
283
|
+
w = pair_weight(shards[i], shards[j])
|
|
284
|
+
if w > best_w:
|
|
285
|
+
best_w, best = w, (i, j)
|
|
286
|
+
if best is None:
|
|
287
|
+
break
|
|
288
|
+
i, j = best
|
|
289
|
+
shards[i] |= shards[j]
|
|
290
|
+
sizes[i] += sizes[j]
|
|
291
|
+
del shards[j]
|
|
292
|
+
del sizes[j]
|
|
293
|
+
|
|
294
|
+
# Final consolidation: first-fit pack any shards that still fit together.
|
|
295
|
+
# Merging zero-coupling shards (e.g. isolated leaf modules) costs no cut
|
|
296
|
+
# weight and avoids spawning a PyCG process per trivial file. Packing
|
|
297
|
+
# never severs an edge, so it is strictly safe; the budget still bounds
|
|
298
|
+
# per-shard PyCG divergence risk.
|
|
299
|
+
units = [(frozenset(s), sz) for s, sz in zip(shards, sizes)]
|
|
300
|
+
return _greedy_bin_pack(units, budget)
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
# ----------------------------------------------------------------------------
|
|
304
|
+
# Public entry point
|
|
305
|
+
# ----------------------------------------------------------------------------
|
|
306
|
+
|
|
307
|
+
def plan_shards(
|
|
308
|
+
symbol_table: Dict[str, PyModule],
|
|
309
|
+
jedi_edges: List[PyCallEdge],
|
|
310
|
+
budget: int = 100,
|
|
311
|
+
seed: int = 42,
|
|
312
|
+
merge_small: bool = True,
|
|
313
|
+
) -> ShardPlan:
|
|
314
|
+
"""Partition a project into coupling-aware PyCG shards.
|
|
315
|
+
|
|
316
|
+
Args:
|
|
317
|
+
symbol_table: The level-1 symbol table (``file_path -> PyModule``).
|
|
318
|
+
jedi_edges: Jedi-provenance call edges from the level-1 call graph.
|
|
319
|
+
budget: Maximum number of files per shard.
|
|
320
|
+
seed: Determinism seed for Louvain.
|
|
321
|
+
merge_small: Agglomeratively merge under-budget shards to cut shard
|
|
322
|
+
count and recover edges.
|
|
323
|
+
|
|
324
|
+
Returns:
|
|
325
|
+
A :class:`ShardPlan`. Shards never silently drop files: every project
|
|
326
|
+
module appears in exactly one shard (an atomic SCC larger than *budget*
|
|
327
|
+
yields a single oversized shard, flagged in ``metrics``).
|
|
328
|
+
"""
|
|
329
|
+
g = build_module_graph(symbol_table, jedi_edges)
|
|
330
|
+
total_weight = float(sum(w for _, _, w in g.edges(data="weight", default=1)))
|
|
331
|
+
|
|
332
|
+
# SCC condensation: each strongly-connected component is one atomic unit.
|
|
333
|
+
condensation = nx.condensation(g) # DAG; node attr 'members' = set of modules
|
|
334
|
+
mapping: Dict[str, int] = condensation.graph["mapping"] # module -> scc id
|
|
335
|
+
unit_members: Dict[int, Set[str]] = {
|
|
336
|
+
scc: set(condensation.nodes[scc]["members"]) for scc in condensation.nodes
|
|
337
|
+
}
|
|
338
|
+
unit_size: Dict[int, int] = {scc: len(m) for scc, m in unit_members.items()}
|
|
339
|
+
|
|
340
|
+
# Weighted undirected graph over SCC units.
|
|
341
|
+
hu = nx.Graph()
|
|
342
|
+
hu.add_nodes_from(condensation.nodes())
|
|
343
|
+
for u, v, w in g.edges(data="weight", default=1):
|
|
344
|
+
su, sv = mapping[u], mapping[v]
|
|
345
|
+
if su == sv:
|
|
346
|
+
continue
|
|
347
|
+
if hu.has_edge(su, sv):
|
|
348
|
+
hu[su][sv]["weight"] += w
|
|
349
|
+
else:
|
|
350
|
+
hu.add_edge(su, sv, weight=w)
|
|
351
|
+
|
|
352
|
+
# Community detection over units, then budget enforcement.
|
|
353
|
+
communities = _communities(hu, resolution=1.0, seed=seed)
|
|
354
|
+
unit_shards: List[Set[str]] = [] # sets of SCC ids
|
|
355
|
+
for community in communities:
|
|
356
|
+
unit_shards.extend(
|
|
357
|
+
_split_oversized(hu, community, unit_size, budget, seed)
|
|
358
|
+
)
|
|
359
|
+
|
|
360
|
+
if merge_small:
|
|
361
|
+
unit_shards = _merge_small(unit_shards, hu, unit_size, budget)
|
|
362
|
+
|
|
363
|
+
# Expand SCC units back to file paths (graph nodes are files).
|
|
364
|
+
file_shards: List[List[str]] = []
|
|
365
|
+
for units in unit_shards:
|
|
366
|
+
files: Set[str] = set()
|
|
367
|
+
for scc in units:
|
|
368
|
+
files |= unit_members[scc]
|
|
369
|
+
if files:
|
|
370
|
+
file_shards.append(sorted(files))
|
|
371
|
+
|
|
372
|
+
# Parallel view in module names (file stems) for human-readable reporting.
|
|
373
|
+
module_shards = [
|
|
374
|
+
[g.nodes[f].get("module_name", f) for f in files] for files in file_shards
|
|
375
|
+
]
|
|
376
|
+
|
|
377
|
+
# Metrics: how much Jedi edge weight does this partition sever?
|
|
378
|
+
shard_of: Dict[str, int] = {}
|
|
379
|
+
for idx, files in enumerate(file_shards):
|
|
380
|
+
for f in files:
|
|
381
|
+
shard_of[f] = idx
|
|
382
|
+
cut_weight = 0.0
|
|
383
|
+
for u, v, w in g.edges(data="weight", default=1):
|
|
384
|
+
if shard_of.get(u) != shard_of.get(v):
|
|
385
|
+
cut_weight += w
|
|
386
|
+
|
|
387
|
+
sizes = [len(s) for s in module_shards] or [0]
|
|
388
|
+
metrics = {
|
|
389
|
+
"modules": float(g.number_of_nodes()),
|
|
390
|
+
"module_edges": float(g.number_of_edges()),
|
|
391
|
+
"total_edge_weight": total_weight,
|
|
392
|
+
"cut_weight": cut_weight,
|
|
393
|
+
"cut_ratio": (cut_weight / total_weight) if total_weight else 0.0,
|
|
394
|
+
"num_shards": float(len(module_shards)),
|
|
395
|
+
"max_shard_files": float(max(sizes)),
|
|
396
|
+
"oversized_shards": float(sum(1 for s in sizes if s > budget)),
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
plan = ShardPlan(shards=file_shards, module_shards=module_shards, metrics=metrics)
|
|
400
|
+
logger.info("Shard planner: %s", plan)
|
|
401
|
+
return plan
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Static resolution of import spellings against the analyzed module set.
|
|
2
|
+
|
|
3
|
+
Pure post-pass over a built ``PyApplication``: no filesystem access, no
|
|
4
|
+
sys.path semantics — a spelling resolves iff it names a module that was
|
|
5
|
+
itself analyzed (issue #82). External/library imports stay unresolved by
|
|
6
|
+
design and keep their :PyPackage projection.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Dict, Optional, Union
|
|
13
|
+
|
|
14
|
+
from codeanalyzer.schema.py_schema import PyApplication
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _dotted_candidates(app: PyApplication, project_dir: Union[Path, str]) -> Dict[str, str]:
|
|
18
|
+
"""dotted module path -> file_key, for every analyzed module.
|
|
19
|
+
|
|
20
|
+
``pkg/util.py`` -> ``pkg.util``; ``pkg/__init__.py`` -> ``pkg``.
|
|
21
|
+
file_keys share project_dir's form (both come from the same CLI arg),
|
|
22
|
+
so os.path.relpath keeps mixed absolute/relative setups consistent.
|
|
23
|
+
"""
|
|
24
|
+
mapping: Dict[str, str] = {}
|
|
25
|
+
for file_key in app.symbol_table:
|
|
26
|
+
rel = os.path.relpath(file_key, str(project_dir))
|
|
27
|
+
if rel.startswith(".."):
|
|
28
|
+
continue
|
|
29
|
+
parts = Path(rel).with_suffix("").parts
|
|
30
|
+
if parts and parts[-1] == "__init__":
|
|
31
|
+
parts = parts[:-1]
|
|
32
|
+
if parts:
|
|
33
|
+
mapping[".".join(parts)] = file_key
|
|
34
|
+
return mapping
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _resolve_one(
|
|
38
|
+
spelling: str, original_name: str, importer_rel_parts: tuple, candidates: Dict[str, str]
|
|
39
|
+
) -> Optional[str]:
|
|
40
|
+
if spelling.startswith("."):
|
|
41
|
+
level = len(spelling) - len(spelling.lstrip("."))
|
|
42
|
+
suffix = spelling.lstrip(".")
|
|
43
|
+
# level 1 = the importer's own package; each extra dot walks one up.
|
|
44
|
+
package_parts = importer_rel_parts[:-1] # drop the filename
|
|
45
|
+
if level - 1 > len(package_parts):
|
|
46
|
+
return None
|
|
47
|
+
base = package_parts[: len(package_parts) - (level - 1)]
|
|
48
|
+
stems = list(base) + (suffix.split(".") if suffix else [])
|
|
49
|
+
else:
|
|
50
|
+
stems = spelling.split(".")
|
|
51
|
+
dotted = ".".join(stems)
|
|
52
|
+
with_name = f"{dotted}.{original_name}" if dotted else original_name
|
|
53
|
+
return candidates.get(with_name) or candidates.get(dotted)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def resolve_imports(app: PyApplication, project_dir: Union[Path, str]) -> None:
|
|
57
|
+
"""Stamp ``resolved_module`` on every import of every module, in place."""
|
|
58
|
+
candidates = _dotted_candidates(app, project_dir)
|
|
59
|
+
for file_key, module in app.symbol_table.items():
|
|
60
|
+
rel = os.path.relpath(file_key, str(project_dir))
|
|
61
|
+
importer_parts = Path(rel).parts
|
|
62
|
+
for im in module.imports or []:
|
|
63
|
+
if not im.module:
|
|
64
|
+
im.resolved_module = None
|
|
65
|
+
continue
|
|
66
|
+
original = im.alias or im.name
|
|
67
|
+
im.resolved_module = _resolve_one(im.module, original, importer_parts, candidates)
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import ast
|
|
2
2
|
import hashlib
|
|
3
|
+
import os
|
|
3
4
|
import tokenize
|
|
4
5
|
from ast import AST, ClassDef
|
|
5
6
|
from io import StringIO
|
|
@@ -13,6 +14,7 @@ from jedi.api.project import Project
|
|
|
13
14
|
from codeanalyzer.schema.py_schema import (
|
|
14
15
|
PyCallable,
|
|
15
16
|
PyCallableParameter,
|
|
17
|
+
PyCallArgument,
|
|
16
18
|
PyCallsite,
|
|
17
19
|
PyClass,
|
|
18
20
|
PyClassAttribute,
|
|
@@ -28,7 +30,11 @@ class SymbolTableBuilder:
|
|
|
28
30
|
"""A class for building a symbol table for a Python project."""
|
|
29
31
|
|
|
30
32
|
def __init__(self, project_dir: Union[Path, str], virtualenv: Union[Path, str, None]) -> None:
|
|
31
|
-
|
|
33
|
+
# Jedi reports absolute Script paths, so a relative project_dir would
|
|
34
|
+
# crash every relative_to() fallback below. abspath (not resolve())
|
|
35
|
+
# keeps symlinks intact, matching Jedi's own absolute()-style
|
|
36
|
+
# normalization under symlinked roots like macOS /tmp.
|
|
37
|
+
self.project_dir = Path(os.path.abspath(project_dir))
|
|
32
38
|
if virtualenv is None:
|
|
33
39
|
# If no virtual environment is provided, create a jedi project without an environment.
|
|
34
40
|
self.jedi_project: Project = jedi.Project(path=self.project_dir)
|
|
@@ -39,6 +45,16 @@ class SymbolTableBuilder:
|
|
|
39
45
|
environment_path=Path(virtualenv) / "bin" / "python",
|
|
40
46
|
)
|
|
41
47
|
|
|
48
|
+
def _fallback_signature(self, script_path: Union[Path, str], name: str) -> str:
|
|
49
|
+
"""Path-derived qualified name used when Jedi can't name a definition.
|
|
50
|
+
|
|
51
|
+
Strips only the terminal ``.py`` suffix — a bare ``str.replace``
|
|
52
|
+
would also eat interior ``.py`` substrings and corrupt the module
|
|
53
|
+
prefix (``odoo/tools/pycompat.py`` → ``odoo.toolscompat``).
|
|
54
|
+
"""
|
|
55
|
+
relative = Path(script_path).relative_to(self.project_dir)
|
|
56
|
+
return ".".join(relative.with_suffix("").parts) + f".{name}"
|
|
57
|
+
|
|
42
58
|
@staticmethod
|
|
43
59
|
def _infer_type(script: Script, line: int, column: int) -> str:
|
|
44
60
|
"""Tries to infer the type at a given position using Jedi."""
|
|
@@ -97,6 +113,44 @@ class SymbolTableBuilder:
|
|
|
97
113
|
except Exception:
|
|
98
114
|
return None, False
|
|
99
115
|
|
|
116
|
+
@staticmethod
|
|
117
|
+
def _callee_anchor(node: ast.Call) -> Tuple[int, int]:
|
|
118
|
+
"""Position of the callee *name* for Jedi inference.
|
|
119
|
+
|
|
120
|
+
An ``ast.Call``'s own ``lineno``/``col_offset`` is the first
|
|
121
|
+
character of the whole call expression — for an attribute call
|
|
122
|
+
``receiver.method(...)`` that is the receiver token, and Jedi
|
|
123
|
+
would infer the receiver's type instead of the invoked method
|
|
124
|
+
(issue #80). Anchor attribute calls inside the attribute name —
|
|
125
|
+
its last character, so one-character names stay in range; other
|
|
126
|
+
callee shapes keep the call-expression start.
|
|
127
|
+
"""
|
|
128
|
+
func_expr = node.func
|
|
129
|
+
if isinstance(func_expr, ast.Attribute):
|
|
130
|
+
return func_expr.end_lineno, func_expr.end_col_offset - 1
|
|
131
|
+
return node.lineno, node.col_offset
|
|
132
|
+
|
|
133
|
+
@staticmethod
|
|
134
|
+
def _infer_call_return_type(script: Script, line: int, column: int) -> Optional[str]:
|
|
135
|
+
"""Inferred type of the call's *result*, not of the callee itself.
|
|
136
|
+
|
|
137
|
+
``Script.infer`` at the callee name yields the function/class
|
|
138
|
+
being called; executing that definition yields what the call
|
|
139
|
+
evaluates to — a function's inferred return type, or the
|
|
140
|
+
instance for a constructor call. Returns ``None`` when Jedi
|
|
141
|
+
can't tell, so an unknown stays absent instead of masquerading
|
|
142
|
+
as the callee's own name.
|
|
143
|
+
"""
|
|
144
|
+
try:
|
|
145
|
+
definitions = script.infer(line=line, column=column)
|
|
146
|
+
if definitions:
|
|
147
|
+
results = definitions[0].execute()
|
|
148
|
+
if results:
|
|
149
|
+
return results[0].name
|
|
150
|
+
except Exception:
|
|
151
|
+
pass
|
|
152
|
+
return None
|
|
153
|
+
|
|
100
154
|
def build_pymodule_from_file(self, py_file: Path) -> PyModule:
|
|
101
155
|
"""Builds a PyModule from a Python file.
|
|
102
156
|
|
|
@@ -203,10 +257,10 @@ class SymbolTableBuilder:
|
|
|
203
257
|
definitions = script.goto(line=start_line, column=child.col_offset)
|
|
204
258
|
signature = next(
|
|
205
259
|
(d.full_name for d in definitions if d.type == "class"),
|
|
206
|
-
|
|
260
|
+
self._fallback_signature(script.path, class_name),
|
|
207
261
|
)
|
|
208
262
|
except Exception:
|
|
209
|
-
signature =
|
|
263
|
+
signature = self._fallback_signature(script.path, class_name)
|
|
210
264
|
py_class = (
|
|
211
265
|
PyClass.builder()
|
|
212
266
|
.name(class_name)
|
|
@@ -258,8 +312,7 @@ class SymbolTableBuilder:
|
|
|
258
312
|
|
|
259
313
|
# If Jedi didn't provide a signature, build one relative to project_dir
|
|
260
314
|
if not signature:
|
|
261
|
-
|
|
262
|
-
signature = f"{str(relative_path).replace('/', '.').replace('.py', '')}.{method_name}"
|
|
315
|
+
signature = self._fallback_signature(script.path, method_name)
|
|
263
316
|
py_callable = (
|
|
264
317
|
PyCallable.builder()
|
|
265
318
|
.name(method_name) # Use the actual method name, not the full signature
|
|
@@ -380,6 +433,7 @@ class SymbolTableBuilder:
|
|
|
380
433
|
script, target.lineno, target.col_offset
|
|
381
434
|
)
|
|
382
435
|
)
|
|
436
|
+
.initializer(ast.unparse(stmt.value) if stmt.value else None)
|
|
383
437
|
.start_line(getattr(target, "lineno", -1))
|
|
384
438
|
.end_line(getattr(stmt, "end_lineno", stmt.lineno))
|
|
385
439
|
.build()
|
|
@@ -398,6 +452,7 @@ class SymbolTableBuilder:
|
|
|
398
452
|
script, target.lineno, target.col_offset
|
|
399
453
|
)
|
|
400
454
|
)
|
|
455
|
+
.initializer(ast.unparse(stmt.value) if stmt.value else None)
|
|
401
456
|
.start_line(getattr(target, "lineno", -1))
|
|
402
457
|
.end_line(getattr(stmt, "end_lineno", stmt.lineno))
|
|
403
458
|
.build()
|
|
@@ -588,10 +643,11 @@ class SymbolTableBuilder:
|
|
|
588
643
|
func_expr = node.func
|
|
589
644
|
|
|
590
645
|
method_name = "<unknown>"
|
|
646
|
+
anchor_line, anchor_col = self._callee_anchor(node)
|
|
591
647
|
callee_signature, is_constructor = self._infer_callee(
|
|
592
|
-
script,
|
|
648
|
+
script, anchor_line, anchor_col
|
|
593
649
|
)
|
|
594
|
-
return_type = self.
|
|
650
|
+
return_type = self._infer_call_return_type(script, anchor_line, anchor_col)
|
|
595
651
|
|
|
596
652
|
receiver_expr = None
|
|
597
653
|
receiver_type = None
|
|
@@ -604,18 +660,26 @@ class SymbolTableBuilder:
|
|
|
604
660
|
elif isinstance(func_expr, ast.Name):
|
|
605
661
|
method_name = func_expr.id
|
|
606
662
|
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
663
|
+
arguments = [
|
|
664
|
+
PyCallArgument(
|
|
665
|
+
ast_kind=type(arg).__name__,
|
|
666
|
+
inferred_type=self._infer_type(script, arg.lineno, arg.col_offset),
|
|
667
|
+
)
|
|
610
668
|
for arg in node.args
|
|
611
669
|
]
|
|
612
670
|
|
|
671
|
+
# Legacy field, derived from the structured arguments above rather
|
|
672
|
+
# than re-inferring: byte-identical to the old
|
|
673
|
+
# `self._infer_type(...) or type(arg).__name__` per argument.
|
|
674
|
+
argument_types = [a.inferred_type or a.ast_kind for a in arguments]
|
|
675
|
+
|
|
613
676
|
call_sites.append(
|
|
614
677
|
PyCallsite.builder()
|
|
615
678
|
.method_name(method_name)
|
|
616
679
|
.receiver_expr(receiver_expr)
|
|
617
680
|
.receiver_type(receiver_type)
|
|
618
681
|
.argument_types(argument_types)
|
|
682
|
+
.arguments(arguments)
|
|
619
683
|
.return_type(return_type)
|
|
620
684
|
.callee_signature(callee_signature)
|
|
621
685
|
.is_constructor_call(is_constructor)
|
codeanalyzer/utils/logging.py
CHANGED
|
@@ -3,8 +3,11 @@ import logging
|
|
|
3
3
|
from rich.console import Console
|
|
4
4
|
from rich.logging import RichHandler
|
|
5
5
|
|
|
6
|
-
#
|
|
7
|
-
console
|
|
6
|
+
# Logs go to stderr so stdout stays clean for piped output (e.g. --emit json | jq).
|
|
7
|
+
# The same console instance is shared with ProgressBar so Rich can coordinate
|
|
8
|
+
# live-display updates with log messages without the two consoles stomping on each
|
|
9
|
+
# other (which causes progress bars to appear twice when a warning interrupts them).
|
|
10
|
+
console = Console(stderr=True)
|
|
8
11
|
handler = RichHandler(console=console, show_time=True, show_level=True, show_path=False)
|
|
9
12
|
|
|
10
13
|
logger = logging.getLogger("codeanalyzer")
|