codeanalyzer-python 0.1.13__py3-none-any.whl → 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codeanalyzer/__main__.py +99 -11
- codeanalyzer/core.py +154 -19
- codeanalyzer/neo4j/__init__.py +46 -0
- codeanalyzer/neo4j/bolt.py +223 -0
- codeanalyzer/neo4j/catalog.py +245 -0
- codeanalyzer/neo4j/cypher.py +138 -0
- codeanalyzer/neo4j/emit.py +74 -0
- codeanalyzer/neo4j/project.py +322 -0
- codeanalyzer/neo4j/rows.py +176 -0
- codeanalyzer/neo4j/schema.py +39 -0
- codeanalyzer/options/__init__.py +2 -2
- codeanalyzer/options/options.py +20 -1
- codeanalyzer/schema/py_schema.py +20 -0
- codeanalyzer/semantic_analysis/call_graph.py +266 -0
- codeanalyzer/semantic_analysis/codeql/codeql_analysis.py +318 -69
- codeanalyzer/semantic_analysis/codeql/codeql_loader.py +32 -4
- codeanalyzer/semantic_analysis/codeql/codeql_query_runner.py +51 -31
- codeanalyzer/syntactic_analysis/symbol_table_builder.py +87 -4
- codeanalyzer_python-0.2.0.dist-info/METADATA +393 -0
- codeanalyzer_python-0.2.0.dist-info/RECORD +39 -0
- {codeanalyzer_python-0.1.13.dist-info → codeanalyzer_python-0.2.0.dist-info}/WHEEL +1 -1
- codeanalyzer_python-0.2.0.dist-info/entry_points.txt +3 -0
- codeanalyzer/semantic_analysis/wala/__init__.py +0 -15
- codeanalyzer_python-0.1.13.dist-info/METADATA +0 -414
- codeanalyzer_python-0.1.13.dist-info/RECORD +0 -31
- codeanalyzer_python-0.1.13.dist-info/entry_points.txt +0 -2
- {codeanalyzer_python-0.1.13.dist-info → codeanalyzer_python-0.2.0.dist-info}/licenses/LICENSE +0 -0
- {codeanalyzer_python-0.1.13.dist-info → codeanalyzer_python-0.2.0.dist-info}/licenses/NOTICE +0 -0
|
@@ -20,13 +20,77 @@ This module provides functionality to create and manage CodeQL databases
|
|
|
20
20
|
for Python projects and execute queries against them.
|
|
21
21
|
"""
|
|
22
22
|
|
|
23
|
+
from collections import Counter
|
|
23
24
|
from pathlib import Path
|
|
24
|
-
from typing import Union
|
|
25
|
+
from typing import Any, Dict, Iterator, List, Tuple, Union
|
|
25
26
|
|
|
26
|
-
from networkx import DiGraph
|
|
27
27
|
from pandas import DataFrame
|
|
28
28
|
|
|
29
|
+
from codeanalyzer.schema.py_schema import PyCallEdge, PyModule
|
|
30
|
+
from codeanalyzer.semantic_analysis.call_graph import iter_callables_in_symbol_table
|
|
29
31
|
from codeanalyzer.semantic_analysis.codeql.codeql_query_runner import CodeQLQueryRunner
|
|
32
|
+
from codeanalyzer.utils import logger
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class _CallableResolver:
|
|
36
|
+
"""Maps a CodeQL endpoint ``(file, start_line, name, arity)`` to a Jedi
|
|
37
|
+
``PyCallable``.
|
|
38
|
+
|
|
39
|
+
Resolution ladder:
|
|
40
|
+
1. exact ``(abs_path, start_line)`` — the precise join;
|
|
41
|
+
2. on miss, candidates sharing ``(abs_path, short_name)``: a single
|
|
42
|
+
candidate is taken directly; otherwise prefer those whose
|
|
43
|
+
parameter count equals the CodeQL positional arity, then the
|
|
44
|
+
nearest ``start_line``;
|
|
45
|
+
3. no name match -> ``None`` (caller row skipped / callee becomes
|
|
46
|
+
a ghost node).
|
|
47
|
+
|
|
48
|
+
Step 2 recovers edges the ``(file, line)`` join silently drops when
|
|
49
|
+
CodeQL and Jedi disagree on a definition's start line (e.g. decorator
|
|
50
|
+
handling). Jedi's ``parameters`` counts every declared slot (incl.
|
|
51
|
+
``*args``/``**kwargs``/keyword-only) whereas CodeQL's arity is
|
|
52
|
+
positional only, so the arity filter is exact for plain signatures
|
|
53
|
+
and otherwise yields to the nearest-line tiebreak.
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
def __init__(self) -> None:
|
|
57
|
+
self._by_loc: Dict[Tuple[str, int], Any] = {}
|
|
58
|
+
self._by_name: Dict[Tuple[str, str], List[Any]] = {}
|
|
59
|
+
|
|
60
|
+
@staticmethod
|
|
61
|
+
def _abs(path: str) -> str:
|
|
62
|
+
try:
|
|
63
|
+
return str(Path(path).resolve())
|
|
64
|
+
except (OSError, RuntimeError):
|
|
65
|
+
return path
|
|
66
|
+
|
|
67
|
+
@classmethod
|
|
68
|
+
def from_symbol_table(
|
|
69
|
+
cls, symbol_table: Dict[str, PyModule]
|
|
70
|
+
) -> "_CallableResolver":
|
|
71
|
+
resolver = cls()
|
|
72
|
+
for c in iter_callables_in_symbol_table(symbol_table):
|
|
73
|
+
abs_path = cls._abs(c.path)
|
|
74
|
+
resolver._by_loc[(abs_path, c.start_line)] = c
|
|
75
|
+
resolver._by_name.setdefault((abs_path, c.name), []).append(c)
|
|
76
|
+
return resolver
|
|
77
|
+
|
|
78
|
+
def resolve(
|
|
79
|
+
self, file: str, start_line: int, name: str, arity: int
|
|
80
|
+
) -> Any:
|
|
81
|
+
exact = self._by_loc.get((file, start_line))
|
|
82
|
+
if exact is not None:
|
|
83
|
+
return exact
|
|
84
|
+
if not name:
|
|
85
|
+
return None
|
|
86
|
+
candidates = self._by_name.get((file, name))
|
|
87
|
+
if not candidates:
|
|
88
|
+
return None
|
|
89
|
+
if len(candidates) == 1:
|
|
90
|
+
return candidates[0]
|
|
91
|
+
arity_matched = [c for c in candidates if len(c.parameters) == arity]
|
|
92
|
+
pool = arity_matched or candidates
|
|
93
|
+
return min(pool, key=lambda c: abs(c.start_line - start_line))
|
|
30
94
|
|
|
31
95
|
|
|
32
96
|
class CodeQL:
|
|
@@ -40,94 +104,279 @@ class CodeQL:
|
|
|
40
104
|
temp_db (TemporaryDirectory or None): The temporary directory object if a temporary database was created.
|
|
41
105
|
"""
|
|
42
106
|
|
|
43
|
-
def __init__(
|
|
107
|
+
def __init__(
|
|
108
|
+
self,
|
|
109
|
+
project_dir: Union[str, Path],
|
|
110
|
+
db_path: Path,
|
|
111
|
+
codeql_bin: Union[str, Path, None] = None,
|
|
112
|
+
codeql_packs_dir: Union[str, Path, None] = None,
|
|
113
|
+
) -> None:
|
|
44
114
|
self.project_dir = project_dir
|
|
45
115
|
self.db_path = db_path
|
|
116
|
+
self.codeql_bin = codeql_bin
|
|
117
|
+
self.codeql_packs_dir = codeql_packs_dir
|
|
118
|
+
self._cached_df: "DataFrame | None" = None
|
|
46
119
|
|
|
47
|
-
def
|
|
48
|
-
"""
|
|
120
|
+
def _query_call_edges(self) -> DataFrame:
|
|
121
|
+
"""Runs the CodeQL query that emits one row per resolved call site.
|
|
49
122
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
123
|
+
The query is written against CodeQL's Python library (``import python``).
|
|
124
|
+
It returns physical location handles for both endpoints so the
|
|
125
|
+
downstream post-processor can join into Jedi's existing
|
|
126
|
+
``PyCallable.signature`` space via ``(file_path, start_line)`` —
|
|
127
|
+
no signature normalization required.
|
|
54
128
|
|
|
55
|
-
|
|
56
|
-
|
|
129
|
+
Filters:
|
|
130
|
+
* Caller must be a ``Function`` (skip module-level / class-body
|
|
131
|
+
calls — they have no ``PyCallable`` to anchor to).
|
|
132
|
+
* Callee may resolve to anything (in-source or library stub);
|
|
133
|
+
non-application callees become **ghost** nodes downstream so
|
|
134
|
+
RPC / third-party / framework edges are preserved.
|
|
57
135
|
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
136
|
+
Returns:
|
|
137
|
+
DataFrame: one row per resolved (caller, callee, call-site)
|
|
138
|
+
triple. Duplicate ``(caller_file, caller_start_line,
|
|
139
|
+
callee_file, callee_start_line)`` tuples represent multiple
|
|
140
|
+
call sites in the same caller targeting the same callee and
|
|
141
|
+
are coalesced into a single ``PyCallEdge`` (weight = count)
|
|
142
|
+
by the post-processor.
|
|
143
|
+
"""
|
|
144
|
+
query = [
|
|
145
|
+
"/**",
|
|
146
|
+
" * @name Python call-graph edges",
|
|
147
|
+
" * @description One row per resolved call site: caller, callee,",
|
|
148
|
+
" * and the call-expression location.",
|
|
149
|
+
" * @kind table",
|
|
150
|
+
" * @id py/codeanalyzer/call-graph-edges",
|
|
151
|
+
" */",
|
|
152
|
+
"import python",
|
|
153
|
+
# ``FunctionValue`` / ``ClassValue`` / the ``pointsTo`` predicate
|
|
154
|
+
# live in ObjectAPI, which ``import python`` only brings in as a
|
|
155
|
+
# private import — they aren't re-exported. Pull them in
|
|
156
|
+
# explicitly.
|
|
157
|
+
"import semmle.python.objects.ObjectAPI",
|
|
158
|
+
"",
|
|
159
|
+
# ``Value.getACall()`` is the modern call-resolution API in
|
|
160
|
+
# codeql/python-all 7.x — it returns the ``CallNode`` (CFG)
|
|
161
|
+
# whose target was resolved to that ``Value``. Cleaner than
|
|
162
|
+
# poking at ``pointsTo`` directly.
|
|
163
|
+
# ``callee`` is bound to the FunctionValue's scope so the
|
|
164
|
+
# endpoint emits the same Function-level facts (name, arity,
|
|
165
|
+
# location) the post-processor needs for the name+arity
|
|
166
|
+
# fallback when the (file, start_line) join misses.
|
|
167
|
+
"from CallNode call, Function caller, FunctionValue calleeVal, Function callee",
|
|
61
168
|
"where",
|
|
62
|
-
"
|
|
63
|
-
"callee.
|
|
64
|
-
"
|
|
169
|
+
" call.getScope() = caller and",
|
|
170
|
+
" callee = calleeVal.getScope() and",
|
|
171
|
+
" (",
|
|
172
|
+
# Direct function / bound-method call: foo() or obj.foo()
|
|
173
|
+
" call = calleeVal.getACall()",
|
|
174
|
+
" or",
|
|
175
|
+
# Constructor call: A(...) resolves to a ClassValue; the actual
|
|
176
|
+
# callee is the class's __init__ (via MRO lookup so subclasses
|
|
177
|
+
# without an explicit __init__ still resolve to the inherited one).
|
|
178
|
+
" exists(ClassValue clsVal |",
|
|
179
|
+
" call = clsVal.getACall() and",
|
|
180
|
+
' clsVal.lookup("__init__") = calleeVal',
|
|
181
|
+
" )",
|
|
182
|
+
" )",
|
|
65
183
|
"select",
|
|
184
|
+
# --- Caller endpoint --- (joins to PyCallable: exact by
|
|
185
|
+
# (file, start_line), else by (file, name) + arity)
|
|
186
|
+
" caller.getLocation().getFile().getAbsolutePath(),",
|
|
187
|
+
" caller.getLocation().getStartLine(),",
|
|
188
|
+
" caller.getQualifiedName(),",
|
|
189
|
+
" caller.getName(),",
|
|
190
|
+
" count(caller.getArg(_)),",
|
|
191
|
+
# --- Callee endpoint --- (file/line may live in a library stub;
|
|
192
|
+
# post-processor classifies as in-source or ghost)
|
|
193
|
+
" callee.getLocation().getFile().getAbsolutePath(),",
|
|
194
|
+
" callee.getLocation().getStartLine(),",
|
|
195
|
+
" calleeVal.getQualifiedName(),",
|
|
196
|
+
" callee.getName(),",
|
|
197
|
+
" count(callee.getArg(_)),",
|
|
198
|
+
# --- Call-site location --- (for PyCallsite augmentation)
|
|
199
|
+
" call.getLocation().getStartLine(),",
|
|
200
|
+
" call.getLocation().getStartColumn(),",
|
|
201
|
+
" call.getLocation().getEndLine(),",
|
|
202
|
+
" call.getLocation().getEndColumn()",
|
|
203
|
+
# ``is_constructor`` is derived in the post-processor by
|
|
204
|
+
# checking whether ``callee_qname`` ends in ``.__init__``;
|
|
205
|
+
# avoids QL's restrictive ``if-then-else`` typing here.
|
|
66
206
|
]
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
query += [
|
|
70
|
-
"caller.getFile().getAbsolutePath(),",
|
|
71
|
-
'"[" + caller.getBody().getLocation().getStartLine() + ", " + caller.getBody().getLocation().getEndLine() + "]", //Caller body slice indices',
|
|
72
|
-
"caller.getQualifiedName(), // Caller's fullsignature",
|
|
73
|
-
"caller.getAModifier(), // caller's method modifier",
|
|
74
|
-
"caller.paramsString(), // caller's method parameter types",
|
|
75
|
-
"caller.getReturnType().toString(), // Caller's return type",
|
|
76
|
-
"caller.getDeclaringType().getQualifiedName(), // Caller's class",
|
|
77
|
-
"caller.getDeclaringType().getAModifier(), // Caller's class modifier",
|
|
78
|
-
]
|
|
79
|
-
|
|
80
|
-
# Callee metadata
|
|
81
|
-
query += [
|
|
82
|
-
"callee.getFile().getAbsolutePath(),",
|
|
83
|
-
'"[" + callee.getBody().getLocation().getStartLine() + ", " + callee.getBody().getLocation().getEndLine() + "]", //Caller body slice indices',
|
|
84
|
-
"callee.getQualifiedName(), // Caller's fullsignature",
|
|
85
|
-
"callee.getAModifier(), // callee's method modifier",
|
|
86
|
-
"callee.paramsString(), // callee's method parameter types",
|
|
87
|
-
"callee.getReturnType().toString(), // Caller's return type",
|
|
88
|
-
"callee.getDeclaringType().getQualifiedName(), // Caller's class",
|
|
89
|
-
"callee.getDeclaringType().getAModifier() // Caller's class modifier",
|
|
90
|
-
]
|
|
207
|
+
if self._cached_df is not None:
|
|
208
|
+
return self._cached_df
|
|
91
209
|
|
|
92
210
|
query_string = "\n".join(query)
|
|
93
211
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
212
|
+
with CodeQLQueryRunner(
|
|
213
|
+
self.db_path,
|
|
214
|
+
codeql_bin=self.codeql_bin,
|
|
215
|
+
codeql_packs_dir=self.codeql_packs_dir,
|
|
216
|
+
) as runner:
|
|
217
|
+
df: DataFrame = runner.execute(
|
|
97
218
|
query_string,
|
|
98
219
|
column_names=[
|
|
99
|
-
# Caller Columns
|
|
100
220
|
"caller_file",
|
|
101
|
-
"
|
|
102
|
-
"
|
|
103
|
-
"
|
|
104
|
-
"
|
|
105
|
-
"caller_return_type",
|
|
106
|
-
"caller_class_signature",
|
|
107
|
-
"caller_class_modifier",
|
|
108
|
-
# Callee Columns
|
|
221
|
+
"caller_start_line",
|
|
222
|
+
"caller_qname",
|
|
223
|
+
"caller_name",
|
|
224
|
+
"caller_arity",
|
|
109
225
|
"callee_file",
|
|
110
|
-
"
|
|
111
|
-
"
|
|
112
|
-
"
|
|
113
|
-
"
|
|
114
|
-
"
|
|
115
|
-
"
|
|
116
|
-
"
|
|
226
|
+
"callee_start_line",
|
|
227
|
+
"callee_qname",
|
|
228
|
+
"callee_name",
|
|
229
|
+
"callee_arity",
|
|
230
|
+
"call_start_line",
|
|
231
|
+
"call_start_column",
|
|
232
|
+
"call_end_line",
|
|
233
|
+
"call_end_column",
|
|
117
234
|
],
|
|
118
235
|
)
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
callgraph: DiGraph = self.__process_call_edges_to_callgraph(query_result)
|
|
122
|
-
return callgraph
|
|
236
|
+
self._cached_df = df
|
|
237
|
+
return df
|
|
123
238
|
|
|
124
239
|
@staticmethod
|
|
125
|
-
def
|
|
126
|
-
|
|
240
|
+
def _build_callable_resolver(
|
|
241
|
+
symbol_table: Dict[str, PyModule],
|
|
242
|
+
) -> _CallableResolver:
|
|
243
|
+
"""Build the endpoint -> ``PyCallable`` resolver from Jedi.
|
|
244
|
+
|
|
245
|
+
Paths are resolved so they match CodeQL's ``getAbsolutePath()``
|
|
246
|
+
regardless of symlinks or the current working directory.
|
|
247
|
+
"""
|
|
248
|
+
return _CallableResolver.from_symbol_table(symbol_table)
|
|
127
249
|
|
|
128
|
-
|
|
129
|
-
|
|
250
|
+
def _iter_resolved_rows(
|
|
251
|
+
self, symbol_table: Dict[str, PyModule]
|
|
252
|
+
) -> "Iterator[Tuple[str, str, Any]]":
|
|
253
|
+
"""Yield ``(source_sig, target_sig, row)`` for every CodeQL row.
|
|
254
|
+
|
|
255
|
+
Rows whose caller can't be matched to a ``PyCallable`` in the
|
|
256
|
+
symbol table are skipped. Callee misses fall back to
|
|
257
|
+
``row.callee_qname`` (ghost). Used by both edge construction and
|
|
258
|
+
call-site augmentation so a single CodeQL query feeds both.
|
|
259
|
+
"""
|
|
260
|
+
df = self._query_call_edges()
|
|
261
|
+
if df.empty:
|
|
262
|
+
return
|
|
263
|
+
resolver = self._build_callable_resolver(symbol_table)
|
|
264
|
+
|
|
265
|
+
skipped_unknown_caller = 0
|
|
266
|
+
ghost_callees = 0
|
|
267
|
+
for row in df.itertuples(index=False):
|
|
268
|
+
caller = resolver.resolve(
|
|
269
|
+
row.caller_file,
|
|
270
|
+
int(row.caller_start_line),
|
|
271
|
+
row.caller_name,
|
|
272
|
+
int(row.caller_arity),
|
|
273
|
+
)
|
|
274
|
+
if caller is None:
|
|
275
|
+
skipped_unknown_caller += 1
|
|
276
|
+
continue
|
|
277
|
+
|
|
278
|
+
callee = resolver.resolve(
|
|
279
|
+
row.callee_file,
|
|
280
|
+
int(row.callee_start_line),
|
|
281
|
+
row.callee_name,
|
|
282
|
+
int(row.callee_arity),
|
|
283
|
+
)
|
|
284
|
+
if callee is not None:
|
|
285
|
+
target_sig = callee.signature
|
|
286
|
+
else:
|
|
287
|
+
target_sig = row.callee_qname
|
|
288
|
+
ghost_callees += 1
|
|
289
|
+
|
|
290
|
+
yield caller.signature, target_sig, row
|
|
291
|
+
|
|
292
|
+
if skipped_unknown_caller:
|
|
293
|
+
logger.debug(
|
|
294
|
+
f"CodeQL: skipped {skipped_unknown_caller} rows whose caller "
|
|
295
|
+
f"was not in Jedi's symbol table."
|
|
296
|
+
)
|
|
297
|
+
if ghost_callees:
|
|
298
|
+
logger.debug(
|
|
299
|
+
f"CodeQL: {ghost_callees} rows resolved to ghost (external) callees."
|
|
300
|
+
)
|
|
301
|
+
|
|
302
|
+
def build_call_graph_edges(
|
|
303
|
+
self, symbol_table: Dict[str, PyModule]
|
|
304
|
+
) -> List[PyCallEdge]:
|
|
305
|
+
"""Run the CodeQL query and turn each row into a ``PyCallEdge``.
|
|
306
|
+
|
|
307
|
+
Edges are coalesced on ``(source, target)`` — ``weight`` is the
|
|
308
|
+
number of distinct call sites in the caller targeting the callee.
|
|
309
|
+
Provenance is always ``["codeql"]``; combine with Jedi-derived
|
|
310
|
+
edges via ``call_graph.merge_edges``.
|
|
311
|
+
"""
|
|
312
|
+
edge_counts: Counter = Counter()
|
|
313
|
+
for source_sig, target_sig, _row in self._iter_resolved_rows(symbol_table):
|
|
314
|
+
edge_counts[(source_sig, target_sig)] += 1
|
|
315
|
+
|
|
316
|
+
return [
|
|
317
|
+
PyCallEdge(
|
|
318
|
+
source=src,
|
|
319
|
+
target=dst,
|
|
320
|
+
weight=count,
|
|
321
|
+
provenance=["codeql"],
|
|
322
|
+
)
|
|
323
|
+
for (src, dst), count in edge_counts.items()
|
|
324
|
+
]
|
|
325
|
+
|
|
326
|
+
def augment_call_sites(self, symbol_table: Dict[str, PyModule]) -> int:
|
|
327
|
+
"""Backfill ``PyCallsite.callee_signature`` using CodeQL resolution.
|
|
328
|
+
|
|
329
|
+
Walks every CodeQL row, locates the matching ``PyCallsite`` inside
|
|
330
|
+
the caller's ``PyCallable.call_sites`` by call-expression line range
|
|
331
|
+
(``start_line``, ``end_line``), and fills in ``callee_signature``
|
|
332
|
+
**only when Jedi left it empty**. Existing Jedi-resolved signatures
|
|
333
|
+
are kept (Jedi sees lexical context CodeQL can't, e.g. closures).
|
|
334
|
+
|
|
335
|
+
Match is by line range — column matching is brittle across the two
|
|
336
|
+
tools' 0- vs 1-based conventions. Ambiguity on a single line
|
|
337
|
+
(e.g. ``a.b().c()``) resolves to the first matching site, which is
|
|
338
|
+
an acceptable approximation given how rarely Jedi misses callees
|
|
339
|
+
on chained call lines.
|
|
130
340
|
|
|
131
341
|
Returns:
|
|
132
|
-
|
|
342
|
+
Number of ``PyCallsite`` entries augmented.
|
|
133
343
|
"""
|
|
344
|
+
resolver = self._build_callable_resolver(symbol_table)
|
|
345
|
+
df = self._query_call_edges()
|
|
346
|
+
if df.empty:
|
|
347
|
+
return 0
|
|
348
|
+
|
|
349
|
+
augmented = 0
|
|
350
|
+
for row in df.itertuples(index=False):
|
|
351
|
+
caller = resolver.resolve(
|
|
352
|
+
row.caller_file,
|
|
353
|
+
int(row.caller_start_line),
|
|
354
|
+
row.caller_name,
|
|
355
|
+
int(row.caller_arity),
|
|
356
|
+
)
|
|
357
|
+
if caller is None:
|
|
358
|
+
continue
|
|
359
|
+
|
|
360
|
+
callee = resolver.resolve(
|
|
361
|
+
row.callee_file,
|
|
362
|
+
int(row.callee_start_line),
|
|
363
|
+
row.callee_name,
|
|
364
|
+
int(row.callee_arity),
|
|
365
|
+
)
|
|
366
|
+
resolved_sig = callee.signature if callee is not None else row.callee_qname
|
|
367
|
+
|
|
368
|
+
call_start = int(row.call_start_line)
|
|
369
|
+
call_end = int(row.call_end_line)
|
|
370
|
+
for site in caller.call_sites:
|
|
371
|
+
if site.start_line != call_start or site.end_line != call_end:
|
|
372
|
+
continue
|
|
373
|
+
if not site.callee_signature:
|
|
374
|
+
site.callee_signature = resolved_sig
|
|
375
|
+
augmented += 1
|
|
376
|
+
break
|
|
377
|
+
|
|
378
|
+
if augmented:
|
|
379
|
+
logger.debug(
|
|
380
|
+
f"CodeQL: augmented {augmented} PyCallsite.callee_signature entries."
|
|
381
|
+
)
|
|
382
|
+
return augmented
|
|
@@ -1,4 +1,6 @@
|
|
|
1
|
+
import os
|
|
1
2
|
import platform
|
|
3
|
+
import stat
|
|
2
4
|
import zipfile
|
|
3
5
|
from pathlib import Path
|
|
4
6
|
|
|
@@ -52,12 +54,38 @@ class CodeQLLoader:
|
|
|
52
54
|
extract_dir = temp_dir / filename.replace(".zip", "")
|
|
53
55
|
extract_dir.mkdir(exist_ok=True)
|
|
54
56
|
|
|
55
|
-
|
|
57
|
+
logger.info(f"Extracting CodeQL CLI to {extract_dir}")
|
|
58
|
+
# zipfile.extractall drops Unix permissions (the executable bit), so
|
|
59
|
+
# we extract entries manually and copy each one's stored mode onto
|
|
60
|
+
# the file system. Without this, the CodeQL launcher script can't
|
|
61
|
+
# be executed and the next subprocess.Popen raises PermissionError.
|
|
56
62
|
with zipfile.ZipFile(archive_path, "r") as zip_ref:
|
|
57
|
-
zip_ref.
|
|
63
|
+
for info in zip_ref.infolist():
|
|
64
|
+
extracted_path = zip_ref.extract(info, extract_dir)
|
|
65
|
+
stored_mode = info.external_attr >> 16
|
|
66
|
+
if stored_mode:
|
|
67
|
+
os.chmod(extracted_path, stored_mode)
|
|
58
68
|
|
|
59
|
-
|
|
60
|
-
|
|
69
|
+
# Archive is no longer needed once extracted.
|
|
70
|
+
try:
|
|
71
|
+
archive_path.unlink()
|
|
72
|
+
except OSError as exc:
|
|
73
|
+
logger.warning(f"Could not remove CodeQL archive {archive_path}: {exc}")
|
|
74
|
+
|
|
75
|
+
# rglob("codeql") returns both the launcher file *and* an internal
|
|
76
|
+
# directory of the same name (CodeQL ships its own runtime under
|
|
77
|
+
# ``codeql/codeql/``); insist on a regular file so we never bind to
|
|
78
|
+
# the directory.
|
|
79
|
+
codeql_bin = next(
|
|
80
|
+
(p for p in extract_dir.rglob("codeql") if p.is_file()),
|
|
81
|
+
None,
|
|
82
|
+
)
|
|
83
|
+
if not codeql_bin:
|
|
61
84
|
raise FileNotFoundError("CodeQL binary not found in extracted contents.")
|
|
62
85
|
|
|
86
|
+
# Belt-and-suspenders: ensure the binary is executable even if the
|
|
87
|
+
# archive entry's mode was zero (some older zip producers omit it).
|
|
88
|
+
st = codeql_bin.stat()
|
|
89
|
+
codeql_bin.chmod(st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
|
90
|
+
|
|
63
91
|
return codeql_bin.resolve()
|
|
@@ -40,9 +40,13 @@ class CodeQLQueryRunner:
|
|
|
40
40
|
|
|
41
41
|
Args:
|
|
42
42
|
database_path (str): The path to the CodeQL database.
|
|
43
|
+
codeql_bin (str | Path | None): Absolute path to the CodeQL CLI
|
|
44
|
+
binary. When ``None``, falls back to whatever ``codeql`` is on
|
|
45
|
+
``PATH``.
|
|
43
46
|
|
|
44
47
|
Attributes:
|
|
45
48
|
database_path (Path): The path to the CodeQL database.
|
|
49
|
+
codeql_bin (str): Resolved binary path or the literal ``"codeql"``.
|
|
46
50
|
temp_file_path (Path): The path to the temporary query file.
|
|
47
51
|
csv_output_file (Path): The path to the CSV output file.
|
|
48
52
|
temp_bqrs_file_path (Path): The path to the temporary bqrs file.
|
|
@@ -52,39 +56,46 @@ class CodeQLQueryRunner:
|
|
|
52
56
|
CodeQLQueryExecutionException: If there is an error executing the query.
|
|
53
57
|
"""
|
|
54
58
|
|
|
55
|
-
def __init__(self, database_path: str):
|
|
59
|
+
def __init__(self, database_path: str, codeql_bin=None, codeql_packs_dir=None):
|
|
56
60
|
self.database_path: Path = Path(database_path)
|
|
61
|
+
self.codeql_bin: str = str(codeql_bin) if codeql_bin else "codeql"
|
|
62
|
+
self.codeql_packs_dir = (
|
|
63
|
+
Path(codeql_packs_dir) if codeql_packs_dir is not None else None
|
|
64
|
+
)
|
|
57
65
|
self.temp_file_path: Path = None
|
|
58
66
|
|
|
59
67
|
def __enter__(self):
|
|
60
|
-
"""Context entry that
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
68
|
+
"""Context entry that prepares paths to execute a CodeQL query.
|
|
69
|
+
|
|
70
|
+
The ``.ql`` file is written **inside the prepared qlpack
|
|
71
|
+
directory** (``codeql_packs_dir``) so ``import python`` resolves
|
|
72
|
+
against that pack's installed dependencies — no
|
|
73
|
+
``--additional-packs`` or ``--search-path`` needed. The CSV /
|
|
74
|
+
BQRS output files live in ``tempfile`` because they're transient
|
|
75
|
+
per-query artifacts.
|
|
67
76
|
"""
|
|
68
|
-
|
|
69
|
-
# Create a temporary file to hold the query and store its path
|
|
70
|
-
temp_file = tempfile.NamedTemporaryFile("w", delete=False, suffix=".ql")
|
|
77
|
+
# CSV and BQRS files are transient per-query — fine in /tmp.
|
|
71
78
|
csv_file = tempfile.NamedTemporaryFile("w", delete=False, suffix=".csv")
|
|
72
79
|
bqrs_file = tempfile.NamedTemporaryFile("w", delete=False, suffix=".bqrs")
|
|
73
|
-
self.temp_file_path = Path(temp_file.name)
|
|
74
80
|
self.csv_output_file = Path(csv_file.name)
|
|
75
81
|
self.temp_bqrs_file_path = Path(bqrs_file.name)
|
|
76
|
-
|
|
77
|
-
# Let's close the files, we'll reopen them by path when needed.
|
|
78
|
-
temp_file.close()
|
|
79
|
-
bqrs_file.close()
|
|
80
82
|
csv_file.close()
|
|
83
|
+
bqrs_file.close()
|
|
81
84
|
|
|
82
|
-
#
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
85
|
+
# The .ql file MUST live inside the prepared qlpack so its
|
|
86
|
+
# ``import python`` resolves via that pack's lock file. Writing
|
|
87
|
+
# outside the pack means CodeQL falls back to a default
|
|
88
|
+
# search-path that doesn't include downloaded library packs.
|
|
89
|
+
if self.codeql_packs_dir is None:
|
|
90
|
+
raise RuntimeError(
|
|
91
|
+
"CodeQLQueryRunner requires codeql_packs_dir — the directory "
|
|
92
|
+
"of an installed qlpack that depends on codeql/python-all."
|
|
93
|
+
)
|
|
94
|
+
ql_file = tempfile.NamedTemporaryFile(
|
|
95
|
+
"w", delete=False, suffix=".ql", dir=str(self.codeql_packs_dir)
|
|
96
|
+
)
|
|
97
|
+
self.temp_file_path = Path(ql_file.name)
|
|
98
|
+
ql_file.close()
|
|
88
99
|
|
|
89
100
|
return self
|
|
90
101
|
|
|
@@ -108,32 +119,41 @@ class CodeQLQueryRunner:
|
|
|
108
119
|
# Write the query to the temp file so we can execute it.
|
|
109
120
|
self.temp_file_path.write_text(query_string)
|
|
110
121
|
|
|
111
|
-
#
|
|
122
|
+
# The .ql file sits inside the qlpack directory whose lock file
|
|
123
|
+
# already resolves ``codeql/python-all`` and its transitive
|
|
124
|
+
# dependencies. ``codeql query run`` auto-discovers the enclosing
|
|
125
|
+
# qlpack — no extra flags required.
|
|
112
126
|
codeql_query_cmd = shlex.split(
|
|
113
|
-
f"
|
|
127
|
+
f"{shlex.quote(self.codeql_bin)} query run {self.temp_file_path} "
|
|
128
|
+
f"--database={self.database_path} "
|
|
129
|
+
f"--output={self.temp_bqrs_file_path}",
|
|
114
130
|
posix=False,
|
|
115
131
|
)
|
|
116
132
|
|
|
117
|
-
call = subprocess.Popen(
|
|
133
|
+
call = subprocess.Popen(
|
|
134
|
+
codeql_query_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE
|
|
135
|
+
)
|
|
118
136
|
_, err = call.communicate()
|
|
119
137
|
if call.returncode != 0:
|
|
120
138
|
raise CodeQLExceptions.CodeQLQueryExecutionException(
|
|
121
|
-
f"Error executing query: {err.
|
|
139
|
+
f"Error executing query: {(err or b'').decode(errors='replace')}"
|
|
122
140
|
)
|
|
123
141
|
|
|
124
142
|
# Convert the bqrs file to a CSV file
|
|
125
143
|
bqrs2csv_command = shlex.split(
|
|
126
|
-
f"
|
|
144
|
+
f"{shlex.quote(self.codeql_bin)} bqrs decode --format=csv --output={self.csv_output_file} {self.temp_bqrs_file_path}",
|
|
127
145
|
posix=False,
|
|
128
146
|
)
|
|
129
147
|
|
|
130
148
|
# Read the CSV file content and cast it to a DataFrame
|
|
131
149
|
|
|
132
|
-
call = subprocess.Popen(
|
|
150
|
+
call = subprocess.Popen(
|
|
151
|
+
bqrs2csv_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE
|
|
152
|
+
)
|
|
133
153
|
_, err = call.communicate()
|
|
134
154
|
if call.returncode != 0:
|
|
135
155
|
raise CodeQLExceptions.CodeQLQueryExecutionException(
|
|
136
|
-
f"Error
|
|
156
|
+
f"Error decoding bqrs: {(err or b'').decode(errors='replace')}"
|
|
137
157
|
)
|
|
138
158
|
else:
|
|
139
159
|
return pd.read_csv(
|
|
@@ -161,5 +181,5 @@ class CodeQLQueryRunner:
|
|
|
161
181
|
if self.csv_output_file and self.csv_output_file.exists():
|
|
162
182
|
self.csv_output_file.unlink()
|
|
163
183
|
|
|
164
|
-
if self.
|
|
165
|
-
self.
|
|
184
|
+
if self.temp_bqrs_file_path and self.temp_bqrs_file_path.exists():
|
|
185
|
+
self.temp_bqrs_file_path.unlink()
|