codeanalyzer-python 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codeanalyzer/__main__.py +99 -6
- codeanalyzer/core.py +192 -215
- codeanalyzer/neo4j/bolt.py +13 -2
- codeanalyzer/neo4j/catalog.py +2 -2
- codeanalyzer/neo4j/project.py +18 -10
- codeanalyzer/neo4j/rows.py +10 -5
- codeanalyzer/options/__init__.py +2 -2
- codeanalyzer/options/options.py +21 -1
- codeanalyzer/schema/__init__.py +2 -0
- codeanalyzer/schema/py_schema.py +16 -1
- codeanalyzer/semantic_analysis/call_graph.py +24 -3
- codeanalyzer/semantic_analysis/pycg/__init__.py +20 -0
- codeanalyzer/semantic_analysis/pycg/pycg_analysis.py +1054 -0
- codeanalyzer/semantic_analysis/{codeql/__init__.py → pycg/pycg_exceptions.py} +5 -8
- codeanalyzer/semantic_analysis/pycg/shard_planner.py +401 -0
- codeanalyzer/utils/logging.py +5 -2
- codeanalyzer/utils/progress_bar.py +15 -7
- codeanalyzer_python-0.3.0.dist-info/METADATA +550 -0
- codeanalyzer_python-0.3.0.dist-info/RECORD +38 -0
- codeanalyzer/semantic_analysis/codeql/codeql_analysis.py +0 -382
- codeanalyzer/semantic_analysis/codeql/codeql_exceptions.py +0 -12
- codeanalyzer/semantic_analysis/codeql/codeql_loader.py +0 -91
- codeanalyzer/semantic_analysis/codeql/codeql_query_runner.py +0 -185
- codeanalyzer_python-0.2.0.dist-info/METADATA +0 -393
- codeanalyzer_python-0.2.0.dist-info/RECORD +0 -39
- {codeanalyzer_python-0.2.0.dist-info → codeanalyzer_python-0.3.0.dist-info}/WHEEL +0 -0
- {codeanalyzer_python-0.2.0.dist-info → codeanalyzer_python-0.3.0.dist-info}/entry_points.txt +0 -0
- {codeanalyzer_python-0.2.0.dist-info → codeanalyzer_python-0.3.0.dist-info}/licenses/LICENSE +0 -0
- {codeanalyzer_python-0.2.0.dist-info → codeanalyzer_python-0.3.0.dist-info}/licenses/NOTICE +0 -0
|
@@ -1,382 +0,0 @@
|
|
|
1
|
-
################################################################################
|
|
2
|
-
# Copyright IBM Corporation 2025
|
|
3
|
-
#
|
|
4
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
-
# you may not use this file except in compliance with the License.
|
|
6
|
-
# You may obtain a copy of the License at
|
|
7
|
-
#
|
|
8
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
-
#
|
|
10
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
-
# See the License for the specific language governing permissions and
|
|
14
|
-
# limitations under the License.
|
|
15
|
-
################################################################################
|
|
16
|
-
|
|
17
|
-
"""CodeQL module for analyzing Python code using CodeQL.
|
|
18
|
-
|
|
19
|
-
This module provides functionality to create and manage CodeQL databases
|
|
20
|
-
for Python projects and execute queries against them.
|
|
21
|
-
"""
|
|
22
|
-
|
|
23
|
-
from collections import Counter
|
|
24
|
-
from pathlib import Path
|
|
25
|
-
from typing import Any, Dict, Iterator, List, Tuple, Union
|
|
26
|
-
|
|
27
|
-
from pandas import DataFrame
|
|
28
|
-
|
|
29
|
-
from codeanalyzer.schema.py_schema import PyCallEdge, PyModule
|
|
30
|
-
from codeanalyzer.semantic_analysis.call_graph import iter_callables_in_symbol_table
|
|
31
|
-
from codeanalyzer.semantic_analysis.codeql.codeql_query_runner import CodeQLQueryRunner
|
|
32
|
-
from codeanalyzer.utils import logger
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
class _CallableResolver:
|
|
36
|
-
"""Maps a CodeQL endpoint ``(file, start_line, name, arity)`` to a Jedi
|
|
37
|
-
``PyCallable``.
|
|
38
|
-
|
|
39
|
-
Resolution ladder:
|
|
40
|
-
1. exact ``(abs_path, start_line)`` — the precise join;
|
|
41
|
-
2. on miss, candidates sharing ``(abs_path, short_name)``: a single
|
|
42
|
-
candidate is taken directly; otherwise prefer those whose
|
|
43
|
-
parameter count equals the CodeQL positional arity, then the
|
|
44
|
-
nearest ``start_line``;
|
|
45
|
-
3. no name match -> ``None`` (caller row skipped / callee becomes
|
|
46
|
-
a ghost node).
|
|
47
|
-
|
|
48
|
-
Step 2 recovers edges the ``(file, line)`` join silently drops when
|
|
49
|
-
CodeQL and Jedi disagree on a definition's start line (e.g. decorator
|
|
50
|
-
handling). Jedi's ``parameters`` counts every declared slot (incl.
|
|
51
|
-
``*args``/``**kwargs``/keyword-only) whereas CodeQL's arity is
|
|
52
|
-
positional only, so the arity filter is exact for plain signatures
|
|
53
|
-
and otherwise yields to the nearest-line tiebreak.
|
|
54
|
-
"""
|
|
55
|
-
|
|
56
|
-
def __init__(self) -> None:
|
|
57
|
-
self._by_loc: Dict[Tuple[str, int], Any] = {}
|
|
58
|
-
self._by_name: Dict[Tuple[str, str], List[Any]] = {}
|
|
59
|
-
|
|
60
|
-
@staticmethod
|
|
61
|
-
def _abs(path: str) -> str:
|
|
62
|
-
try:
|
|
63
|
-
return str(Path(path).resolve())
|
|
64
|
-
except (OSError, RuntimeError):
|
|
65
|
-
return path
|
|
66
|
-
|
|
67
|
-
@classmethod
|
|
68
|
-
def from_symbol_table(
|
|
69
|
-
cls, symbol_table: Dict[str, PyModule]
|
|
70
|
-
) -> "_CallableResolver":
|
|
71
|
-
resolver = cls()
|
|
72
|
-
for c in iter_callables_in_symbol_table(symbol_table):
|
|
73
|
-
abs_path = cls._abs(c.path)
|
|
74
|
-
resolver._by_loc[(abs_path, c.start_line)] = c
|
|
75
|
-
resolver._by_name.setdefault((abs_path, c.name), []).append(c)
|
|
76
|
-
return resolver
|
|
77
|
-
|
|
78
|
-
def resolve(
|
|
79
|
-
self, file: str, start_line: int, name: str, arity: int
|
|
80
|
-
) -> Any:
|
|
81
|
-
exact = self._by_loc.get((file, start_line))
|
|
82
|
-
if exact is not None:
|
|
83
|
-
return exact
|
|
84
|
-
if not name:
|
|
85
|
-
return None
|
|
86
|
-
candidates = self._by_name.get((file, name))
|
|
87
|
-
if not candidates:
|
|
88
|
-
return None
|
|
89
|
-
if len(candidates) == 1:
|
|
90
|
-
return candidates[0]
|
|
91
|
-
arity_matched = [c for c in candidates if len(c.parameters) == arity]
|
|
92
|
-
pool = arity_matched or candidates
|
|
93
|
-
return min(pool, key=lambda c: abs(c.start_line - start_line))
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
class CodeQL:
|
|
97
|
-
"""A class for building the application view of a Python application using CodeQL.
|
|
98
|
-
|
|
99
|
-
Args:
|
|
100
|
-
project_dir (str or Path): The path to the root of the Python project.
|
|
101
|
-
|
|
102
|
-
Attributes:
|
|
103
|
-
db_path (Path): The path to the CodeQL database.
|
|
104
|
-
temp_db (TemporaryDirectory or None): The temporary directory object if a temporary database was created.
|
|
105
|
-
"""
|
|
106
|
-
|
|
107
|
-
def __init__(
|
|
108
|
-
self,
|
|
109
|
-
project_dir: Union[str, Path],
|
|
110
|
-
db_path: Path,
|
|
111
|
-
codeql_bin: Union[str, Path, None] = None,
|
|
112
|
-
codeql_packs_dir: Union[str, Path, None] = None,
|
|
113
|
-
) -> None:
|
|
114
|
-
self.project_dir = project_dir
|
|
115
|
-
self.db_path = db_path
|
|
116
|
-
self.codeql_bin = codeql_bin
|
|
117
|
-
self.codeql_packs_dir = codeql_packs_dir
|
|
118
|
-
self._cached_df: "DataFrame | None" = None
|
|
119
|
-
|
|
120
|
-
def _query_call_edges(self) -> DataFrame:
|
|
121
|
-
"""Runs the CodeQL query that emits one row per resolved call site.
|
|
122
|
-
|
|
123
|
-
The query is written against CodeQL's Python library (``import python``).
|
|
124
|
-
It returns physical location handles for both endpoints so the
|
|
125
|
-
downstream post-processor can join into Jedi's existing
|
|
126
|
-
``PyCallable.signature`` space via ``(file_path, start_line)`` —
|
|
127
|
-
no signature normalization required.
|
|
128
|
-
|
|
129
|
-
Filters:
|
|
130
|
-
* Caller must be a ``Function`` (skip module-level / class-body
|
|
131
|
-
calls — they have no ``PyCallable`` to anchor to).
|
|
132
|
-
* Callee may resolve to anything (in-source or library stub);
|
|
133
|
-
non-application callees become **ghost** nodes downstream so
|
|
134
|
-
RPC / third-party / framework edges are preserved.
|
|
135
|
-
|
|
136
|
-
Returns:
|
|
137
|
-
DataFrame: one row per resolved (caller, callee, call-site)
|
|
138
|
-
triple. Duplicate ``(caller_file, caller_start_line,
|
|
139
|
-
callee_file, callee_start_line)`` tuples represent multiple
|
|
140
|
-
call sites in the same caller targeting the same callee and
|
|
141
|
-
are coalesced into a single ``PyCallEdge`` (weight = count)
|
|
142
|
-
by the post-processor.
|
|
143
|
-
"""
|
|
144
|
-
query = [
|
|
145
|
-
"/**",
|
|
146
|
-
" * @name Python call-graph edges",
|
|
147
|
-
" * @description One row per resolved call site: caller, callee,",
|
|
148
|
-
" * and the call-expression location.",
|
|
149
|
-
" * @kind table",
|
|
150
|
-
" * @id py/codeanalyzer/call-graph-edges",
|
|
151
|
-
" */",
|
|
152
|
-
"import python",
|
|
153
|
-
# ``FunctionValue`` / ``ClassValue`` / the ``pointsTo`` predicate
|
|
154
|
-
# live in ObjectAPI, which ``import python`` only brings in as a
|
|
155
|
-
# private import — they aren't re-exported. Pull them in
|
|
156
|
-
# explicitly.
|
|
157
|
-
"import semmle.python.objects.ObjectAPI",
|
|
158
|
-
"",
|
|
159
|
-
# ``Value.getACall()`` is the modern call-resolution API in
|
|
160
|
-
# codeql/python-all 7.x — it returns the ``CallNode`` (CFG)
|
|
161
|
-
# whose target was resolved to that ``Value``. Cleaner than
|
|
162
|
-
# poking at ``pointsTo`` directly.
|
|
163
|
-
# ``callee`` is bound to the FunctionValue's scope so the
|
|
164
|
-
# endpoint emits the same Function-level facts (name, arity,
|
|
165
|
-
# location) the post-processor needs for the name+arity
|
|
166
|
-
# fallback when the (file, start_line) join misses.
|
|
167
|
-
"from CallNode call, Function caller, FunctionValue calleeVal, Function callee",
|
|
168
|
-
"where",
|
|
169
|
-
" call.getScope() = caller and",
|
|
170
|
-
" callee = calleeVal.getScope() and",
|
|
171
|
-
" (",
|
|
172
|
-
# Direct function / bound-method call: foo() or obj.foo()
|
|
173
|
-
" call = calleeVal.getACall()",
|
|
174
|
-
" or",
|
|
175
|
-
# Constructor call: A(...) resolves to a ClassValue; the actual
|
|
176
|
-
# callee is the class's __init__ (via MRO lookup so subclasses
|
|
177
|
-
# without an explicit __init__ still resolve to the inherited one).
|
|
178
|
-
" exists(ClassValue clsVal |",
|
|
179
|
-
" call = clsVal.getACall() and",
|
|
180
|
-
' clsVal.lookup("__init__") = calleeVal',
|
|
181
|
-
" )",
|
|
182
|
-
" )",
|
|
183
|
-
"select",
|
|
184
|
-
# --- Caller endpoint --- (joins to PyCallable: exact by
|
|
185
|
-
# (file, start_line), else by (file, name) + arity)
|
|
186
|
-
" caller.getLocation().getFile().getAbsolutePath(),",
|
|
187
|
-
" caller.getLocation().getStartLine(),",
|
|
188
|
-
" caller.getQualifiedName(),",
|
|
189
|
-
" caller.getName(),",
|
|
190
|
-
" count(caller.getArg(_)),",
|
|
191
|
-
# --- Callee endpoint --- (file/line may live in a library stub;
|
|
192
|
-
# post-processor classifies as in-source or ghost)
|
|
193
|
-
" callee.getLocation().getFile().getAbsolutePath(),",
|
|
194
|
-
" callee.getLocation().getStartLine(),",
|
|
195
|
-
" calleeVal.getQualifiedName(),",
|
|
196
|
-
" callee.getName(),",
|
|
197
|
-
" count(callee.getArg(_)),",
|
|
198
|
-
# --- Call-site location --- (for PyCallsite augmentation)
|
|
199
|
-
" call.getLocation().getStartLine(),",
|
|
200
|
-
" call.getLocation().getStartColumn(),",
|
|
201
|
-
" call.getLocation().getEndLine(),",
|
|
202
|
-
" call.getLocation().getEndColumn()",
|
|
203
|
-
# ``is_constructor`` is derived in the post-processor by
|
|
204
|
-
# checking whether ``callee_qname`` ends in ``.__init__``;
|
|
205
|
-
# avoids QL's restrictive ``if-then-else`` typing here.
|
|
206
|
-
]
|
|
207
|
-
if self._cached_df is not None:
|
|
208
|
-
return self._cached_df
|
|
209
|
-
|
|
210
|
-
query_string = "\n".join(query)
|
|
211
|
-
|
|
212
|
-
with CodeQLQueryRunner(
|
|
213
|
-
self.db_path,
|
|
214
|
-
codeql_bin=self.codeql_bin,
|
|
215
|
-
codeql_packs_dir=self.codeql_packs_dir,
|
|
216
|
-
) as runner:
|
|
217
|
-
df: DataFrame = runner.execute(
|
|
218
|
-
query_string,
|
|
219
|
-
column_names=[
|
|
220
|
-
"caller_file",
|
|
221
|
-
"caller_start_line",
|
|
222
|
-
"caller_qname",
|
|
223
|
-
"caller_name",
|
|
224
|
-
"caller_arity",
|
|
225
|
-
"callee_file",
|
|
226
|
-
"callee_start_line",
|
|
227
|
-
"callee_qname",
|
|
228
|
-
"callee_name",
|
|
229
|
-
"callee_arity",
|
|
230
|
-
"call_start_line",
|
|
231
|
-
"call_start_column",
|
|
232
|
-
"call_end_line",
|
|
233
|
-
"call_end_column",
|
|
234
|
-
],
|
|
235
|
-
)
|
|
236
|
-
self._cached_df = df
|
|
237
|
-
return df
|
|
238
|
-
|
|
239
|
-
@staticmethod
|
|
240
|
-
def _build_callable_resolver(
|
|
241
|
-
symbol_table: Dict[str, PyModule],
|
|
242
|
-
) -> _CallableResolver:
|
|
243
|
-
"""Build the endpoint -> ``PyCallable`` resolver from Jedi.
|
|
244
|
-
|
|
245
|
-
Paths are resolved so they match CodeQL's ``getAbsolutePath()``
|
|
246
|
-
regardless of symlinks or the current working directory.
|
|
247
|
-
"""
|
|
248
|
-
return _CallableResolver.from_symbol_table(symbol_table)
|
|
249
|
-
|
|
250
|
-
def _iter_resolved_rows(
|
|
251
|
-
self, symbol_table: Dict[str, PyModule]
|
|
252
|
-
) -> "Iterator[Tuple[str, str, Any]]":
|
|
253
|
-
"""Yield ``(source_sig, target_sig, row)`` for every CodeQL row.
|
|
254
|
-
|
|
255
|
-
Rows whose caller can't be matched to a ``PyCallable`` in the
|
|
256
|
-
symbol table are skipped. Callee misses fall back to
|
|
257
|
-
``row.callee_qname`` (ghost). Used by both edge construction and
|
|
258
|
-
call-site augmentation so a single CodeQL query feeds both.
|
|
259
|
-
"""
|
|
260
|
-
df = self._query_call_edges()
|
|
261
|
-
if df.empty:
|
|
262
|
-
return
|
|
263
|
-
resolver = self._build_callable_resolver(symbol_table)
|
|
264
|
-
|
|
265
|
-
skipped_unknown_caller = 0
|
|
266
|
-
ghost_callees = 0
|
|
267
|
-
for row in df.itertuples(index=False):
|
|
268
|
-
caller = resolver.resolve(
|
|
269
|
-
row.caller_file,
|
|
270
|
-
int(row.caller_start_line),
|
|
271
|
-
row.caller_name,
|
|
272
|
-
int(row.caller_arity),
|
|
273
|
-
)
|
|
274
|
-
if caller is None:
|
|
275
|
-
skipped_unknown_caller += 1
|
|
276
|
-
continue
|
|
277
|
-
|
|
278
|
-
callee = resolver.resolve(
|
|
279
|
-
row.callee_file,
|
|
280
|
-
int(row.callee_start_line),
|
|
281
|
-
row.callee_name,
|
|
282
|
-
int(row.callee_arity),
|
|
283
|
-
)
|
|
284
|
-
if callee is not None:
|
|
285
|
-
target_sig = callee.signature
|
|
286
|
-
else:
|
|
287
|
-
target_sig = row.callee_qname
|
|
288
|
-
ghost_callees += 1
|
|
289
|
-
|
|
290
|
-
yield caller.signature, target_sig, row
|
|
291
|
-
|
|
292
|
-
if skipped_unknown_caller:
|
|
293
|
-
logger.debug(
|
|
294
|
-
f"CodeQL: skipped {skipped_unknown_caller} rows whose caller "
|
|
295
|
-
f"was not in Jedi's symbol table."
|
|
296
|
-
)
|
|
297
|
-
if ghost_callees:
|
|
298
|
-
logger.debug(
|
|
299
|
-
f"CodeQL: {ghost_callees} rows resolved to ghost (external) callees."
|
|
300
|
-
)
|
|
301
|
-
|
|
302
|
-
def build_call_graph_edges(
|
|
303
|
-
self, symbol_table: Dict[str, PyModule]
|
|
304
|
-
) -> List[PyCallEdge]:
|
|
305
|
-
"""Run the CodeQL query and turn each row into a ``PyCallEdge``.
|
|
306
|
-
|
|
307
|
-
Edges are coalesced on ``(source, target)`` — ``weight`` is the
|
|
308
|
-
number of distinct call sites in the caller targeting the callee.
|
|
309
|
-
Provenance is always ``["codeql"]``; combine with Jedi-derived
|
|
310
|
-
edges via ``call_graph.merge_edges``.
|
|
311
|
-
"""
|
|
312
|
-
edge_counts: Counter = Counter()
|
|
313
|
-
for source_sig, target_sig, _row in self._iter_resolved_rows(symbol_table):
|
|
314
|
-
edge_counts[(source_sig, target_sig)] += 1
|
|
315
|
-
|
|
316
|
-
return [
|
|
317
|
-
PyCallEdge(
|
|
318
|
-
source=src,
|
|
319
|
-
target=dst,
|
|
320
|
-
weight=count,
|
|
321
|
-
provenance=["codeql"],
|
|
322
|
-
)
|
|
323
|
-
for (src, dst), count in edge_counts.items()
|
|
324
|
-
]
|
|
325
|
-
|
|
326
|
-
def augment_call_sites(self, symbol_table: Dict[str, PyModule]) -> int:
|
|
327
|
-
"""Backfill ``PyCallsite.callee_signature`` using CodeQL resolution.
|
|
328
|
-
|
|
329
|
-
Walks every CodeQL row, locates the matching ``PyCallsite`` inside
|
|
330
|
-
the caller's ``PyCallable.call_sites`` by call-expression line range
|
|
331
|
-
(``start_line``, ``end_line``), and fills in ``callee_signature``
|
|
332
|
-
**only when Jedi left it empty**. Existing Jedi-resolved signatures
|
|
333
|
-
are kept (Jedi sees lexical context CodeQL can't, e.g. closures).
|
|
334
|
-
|
|
335
|
-
Match is by line range — column matching is brittle across the two
|
|
336
|
-
tools' 0- vs 1-based conventions. Ambiguity on a single line
|
|
337
|
-
(e.g. ``a.b().c()``) resolves to the first matching site, which is
|
|
338
|
-
an acceptable approximation given how rarely Jedi misses callees
|
|
339
|
-
on chained call lines.
|
|
340
|
-
|
|
341
|
-
Returns:
|
|
342
|
-
Number of ``PyCallsite`` entries augmented.
|
|
343
|
-
"""
|
|
344
|
-
resolver = self._build_callable_resolver(symbol_table)
|
|
345
|
-
df = self._query_call_edges()
|
|
346
|
-
if df.empty:
|
|
347
|
-
return 0
|
|
348
|
-
|
|
349
|
-
augmented = 0
|
|
350
|
-
for row in df.itertuples(index=False):
|
|
351
|
-
caller = resolver.resolve(
|
|
352
|
-
row.caller_file,
|
|
353
|
-
int(row.caller_start_line),
|
|
354
|
-
row.caller_name,
|
|
355
|
-
int(row.caller_arity),
|
|
356
|
-
)
|
|
357
|
-
if caller is None:
|
|
358
|
-
continue
|
|
359
|
-
|
|
360
|
-
callee = resolver.resolve(
|
|
361
|
-
row.callee_file,
|
|
362
|
-
int(row.callee_start_line),
|
|
363
|
-
row.callee_name,
|
|
364
|
-
int(row.callee_arity),
|
|
365
|
-
)
|
|
366
|
-
resolved_sig = callee.signature if callee is not None else row.callee_qname
|
|
367
|
-
|
|
368
|
-
call_start = int(row.call_start_line)
|
|
369
|
-
call_end = int(row.call_end_line)
|
|
370
|
-
for site in caller.call_sites:
|
|
371
|
-
if site.start_line != call_start or site.end_line != call_end:
|
|
372
|
-
continue
|
|
373
|
-
if not site.callee_signature:
|
|
374
|
-
site.callee_signature = resolved_sig
|
|
375
|
-
augmented += 1
|
|
376
|
-
break
|
|
377
|
-
|
|
378
|
-
if augmented:
|
|
379
|
-
logger.debug(
|
|
380
|
-
f"CodeQL: augmented {augmented} PyCallsite.callee_signature entries."
|
|
381
|
-
)
|
|
382
|
-
return augmented
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
class CodeQLExceptions:
|
|
2
|
-
class CodeQLDatabaseBuildException(Exception):
|
|
3
|
-
"""Exception raised when there is an error building the CodeQL database."""
|
|
4
|
-
|
|
5
|
-
def __init__(self, message: str) -> None:
|
|
6
|
-
super().__init__(message)
|
|
7
|
-
|
|
8
|
-
class CodeQLQueryExecutionException(Exception):
|
|
9
|
-
"""Exception raised when there is an error building the CodeQL database."""
|
|
10
|
-
|
|
11
|
-
def __init__(self, message: str) -> None:
|
|
12
|
-
super().__init__(message)
|
|
@@ -1,91 +0,0 @@
|
|
|
1
|
-
import os
|
|
2
|
-
import platform
|
|
3
|
-
import stat
|
|
4
|
-
import zipfile
|
|
5
|
-
from pathlib import Path
|
|
6
|
-
|
|
7
|
-
import requests
|
|
8
|
-
from codeanalyzer.utils import logger
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
class CodeQLLoader:
|
|
12
|
-
@classmethod
|
|
13
|
-
def detect_platform_key(cls) -> str:
|
|
14
|
-
system = platform.system()
|
|
15
|
-
arch = platform.machine().lower()
|
|
16
|
-
|
|
17
|
-
if system == "Linux" and arch in {"x86_64", "amd64"}:
|
|
18
|
-
return "codeql-linux64.zip"
|
|
19
|
-
elif system == "Darwin" and arch in {"x86_64", "arm64"}:
|
|
20
|
-
return "codeql-osx64.zip"
|
|
21
|
-
elif system == "Windows" and arch in {"x86_64", "amd64"}:
|
|
22
|
-
return "codeql-win64.zip"
|
|
23
|
-
else:
|
|
24
|
-
return "codeql.zip" # fallback to generic binary if needed
|
|
25
|
-
|
|
26
|
-
@classmethod
|
|
27
|
-
def get_codeql_download_url(cls, expected_filename: str) -> str:
|
|
28
|
-
response = requests.get(
|
|
29
|
-
"https://api.github.com/repos/github/codeql-cli-binaries/releases/latest"
|
|
30
|
-
)
|
|
31
|
-
response.raise_for_status()
|
|
32
|
-
for asset in response.json()["assets"]:
|
|
33
|
-
if asset["name"] == expected_filename:
|
|
34
|
-
return asset["browser_download_url"]
|
|
35
|
-
raise RuntimeError(f"No asset found for filename: {expected_filename}")
|
|
36
|
-
|
|
37
|
-
@classmethod
|
|
38
|
-
def download_and_extract_codeql(cls, temp_dir: Path) -> Path:
|
|
39
|
-
filename = cls.detect_platform_key()
|
|
40
|
-
download_url = cls.get_codeql_download_url(filename)
|
|
41
|
-
|
|
42
|
-
temp_dir.mkdir(parents=True, exist_ok=True)
|
|
43
|
-
archive_path = temp_dir / filename
|
|
44
|
-
|
|
45
|
-
logger.info(f"Downloading CodeQL CLI from {download_url}")
|
|
46
|
-
with requests.get(download_url, stream=True) as r:
|
|
47
|
-
r.raise_for_status()
|
|
48
|
-
block_size = 8192 # 8KB
|
|
49
|
-
|
|
50
|
-
with open(archive_path, "wb") as f:
|
|
51
|
-
for chunk in r.iter_content(chunk_size=block_size):
|
|
52
|
-
f.write(chunk)
|
|
53
|
-
|
|
54
|
-
extract_dir = temp_dir / filename.replace(".zip", "")
|
|
55
|
-
extract_dir.mkdir(exist_ok=True)
|
|
56
|
-
|
|
57
|
-
logger.info(f"Extracting CodeQL CLI to {extract_dir}")
|
|
58
|
-
# zipfile.extractall drops Unix permissions (the executable bit), so
|
|
59
|
-
# we extract entries manually and copy each one's stored mode onto
|
|
60
|
-
# the file system. Without this, the CodeQL launcher script can't
|
|
61
|
-
# be executed and the next subprocess.Popen raises PermissionError.
|
|
62
|
-
with zipfile.ZipFile(archive_path, "r") as zip_ref:
|
|
63
|
-
for info in zip_ref.infolist():
|
|
64
|
-
extracted_path = zip_ref.extract(info, extract_dir)
|
|
65
|
-
stored_mode = info.external_attr >> 16
|
|
66
|
-
if stored_mode:
|
|
67
|
-
os.chmod(extracted_path, stored_mode)
|
|
68
|
-
|
|
69
|
-
# Archive is no longer needed once extracted.
|
|
70
|
-
try:
|
|
71
|
-
archive_path.unlink()
|
|
72
|
-
except OSError as exc:
|
|
73
|
-
logger.warning(f"Could not remove CodeQL archive {archive_path}: {exc}")
|
|
74
|
-
|
|
75
|
-
# rglob("codeql") returns both the launcher file *and* an internal
|
|
76
|
-
# directory of the same name (CodeQL ships its own runtime under
|
|
77
|
-
# ``codeql/codeql/``); insist on a regular file so we never bind to
|
|
78
|
-
# the directory.
|
|
79
|
-
codeql_bin = next(
|
|
80
|
-
(p for p in extract_dir.rglob("codeql") if p.is_file()),
|
|
81
|
-
None,
|
|
82
|
-
)
|
|
83
|
-
if not codeql_bin:
|
|
84
|
-
raise FileNotFoundError("CodeQL binary not found in extracted contents.")
|
|
85
|
-
|
|
86
|
-
# Belt-and-suspenders: ensure the binary is executable even if the
|
|
87
|
-
# archive entry's mode was zero (some older zip producers omit it).
|
|
88
|
-
st = codeql_bin.stat()
|
|
89
|
-
codeql_bin.chmod(st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
|
90
|
-
|
|
91
|
-
return codeql_bin.resolve()
|
|
@@ -1,185 +0,0 @@
|
|
|
1
|
-
################################################################################
|
|
2
|
-
# Copyright IBM Corporation 2025
|
|
3
|
-
#
|
|
4
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
-
# you may not use this file except in compliance with the License.
|
|
6
|
-
# You may obtain a copy of the License at
|
|
7
|
-
#
|
|
8
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
-
#
|
|
10
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
-
# See the License for the specific language governing permissions and
|
|
14
|
-
# limitations under the License.
|
|
15
|
-
################################################################################
|
|
16
|
-
|
|
17
|
-
"""Backend module for CodeQL query execution.
|
|
18
|
-
|
|
19
|
-
This module provides functionality to run CodeQL queries against CodeQL databases
|
|
20
|
-
and process the results.
|
|
21
|
-
"""
|
|
22
|
-
|
|
23
|
-
import shlex
|
|
24
|
-
import subprocess
|
|
25
|
-
import tempfile
|
|
26
|
-
from pathlib import Path
|
|
27
|
-
from typing import List
|
|
28
|
-
|
|
29
|
-
import pandas as pd
|
|
30
|
-
from pandas import DataFrame
|
|
31
|
-
|
|
32
|
-
from codeanalyzer.semantic_analysis.codeql.codeql_exceptions import CodeQLExceptions
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
class CodeQLQueryRunner:
|
|
36
|
-
"""A class for executing CodeQL queries against a CodeQL database.
|
|
37
|
-
|
|
38
|
-
This class provides a context manager interface for executing CodeQL queries
|
|
39
|
-
and handling temporary resources needed during query execution.
|
|
40
|
-
|
|
41
|
-
Args:
|
|
42
|
-
database_path (str): The path to the CodeQL database.
|
|
43
|
-
codeql_bin (str | Path | None): Absolute path to the CodeQL CLI
|
|
44
|
-
binary. When ``None``, falls back to whatever ``codeql`` is on
|
|
45
|
-
``PATH``.
|
|
46
|
-
|
|
47
|
-
Attributes:
|
|
48
|
-
database_path (Path): The path to the CodeQL database.
|
|
49
|
-
codeql_bin (str): Resolved binary path or the literal ``"codeql"``.
|
|
50
|
-
temp_file_path (Path): The path to the temporary query file.
|
|
51
|
-
csv_output_file (Path): The path to the CSV output file.
|
|
52
|
-
temp_bqrs_file_path (Path): The path to the temporary bqrs file.
|
|
53
|
-
temp_qlpack_file (Path): The path to the temporary qlpack file.
|
|
54
|
-
|
|
55
|
-
Raises:
|
|
56
|
-
CodeQLQueryExecutionException: If there is an error executing the query.
|
|
57
|
-
"""
|
|
58
|
-
|
|
59
|
-
def __init__(self, database_path: str, codeql_bin=None, codeql_packs_dir=None):
|
|
60
|
-
self.database_path: Path = Path(database_path)
|
|
61
|
-
self.codeql_bin: str = str(codeql_bin) if codeql_bin else "codeql"
|
|
62
|
-
self.codeql_packs_dir = (
|
|
63
|
-
Path(codeql_packs_dir) if codeql_packs_dir is not None else None
|
|
64
|
-
)
|
|
65
|
-
self.temp_file_path: Path = None
|
|
66
|
-
|
|
67
|
-
def __enter__(self):
|
|
68
|
-
"""Context entry that prepares paths to execute a CodeQL query.
|
|
69
|
-
|
|
70
|
-
The ``.ql`` file is written **inside the prepared qlpack
|
|
71
|
-
directory** (``codeql_packs_dir``) so ``import python`` resolves
|
|
72
|
-
against that pack's installed dependencies — no
|
|
73
|
-
``--additional-packs`` or ``--search-path`` needed. The CSV /
|
|
74
|
-
BQRS output files live in ``tempfile`` because they're transient
|
|
75
|
-
per-query artifacts.
|
|
76
|
-
"""
|
|
77
|
-
# CSV and BQRS files are transient per-query — fine in /tmp.
|
|
78
|
-
csv_file = tempfile.NamedTemporaryFile("w", delete=False, suffix=".csv")
|
|
79
|
-
bqrs_file = tempfile.NamedTemporaryFile("w", delete=False, suffix=".bqrs")
|
|
80
|
-
self.csv_output_file = Path(csv_file.name)
|
|
81
|
-
self.temp_bqrs_file_path = Path(bqrs_file.name)
|
|
82
|
-
csv_file.close()
|
|
83
|
-
bqrs_file.close()
|
|
84
|
-
|
|
85
|
-
# The .ql file MUST live inside the prepared qlpack so its
|
|
86
|
-
# ``import python`` resolves via that pack's lock file. Writing
|
|
87
|
-
# outside the pack means CodeQL falls back to a default
|
|
88
|
-
# search-path that doesn't include downloaded library packs.
|
|
89
|
-
if self.codeql_packs_dir is None:
|
|
90
|
-
raise RuntimeError(
|
|
91
|
-
"CodeQLQueryRunner requires codeql_packs_dir — the directory "
|
|
92
|
-
"of an installed qlpack that depends on codeql/python-all."
|
|
93
|
-
)
|
|
94
|
-
ql_file = tempfile.NamedTemporaryFile(
|
|
95
|
-
"w", delete=False, suffix=".ql", dir=str(self.codeql_packs_dir)
|
|
96
|
-
)
|
|
97
|
-
self.temp_file_path = Path(ql_file.name)
|
|
98
|
-
ql_file.close()
|
|
99
|
-
|
|
100
|
-
return self
|
|
101
|
-
|
|
102
|
-
def execute(self, query_string: str, column_names: List[str]) -> DataFrame:
|
|
103
|
-
"""Writes the query to the temporary file and executes it against the specified CodeQL database.
|
|
104
|
-
|
|
105
|
-
Args:
|
|
106
|
-
query_string (str): The CodeQL query string to be executed.
|
|
107
|
-
column_names (List[str]): The list of column names for the CSV the CodeQL produces when we execute the query.
|
|
108
|
-
|
|
109
|
-
Returns:
|
|
110
|
-
dict: A dictionary containing the resulting DataFrame.
|
|
111
|
-
|
|
112
|
-
Raises:
|
|
113
|
-
RuntimeError: If the context manager is not entered using the 'with' statement.
|
|
114
|
-
CodeQLQueryExecutionException: If there is an error executing the query.
|
|
115
|
-
"""
|
|
116
|
-
if not self.temp_file_path:
|
|
117
|
-
raise RuntimeError("CodeQLQueryRunner not entered using 'with' statement.")
|
|
118
|
-
|
|
119
|
-
# Write the query to the temp file so we can execute it.
|
|
120
|
-
self.temp_file_path.write_text(query_string)
|
|
121
|
-
|
|
122
|
-
# The .ql file sits inside the qlpack directory whose lock file
|
|
123
|
-
# already resolves ``codeql/python-all`` and its transitive
|
|
124
|
-
# dependencies. ``codeql query run`` auto-discovers the enclosing
|
|
125
|
-
# qlpack — no extra flags required.
|
|
126
|
-
codeql_query_cmd = shlex.split(
|
|
127
|
-
f"{shlex.quote(self.codeql_bin)} query run {self.temp_file_path} "
|
|
128
|
-
f"--database={self.database_path} "
|
|
129
|
-
f"--output={self.temp_bqrs_file_path}",
|
|
130
|
-
posix=False,
|
|
131
|
-
)
|
|
132
|
-
|
|
133
|
-
call = subprocess.Popen(
|
|
134
|
-
codeql_query_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE
|
|
135
|
-
)
|
|
136
|
-
_, err = call.communicate()
|
|
137
|
-
if call.returncode != 0:
|
|
138
|
-
raise CodeQLExceptions.CodeQLQueryExecutionException(
|
|
139
|
-
f"Error executing query: {(err or b'').decode(errors='replace')}"
|
|
140
|
-
)
|
|
141
|
-
|
|
142
|
-
# Convert the bqrs file to a CSV file
|
|
143
|
-
bqrs2csv_command = shlex.split(
|
|
144
|
-
f"{shlex.quote(self.codeql_bin)} bqrs decode --format=csv --output={self.csv_output_file} {self.temp_bqrs_file_path}",
|
|
145
|
-
posix=False,
|
|
146
|
-
)
|
|
147
|
-
|
|
148
|
-
# Read the CSV file content and cast it to a DataFrame
|
|
149
|
-
|
|
150
|
-
call = subprocess.Popen(
|
|
151
|
-
bqrs2csv_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE
|
|
152
|
-
)
|
|
153
|
-
_, err = call.communicate()
|
|
154
|
-
if call.returncode != 0:
|
|
155
|
-
raise CodeQLExceptions.CodeQLQueryExecutionException(
|
|
156
|
-
f"Error decoding bqrs: {(err or b'').decode(errors='replace')}"
|
|
157
|
-
)
|
|
158
|
-
else:
|
|
159
|
-
return pd.read_csv(
|
|
160
|
-
self.csv_output_file,
|
|
161
|
-
header=None,
|
|
162
|
-
names=column_names,
|
|
163
|
-
skiprows=[0],
|
|
164
|
-
)
|
|
165
|
-
|
|
166
|
-
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
167
|
-
"""Clean up resources used by the CodeQL analysis.
|
|
168
|
-
|
|
169
|
-
Args:
|
|
170
|
-
exc_type: The exception type if an exception was raised in the context, otherwise None.
|
|
171
|
-
exc_val: The exception instance if an exception was raised in the context, otherwise None.
|
|
172
|
-
exc_tb: The traceback if an exception was raised in the context, otherwise None.
|
|
173
|
-
|
|
174
|
-
Note:
|
|
175
|
-
Deletes the temporary files created during the analysis, including the temporary file path,
|
|
176
|
-
the CSV output file, and the temporary QL pack file.
|
|
177
|
-
"""
|
|
178
|
-
if self.temp_file_path and self.temp_file_path.exists():
|
|
179
|
-
self.temp_file_path.unlink()
|
|
180
|
-
|
|
181
|
-
if self.csv_output_file and self.csv_output_file.exists():
|
|
182
|
-
self.csv_output_file.unlink()
|
|
183
|
-
|
|
184
|
-
if self.temp_bqrs_file_path and self.temp_bqrs_file_path.exists():
|
|
185
|
-
self.temp_bqrs_file_path.unlink()
|