codeanalyzer-python 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. codeanalyzer/__main__.py +99 -6
  2. codeanalyzer/core.py +192 -215
  3. codeanalyzer/neo4j/bolt.py +13 -2
  4. codeanalyzer/neo4j/catalog.py +2 -2
  5. codeanalyzer/neo4j/project.py +18 -10
  6. codeanalyzer/neo4j/rows.py +10 -5
  7. codeanalyzer/options/__init__.py +2 -2
  8. codeanalyzer/options/options.py +21 -1
  9. codeanalyzer/schema/__init__.py +2 -0
  10. codeanalyzer/schema/py_schema.py +16 -1
  11. codeanalyzer/semantic_analysis/call_graph.py +24 -3
  12. codeanalyzer/semantic_analysis/pycg/__init__.py +20 -0
  13. codeanalyzer/semantic_analysis/pycg/pycg_analysis.py +1054 -0
  14. codeanalyzer/semantic_analysis/{codeql/__init__.py → pycg/pycg_exceptions.py} +5 -8
  15. codeanalyzer/semantic_analysis/pycg/shard_planner.py +401 -0
  16. codeanalyzer/utils/logging.py +5 -2
  17. codeanalyzer/utils/progress_bar.py +15 -7
  18. codeanalyzer_python-0.3.0.dist-info/METADATA +550 -0
  19. codeanalyzer_python-0.3.0.dist-info/RECORD +38 -0
  20. codeanalyzer/semantic_analysis/codeql/codeql_analysis.py +0 -382
  21. codeanalyzer/semantic_analysis/codeql/codeql_exceptions.py +0 -12
  22. codeanalyzer/semantic_analysis/codeql/codeql_loader.py +0 -91
  23. codeanalyzer/semantic_analysis/codeql/codeql_query_runner.py +0 -185
  24. codeanalyzer_python-0.2.0.dist-info/METADATA +0 -393
  25. codeanalyzer_python-0.2.0.dist-info/RECORD +0 -39
  26. {codeanalyzer_python-0.2.0.dist-info → codeanalyzer_python-0.3.0.dist-info}/WHEEL +0 -0
  27. {codeanalyzer_python-0.2.0.dist-info → codeanalyzer_python-0.3.0.dist-info}/entry_points.txt +0 -0
  28. {codeanalyzer_python-0.2.0.dist-info → codeanalyzer_python-0.3.0.dist-info}/licenses/LICENSE +0 -0
  29. {codeanalyzer_python-0.2.0.dist-info → codeanalyzer_python-0.3.0.dist-info}/licenses/NOTICE +0 -0
@@ -1,382 +0,0 @@
1
- ################################################################################
2
- # Copyright IBM Corporation 2025
3
- #
4
- # Licensed under the Apache License, Version 2.0 (the "License");
5
- # you may not use this file except in compliance with the License.
6
- # You may obtain a copy of the License at
7
- #
8
- # http://www.apache.org/licenses/LICENSE-2.0
9
- #
10
- # Unless required by applicable law or agreed to in writing, software
11
- # distributed under the License is distributed on an "AS IS" BASIS,
12
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
- # See the License for the specific language governing permissions and
14
- # limitations under the License.
15
- ################################################################################
16
-
17
- """CodeQL module for analyzing Python code using CodeQL.
18
-
19
- This module provides functionality to create and manage CodeQL databases
20
- for Python projects and execute queries against them.
21
- """
22
-
23
- from collections import Counter
24
- from pathlib import Path
25
- from typing import Any, Dict, Iterator, List, Tuple, Union
26
-
27
- from pandas import DataFrame
28
-
29
- from codeanalyzer.schema.py_schema import PyCallEdge, PyModule
30
- from codeanalyzer.semantic_analysis.call_graph import iter_callables_in_symbol_table
31
- from codeanalyzer.semantic_analysis.codeql.codeql_query_runner import CodeQLQueryRunner
32
- from codeanalyzer.utils import logger
33
-
34
-
35
- class _CallableResolver:
36
- """Maps a CodeQL endpoint ``(file, start_line, name, arity)`` to a Jedi
37
- ``PyCallable``.
38
-
39
- Resolution ladder:
40
- 1. exact ``(abs_path, start_line)`` — the precise join;
41
- 2. on miss, candidates sharing ``(abs_path, short_name)``: a single
42
- candidate is taken directly; otherwise prefer those whose
43
- parameter count equals the CodeQL positional arity, then the
44
- nearest ``start_line``;
45
- 3. no name match -> ``None`` (caller row skipped / callee becomes
46
- a ghost node).
47
-
48
- Step 2 recovers edges the ``(file, line)`` join silently drops when
49
- CodeQL and Jedi disagree on a definition's start line (e.g. decorator
50
- handling). Jedi's ``parameters`` counts every declared slot (incl.
51
- ``*args``/``**kwargs``/keyword-only) whereas CodeQL's arity is
52
- positional only, so the arity filter is exact for plain signatures
53
- and otherwise yields to the nearest-line tiebreak.
54
- """
55
-
56
- def __init__(self) -> None:
57
- self._by_loc: Dict[Tuple[str, int], Any] = {}
58
- self._by_name: Dict[Tuple[str, str], List[Any]] = {}
59
-
60
- @staticmethod
61
- def _abs(path: str) -> str:
62
- try:
63
- return str(Path(path).resolve())
64
- except (OSError, RuntimeError):
65
- return path
66
-
67
- @classmethod
68
- def from_symbol_table(
69
- cls, symbol_table: Dict[str, PyModule]
70
- ) -> "_CallableResolver":
71
- resolver = cls()
72
- for c in iter_callables_in_symbol_table(symbol_table):
73
- abs_path = cls._abs(c.path)
74
- resolver._by_loc[(abs_path, c.start_line)] = c
75
- resolver._by_name.setdefault((abs_path, c.name), []).append(c)
76
- return resolver
77
-
78
- def resolve(
79
- self, file: str, start_line: int, name: str, arity: int
80
- ) -> Any:
81
- exact = self._by_loc.get((file, start_line))
82
- if exact is not None:
83
- return exact
84
- if not name:
85
- return None
86
- candidates = self._by_name.get((file, name))
87
- if not candidates:
88
- return None
89
- if len(candidates) == 1:
90
- return candidates[0]
91
- arity_matched = [c for c in candidates if len(c.parameters) == arity]
92
- pool = arity_matched or candidates
93
- return min(pool, key=lambda c: abs(c.start_line - start_line))
94
-
95
-
96
- class CodeQL:
97
- """A class for building the application view of a Python application using CodeQL.
98
-
99
- Args:
100
- project_dir (str or Path): The path to the root of the Python project.
101
-
102
- Attributes:
103
- db_path (Path): The path to the CodeQL database.
104
- temp_db (TemporaryDirectory or None): The temporary directory object if a temporary database was created.
105
- """
106
-
107
- def __init__(
108
- self,
109
- project_dir: Union[str, Path],
110
- db_path: Path,
111
- codeql_bin: Union[str, Path, None] = None,
112
- codeql_packs_dir: Union[str, Path, None] = None,
113
- ) -> None:
114
- self.project_dir = project_dir
115
- self.db_path = db_path
116
- self.codeql_bin = codeql_bin
117
- self.codeql_packs_dir = codeql_packs_dir
118
- self._cached_df: "DataFrame | None" = None
119
-
120
- def _query_call_edges(self) -> DataFrame:
121
- """Runs the CodeQL query that emits one row per resolved call site.
122
-
123
- The query is written against CodeQL's Python library (``import python``).
124
- It returns physical location handles for both endpoints so the
125
- downstream post-processor can join into Jedi's existing
126
- ``PyCallable.signature`` space via ``(file_path, start_line)`` —
127
- no signature normalization required.
128
-
129
- Filters:
130
- * Caller must be a ``Function`` (skip module-level / class-body
131
- calls — they have no ``PyCallable`` to anchor to).
132
- * Callee may resolve to anything (in-source or library stub);
133
- non-application callees become **ghost** nodes downstream so
134
- RPC / third-party / framework edges are preserved.
135
-
136
- Returns:
137
- DataFrame: one row per resolved (caller, callee, call-site)
138
- triple. Duplicate ``(caller_file, caller_start_line,
139
- callee_file, callee_start_line)`` tuples represent multiple
140
- call sites in the same caller targeting the same callee and
141
- are coalesced into a single ``PyCallEdge`` (weight = count)
142
- by the post-processor.
143
- """
144
- query = [
145
- "/**",
146
- " * @name Python call-graph edges",
147
- " * @description One row per resolved call site: caller, callee,",
148
- " * and the call-expression location.",
149
- " * @kind table",
150
- " * @id py/codeanalyzer/call-graph-edges",
151
- " */",
152
- "import python",
153
- # ``FunctionValue`` / ``ClassValue`` / the ``pointsTo`` predicate
154
- # live in ObjectAPI, which ``import python`` only brings in as a
155
- # private import — they aren't re-exported. Pull them in
156
- # explicitly.
157
- "import semmle.python.objects.ObjectAPI",
158
- "",
159
- # ``Value.getACall()`` is the modern call-resolution API in
160
- # codeql/python-all 7.x — it returns the ``CallNode`` (CFG)
161
- # whose target was resolved to that ``Value``. Cleaner than
162
- # poking at ``pointsTo`` directly.
163
- # ``callee`` is bound to the FunctionValue's scope so the
164
- # endpoint emits the same Function-level facts (name, arity,
165
- # location) the post-processor needs for the name+arity
166
- # fallback when the (file, start_line) join misses.
167
- "from CallNode call, Function caller, FunctionValue calleeVal, Function callee",
168
- "where",
169
- " call.getScope() = caller and",
170
- " callee = calleeVal.getScope() and",
171
- " (",
172
- # Direct function / bound-method call: foo() or obj.foo()
173
- " call = calleeVal.getACall()",
174
- " or",
175
- # Constructor call: A(...) resolves to a ClassValue; the actual
176
- # callee is the class's __init__ (via MRO lookup so subclasses
177
- # without an explicit __init__ still resolve to the inherited one).
178
- " exists(ClassValue clsVal |",
179
- " call = clsVal.getACall() and",
180
- ' clsVal.lookup("__init__") = calleeVal',
181
- " )",
182
- " )",
183
- "select",
184
- # --- Caller endpoint --- (joins to PyCallable: exact by
185
- # (file, start_line), else by (file, name) + arity)
186
- " caller.getLocation().getFile().getAbsolutePath(),",
187
- " caller.getLocation().getStartLine(),",
188
- " caller.getQualifiedName(),",
189
- " caller.getName(),",
190
- " count(caller.getArg(_)),",
191
- # --- Callee endpoint --- (file/line may live in a library stub;
192
- # post-processor classifies as in-source or ghost)
193
- " callee.getLocation().getFile().getAbsolutePath(),",
194
- " callee.getLocation().getStartLine(),",
195
- " calleeVal.getQualifiedName(),",
196
- " callee.getName(),",
197
- " count(callee.getArg(_)),",
198
- # --- Call-site location --- (for PyCallsite augmentation)
199
- " call.getLocation().getStartLine(),",
200
- " call.getLocation().getStartColumn(),",
201
- " call.getLocation().getEndLine(),",
202
- " call.getLocation().getEndColumn()",
203
- # ``is_constructor`` is derived in the post-processor by
204
- # checking whether ``callee_qname`` ends in ``.__init__``;
205
- # avoids QL's restrictive ``if-then-else`` typing here.
206
- ]
207
- if self._cached_df is not None:
208
- return self._cached_df
209
-
210
- query_string = "\n".join(query)
211
-
212
- with CodeQLQueryRunner(
213
- self.db_path,
214
- codeql_bin=self.codeql_bin,
215
- codeql_packs_dir=self.codeql_packs_dir,
216
- ) as runner:
217
- df: DataFrame = runner.execute(
218
- query_string,
219
- column_names=[
220
- "caller_file",
221
- "caller_start_line",
222
- "caller_qname",
223
- "caller_name",
224
- "caller_arity",
225
- "callee_file",
226
- "callee_start_line",
227
- "callee_qname",
228
- "callee_name",
229
- "callee_arity",
230
- "call_start_line",
231
- "call_start_column",
232
- "call_end_line",
233
- "call_end_column",
234
- ],
235
- )
236
- self._cached_df = df
237
- return df
238
-
239
- @staticmethod
240
- def _build_callable_resolver(
241
- symbol_table: Dict[str, PyModule],
242
- ) -> _CallableResolver:
243
- """Build the endpoint -> ``PyCallable`` resolver from Jedi.
244
-
245
- Paths are resolved so they match CodeQL's ``getAbsolutePath()``
246
- regardless of symlinks or the current working directory.
247
- """
248
- return _CallableResolver.from_symbol_table(symbol_table)
249
-
250
- def _iter_resolved_rows(
251
- self, symbol_table: Dict[str, PyModule]
252
- ) -> "Iterator[Tuple[str, str, Any]]":
253
- """Yield ``(source_sig, target_sig, row)`` for every CodeQL row.
254
-
255
- Rows whose caller can't be matched to a ``PyCallable`` in the
256
- symbol table are skipped. Callee misses fall back to
257
- ``row.callee_qname`` (ghost). Used by both edge construction and
258
- call-site augmentation so a single CodeQL query feeds both.
259
- """
260
- df = self._query_call_edges()
261
- if df.empty:
262
- return
263
- resolver = self._build_callable_resolver(symbol_table)
264
-
265
- skipped_unknown_caller = 0
266
- ghost_callees = 0
267
- for row in df.itertuples(index=False):
268
- caller = resolver.resolve(
269
- row.caller_file,
270
- int(row.caller_start_line),
271
- row.caller_name,
272
- int(row.caller_arity),
273
- )
274
- if caller is None:
275
- skipped_unknown_caller += 1
276
- continue
277
-
278
- callee = resolver.resolve(
279
- row.callee_file,
280
- int(row.callee_start_line),
281
- row.callee_name,
282
- int(row.callee_arity),
283
- )
284
- if callee is not None:
285
- target_sig = callee.signature
286
- else:
287
- target_sig = row.callee_qname
288
- ghost_callees += 1
289
-
290
- yield caller.signature, target_sig, row
291
-
292
- if skipped_unknown_caller:
293
- logger.debug(
294
- f"CodeQL: skipped {skipped_unknown_caller} rows whose caller "
295
- f"was not in Jedi's symbol table."
296
- )
297
- if ghost_callees:
298
- logger.debug(
299
- f"CodeQL: {ghost_callees} rows resolved to ghost (external) callees."
300
- )
301
-
302
- def build_call_graph_edges(
303
- self, symbol_table: Dict[str, PyModule]
304
- ) -> List[PyCallEdge]:
305
- """Run the CodeQL query and turn each row into a ``PyCallEdge``.
306
-
307
- Edges are coalesced on ``(source, target)`` — ``weight`` is the
308
- number of distinct call sites in the caller targeting the callee.
309
- Provenance is always ``["codeql"]``; combine with Jedi-derived
310
- edges via ``call_graph.merge_edges``.
311
- """
312
- edge_counts: Counter = Counter()
313
- for source_sig, target_sig, _row in self._iter_resolved_rows(symbol_table):
314
- edge_counts[(source_sig, target_sig)] += 1
315
-
316
- return [
317
- PyCallEdge(
318
- source=src,
319
- target=dst,
320
- weight=count,
321
- provenance=["codeql"],
322
- )
323
- for (src, dst), count in edge_counts.items()
324
- ]
325
-
326
- def augment_call_sites(self, symbol_table: Dict[str, PyModule]) -> int:
327
- """Backfill ``PyCallsite.callee_signature`` using CodeQL resolution.
328
-
329
- Walks every CodeQL row, locates the matching ``PyCallsite`` inside
330
- the caller's ``PyCallable.call_sites`` by call-expression line range
331
- (``start_line``, ``end_line``), and fills in ``callee_signature``
332
- **only when Jedi left it empty**. Existing Jedi-resolved signatures
333
- are kept (Jedi sees lexical context CodeQL can't, e.g. closures).
334
-
335
- Match is by line range — column matching is brittle across the two
336
- tools' 0- vs 1-based conventions. Ambiguity on a single line
337
- (e.g. ``a.b().c()``) resolves to the first matching site, which is
338
- an acceptable approximation given how rarely Jedi misses callees
339
- on chained call lines.
340
-
341
- Returns:
342
- Number of ``PyCallsite`` entries augmented.
343
- """
344
- resolver = self._build_callable_resolver(symbol_table)
345
- df = self._query_call_edges()
346
- if df.empty:
347
- return 0
348
-
349
- augmented = 0
350
- for row in df.itertuples(index=False):
351
- caller = resolver.resolve(
352
- row.caller_file,
353
- int(row.caller_start_line),
354
- row.caller_name,
355
- int(row.caller_arity),
356
- )
357
- if caller is None:
358
- continue
359
-
360
- callee = resolver.resolve(
361
- row.callee_file,
362
- int(row.callee_start_line),
363
- row.callee_name,
364
- int(row.callee_arity),
365
- )
366
- resolved_sig = callee.signature if callee is not None else row.callee_qname
367
-
368
- call_start = int(row.call_start_line)
369
- call_end = int(row.call_end_line)
370
- for site in caller.call_sites:
371
- if site.start_line != call_start or site.end_line != call_end:
372
- continue
373
- if not site.callee_signature:
374
- site.callee_signature = resolved_sig
375
- augmented += 1
376
- break
377
-
378
- if augmented:
379
- logger.debug(
380
- f"CodeQL: augmented {augmented} PyCallsite.callee_signature entries."
381
- )
382
- return augmented
@@ -1,12 +0,0 @@
1
- class CodeQLExceptions:
2
- class CodeQLDatabaseBuildException(Exception):
3
- """Exception raised when there is an error building the CodeQL database."""
4
-
5
- def __init__(self, message: str) -> None:
6
- super().__init__(message)
7
-
8
- class CodeQLQueryExecutionException(Exception):
9
- """Exception raised when there is an error building the CodeQL database."""
10
-
11
- def __init__(self, message: str) -> None:
12
- super().__init__(message)
@@ -1,91 +0,0 @@
1
- import os
2
- import platform
3
- import stat
4
- import zipfile
5
- from pathlib import Path
6
-
7
- import requests
8
- from codeanalyzer.utils import logger
9
-
10
-
11
- class CodeQLLoader:
12
- @classmethod
13
- def detect_platform_key(cls) -> str:
14
- system = platform.system()
15
- arch = platform.machine().lower()
16
-
17
- if system == "Linux" and arch in {"x86_64", "amd64"}:
18
- return "codeql-linux64.zip"
19
- elif system == "Darwin" and arch in {"x86_64", "arm64"}:
20
- return "codeql-osx64.zip"
21
- elif system == "Windows" and arch in {"x86_64", "amd64"}:
22
- return "codeql-win64.zip"
23
- else:
24
- return "codeql.zip" # fallback to generic binary if needed
25
-
26
- @classmethod
27
- def get_codeql_download_url(cls, expected_filename: str) -> str:
28
- response = requests.get(
29
- "https://api.github.com/repos/github/codeql-cli-binaries/releases/latest"
30
- )
31
- response.raise_for_status()
32
- for asset in response.json()["assets"]:
33
- if asset["name"] == expected_filename:
34
- return asset["browser_download_url"]
35
- raise RuntimeError(f"No asset found for filename: {expected_filename}")
36
-
37
- @classmethod
38
- def download_and_extract_codeql(cls, temp_dir: Path) -> Path:
39
- filename = cls.detect_platform_key()
40
- download_url = cls.get_codeql_download_url(filename)
41
-
42
- temp_dir.mkdir(parents=True, exist_ok=True)
43
- archive_path = temp_dir / filename
44
-
45
- logger.info(f"Downloading CodeQL CLI from {download_url}")
46
- with requests.get(download_url, stream=True) as r:
47
- r.raise_for_status()
48
- block_size = 8192 # 8KB
49
-
50
- with open(archive_path, "wb") as f:
51
- for chunk in r.iter_content(chunk_size=block_size):
52
- f.write(chunk)
53
-
54
- extract_dir = temp_dir / filename.replace(".zip", "")
55
- extract_dir.mkdir(exist_ok=True)
56
-
57
- logger.info(f"Extracting CodeQL CLI to {extract_dir}")
58
- # zipfile.extractall drops Unix permissions (the executable bit), so
59
- # we extract entries manually and copy each one's stored mode onto
60
- # the file system. Without this, the CodeQL launcher script can't
61
- # be executed and the next subprocess.Popen raises PermissionError.
62
- with zipfile.ZipFile(archive_path, "r") as zip_ref:
63
- for info in zip_ref.infolist():
64
- extracted_path = zip_ref.extract(info, extract_dir)
65
- stored_mode = info.external_attr >> 16
66
- if stored_mode:
67
- os.chmod(extracted_path, stored_mode)
68
-
69
- # Archive is no longer needed once extracted.
70
- try:
71
- archive_path.unlink()
72
- except OSError as exc:
73
- logger.warning(f"Could not remove CodeQL archive {archive_path}: {exc}")
74
-
75
- # rglob("codeql") returns both the launcher file *and* an internal
76
- # directory of the same name (CodeQL ships its own runtime under
77
- # ``codeql/codeql/``); insist on a regular file so we never bind to
78
- # the directory.
79
- codeql_bin = next(
80
- (p for p in extract_dir.rglob("codeql") if p.is_file()),
81
- None,
82
- )
83
- if not codeql_bin:
84
- raise FileNotFoundError("CodeQL binary not found in extracted contents.")
85
-
86
- # Belt-and-suspenders: ensure the binary is executable even if the
87
- # archive entry's mode was zero (some older zip producers omit it).
88
- st = codeql_bin.stat()
89
- codeql_bin.chmod(st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
90
-
91
- return codeql_bin.resolve()
@@ -1,185 +0,0 @@
1
- ################################################################################
2
- # Copyright IBM Corporation 2025
3
- #
4
- # Licensed under the Apache License, Version 2.0 (the "License");
5
- # you may not use this file except in compliance with the License.
6
- # You may obtain a copy of the License at
7
- #
8
- # http://www.apache.org/licenses/LICENSE-2.0
9
- #
10
- # Unless required by applicable law or agreed to in writing, software
11
- # distributed under the License is distributed on an "AS IS" BASIS,
12
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
- # See the License for the specific language governing permissions and
14
- # limitations under the License.
15
- ################################################################################
16
-
17
- """Backend module for CodeQL query execution.
18
-
19
- This module provides functionality to run CodeQL queries against CodeQL databases
20
- and process the results.
21
- """
22
-
23
- import shlex
24
- import subprocess
25
- import tempfile
26
- from pathlib import Path
27
- from typing import List
28
-
29
- import pandas as pd
30
- from pandas import DataFrame
31
-
32
- from codeanalyzer.semantic_analysis.codeql.codeql_exceptions import CodeQLExceptions
33
-
34
-
35
- class CodeQLQueryRunner:
36
- """A class for executing CodeQL queries against a CodeQL database.
37
-
38
- This class provides a context manager interface for executing CodeQL queries
39
- and handling temporary resources needed during query execution.
40
-
41
- Args:
42
- database_path (str): The path to the CodeQL database.
43
- codeql_bin (str | Path | None): Absolute path to the CodeQL CLI
44
- binary. When ``None``, falls back to whatever ``codeql`` is on
45
- ``PATH``.
46
-
47
- Attributes:
48
- database_path (Path): The path to the CodeQL database.
49
- codeql_bin (str): Resolved binary path or the literal ``"codeql"``.
50
- temp_file_path (Path): The path to the temporary query file.
51
- csv_output_file (Path): The path to the CSV output file.
52
- temp_bqrs_file_path (Path): The path to the temporary bqrs file.
53
- temp_qlpack_file (Path): The path to the temporary qlpack file.
54
-
55
- Raises:
56
- CodeQLQueryExecutionException: If there is an error executing the query.
57
- """
58
-
59
- def __init__(self, database_path: str, codeql_bin=None, codeql_packs_dir=None):
60
- self.database_path: Path = Path(database_path)
61
- self.codeql_bin: str = str(codeql_bin) if codeql_bin else "codeql"
62
- self.codeql_packs_dir = (
63
- Path(codeql_packs_dir) if codeql_packs_dir is not None else None
64
- )
65
- self.temp_file_path: Path = None
66
-
67
- def __enter__(self):
68
- """Context entry that prepares paths to execute a CodeQL query.
69
-
70
- The ``.ql`` file is written **inside the prepared qlpack
71
- directory** (``codeql_packs_dir``) so ``import python`` resolves
72
- against that pack's installed dependencies — no
73
- ``--additional-packs`` or ``--search-path`` needed. The CSV /
74
- BQRS output files live in ``tempfile`` because they're transient
75
- per-query artifacts.
76
- """
77
- # CSV and BQRS files are transient per-query — fine in /tmp.
78
- csv_file = tempfile.NamedTemporaryFile("w", delete=False, suffix=".csv")
79
- bqrs_file = tempfile.NamedTemporaryFile("w", delete=False, suffix=".bqrs")
80
- self.csv_output_file = Path(csv_file.name)
81
- self.temp_bqrs_file_path = Path(bqrs_file.name)
82
- csv_file.close()
83
- bqrs_file.close()
84
-
85
- # The .ql file MUST live inside the prepared qlpack so its
86
- # ``import python`` resolves via that pack's lock file. Writing
87
- # outside the pack means CodeQL falls back to a default
88
- # search-path that doesn't include downloaded library packs.
89
- if self.codeql_packs_dir is None:
90
- raise RuntimeError(
91
- "CodeQLQueryRunner requires codeql_packs_dir — the directory "
92
- "of an installed qlpack that depends on codeql/python-all."
93
- )
94
- ql_file = tempfile.NamedTemporaryFile(
95
- "w", delete=False, suffix=".ql", dir=str(self.codeql_packs_dir)
96
- )
97
- self.temp_file_path = Path(ql_file.name)
98
- ql_file.close()
99
-
100
- return self
101
-
102
- def execute(self, query_string: str, column_names: List[str]) -> DataFrame:
103
- """Writes the query to the temporary file and executes it against the specified CodeQL database.
104
-
105
- Args:
106
- query_string (str): The CodeQL query string to be executed.
107
- column_names (List[str]): The list of column names for the CSV the CodeQL produces when we execute the query.
108
-
109
- Returns:
110
- dict: A dictionary containing the resulting DataFrame.
111
-
112
- Raises:
113
- RuntimeError: If the context manager is not entered using the 'with' statement.
114
- CodeQLQueryExecutionException: If there is an error executing the query.
115
- """
116
- if not self.temp_file_path:
117
- raise RuntimeError("CodeQLQueryRunner not entered using 'with' statement.")
118
-
119
- # Write the query to the temp file so we can execute it.
120
- self.temp_file_path.write_text(query_string)
121
-
122
- # The .ql file sits inside the qlpack directory whose lock file
123
- # already resolves ``codeql/python-all`` and its transitive
124
- # dependencies. ``codeql query run`` auto-discovers the enclosing
125
- # qlpack — no extra flags required.
126
- codeql_query_cmd = shlex.split(
127
- f"{shlex.quote(self.codeql_bin)} query run {self.temp_file_path} "
128
- f"--database={self.database_path} "
129
- f"--output={self.temp_bqrs_file_path}",
130
- posix=False,
131
- )
132
-
133
- call = subprocess.Popen(
134
- codeql_query_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE
135
- )
136
- _, err = call.communicate()
137
- if call.returncode != 0:
138
- raise CodeQLExceptions.CodeQLQueryExecutionException(
139
- f"Error executing query: {(err or b'').decode(errors='replace')}"
140
- )
141
-
142
- # Convert the bqrs file to a CSV file
143
- bqrs2csv_command = shlex.split(
144
- f"{shlex.quote(self.codeql_bin)} bqrs decode --format=csv --output={self.csv_output_file} {self.temp_bqrs_file_path}",
145
- posix=False,
146
- )
147
-
148
- # Read the CSV file content and cast it to a DataFrame
149
-
150
- call = subprocess.Popen(
151
- bqrs2csv_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE
152
- )
153
- _, err = call.communicate()
154
- if call.returncode != 0:
155
- raise CodeQLExceptions.CodeQLQueryExecutionException(
156
- f"Error decoding bqrs: {(err or b'').decode(errors='replace')}"
157
- )
158
- else:
159
- return pd.read_csv(
160
- self.csv_output_file,
161
- header=None,
162
- names=column_names,
163
- skiprows=[0],
164
- )
165
-
166
- def __exit__(self, exc_type, exc_val, exc_tb):
167
- """Clean up resources used by the CodeQL analysis.
168
-
169
- Args:
170
- exc_type: The exception type if an exception was raised in the context, otherwise None.
171
- exc_val: The exception instance if an exception was raised in the context, otherwise None.
172
- exc_tb: The traceback if an exception was raised in the context, otherwise None.
173
-
174
- Note:
175
- Deletes the temporary files created during the analysis, including the temporary file path,
176
- the CSV output file, and the temporary QL pack file.
177
- """
178
- if self.temp_file_path and self.temp_file_path.exists():
179
- self.temp_file_path.unlink()
180
-
181
- if self.csv_output_file and self.csv_output_file.exists():
182
- self.csv_output_file.unlink()
183
-
184
- if self.temp_bqrs_file_path and self.temp_bqrs_file_path.exists():
185
- self.temp_bqrs_file_path.unlink()