cg-code-graph 0.10.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cg_code_graph-0.10.1.dist-info/METADATA +678 -0
- cg_code_graph-0.10.1.dist-info/RECORD +174 -0
- cg_code_graph-0.10.1.dist-info/WHEEL +5 -0
- cg_code_graph-0.10.1.dist-info/entry_points.txt +3 -0
- cg_code_graph-0.10.1.dist-info/licenses/LICENSE +21 -0
- cg_code_graph-0.10.1.dist-info/top_level.txt +1 -0
- codegraph/__init__.py +2 -0
- codegraph/aitools.py +129 -0
- codegraph/apps.py +76 -0
- codegraph/blindspots.py +428 -0
- codegraph/bridges.py +1701 -0
- codegraph/cli.py +725 -0
- codegraph/concepts.py +362 -0
- codegraph/config.py +559 -0
- codegraph/core/__init__.py +0 -0
- codegraph/core/cache.py +375 -0
- codegraph/core/detect.py +80 -0
- codegraph/core/extractors.py +187 -0
- codegraph/core/fsutil.py +61 -0
- codegraph/core/generated.py +575 -0
- codegraph/core/model.py +174 -0
- codegraph/core/paths.py +175 -0
- codegraph/core/plugin.py +160 -0
- codegraph/core/store.py +80 -0
- codegraph/core/syntax_errors.py +132 -0
- codegraph/coverage.py +928 -0
- codegraph/doctor.py +453 -0
- codegraph/external.py +613 -0
- codegraph/indexer.py +336 -0
- codegraph/link.py +434 -0
- codegraph/lint_async.py +524 -0
- codegraph/mcp_server.py +1303 -0
- codegraph/parity.py +473 -0
- codegraph/parity_structure.py +307 -0
- codegraph/payload.py +321 -0
- codegraph/plans.py +1285 -0
- codegraph/platform_scan.py +643 -0
- codegraph/platforms.py +1369 -0
- codegraph/plugins/__init__.py +0 -0
- codegraph/plugins/cfamily/__init__.py +0 -0
- codegraph/plugins/cfamily/plugin.py +930 -0
- codegraph/plugins/cfamily/syntax.py +881 -0
- codegraph/plugins/dart/__init__.py +0 -0
- codegraph/plugins/dart/bridges.py +345 -0
- codegraph/plugins/dart/extractor/bin/extract.dart +717 -0
- codegraph/plugins/dart/extractor/pubspec.lock +149 -0
- codegraph/plugins/dart/extractor/pubspec.yaml +7 -0
- codegraph/plugins/dart/http.py +904 -0
- codegraph/plugins/dart/models.py +308 -0
- codegraph/plugins/dart/plugin.py +625 -0
- codegraph/plugins/dart/program.py +907 -0
- codegraph/plugins/django/__init__.py +0 -0
- codegraph/plugins/django/extras.py +378 -0
- codegraph/plugins/django/models.py +508 -0
- codegraph/plugins/django/plugin.py +728 -0
- codegraph/plugins/django/schemas.py +339 -0
- codegraph/plugins/django/shapes.py +216 -0
- codegraph/plugins/django/urls.py +603 -0
- codegraph/plugins/express/__init__.py +0 -0
- codegraph/plugins/express/plugin.py +428 -0
- codegraph/plugins/flutter/__init__.py +0 -0
- codegraph/plugins/flutter/plugin.py +538 -0
- codegraph/plugins/kotlin/__init__.py +0 -0
- codegraph/plugins/kotlin/exact.py +457 -0
- codegraph/plugins/kotlin/plugin.py +1961 -0
- codegraph/plugins/kotlin/reparse.py +234 -0
- codegraph/plugins/laravel/__init__.py +0 -0
- codegraph/plugins/laravel/broadcast.py +351 -0
- codegraph/plugins/laravel/plugin.py +863 -0
- codegraph/plugins/laravel/tests.py +262 -0
- codegraph/plugins/laravel/values.py +728 -0
- codegraph/plugins/native/__init__.py +0 -0
- codegraph/plugins/native/gates.py +286 -0
- codegraph/plugins/native/runner.py +183 -0
- codegraph/plugins/native/scipread.py +194 -0
- codegraph/plugins/native/ts.py +54 -0
- codegraph/plugins/nest/__init__.py +0 -0
- codegraph/plugins/nest/plugin.py +654 -0
- codegraph/plugins/nextjs/__init__.py +0 -0
- codegraph/plugins/nextjs/plugin.py +336 -0
- codegraph/plugins/nuxt/__init__.py +0 -0
- codegraph/plugins/nuxt/plugin.py +308 -0
- codegraph/plugins/php/__init__.py +0 -0
- codegraph/plugins/php/extractor/composer.json +5 -0
- codegraph/plugins/php/extractor/composer.lock +76 -0
- codegraph/plugins/php/extractor/extract.php +743 -0
- codegraph/plugins/php/gating.py +573 -0
- codegraph/plugins/php/plugin.py +668 -0
- codegraph/plugins/php/strings.py +197 -0
- codegraph/plugins/python/__init__.py +0 -0
- codegraph/plugins/python/aitools.py +664 -0
- codegraph/plugins/python/external.py +245 -0
- codegraph/plugins/python/fields.py +107 -0
- codegraph/plugins/python/plugin.py +1733 -0
- codegraph/plugins/python/refs.py +485 -0
- codegraph/plugins/python/roots.py +412 -0
- codegraph/plugins/python/socketio.py +210 -0
- codegraph/plugins/python/subproc.py +864 -0
- codegraph/plugins/python/tests.py +1040 -0
- codegraph/plugins/python/values.py +179 -0
- codegraph/plugins/pyweb/__init__.py +0 -0
- codegraph/plugins/pyweb/plugin.py +1334 -0
- codegraph/plugins/pyweb/values.py +68 -0
- codegraph/plugins/rust/__init__.py +0 -0
- codegraph/plugins/rust/cargo.py +226 -0
- codegraph/plugins/rust/plugin.py +980 -0
- codegraph/plugins/rust/syntax.py +678 -0
- codegraph/plugins/scip/__init__.py +0 -0
- codegraph/plugins/scip/importer.py +129 -0
- codegraph/plugins/scip/scip.proto +962 -0
- codegraph/plugins/scip/scip_pb2.py +97 -0
- codegraph/plugins/stubs/__init__.py +0 -0
- codegraph/plugins/stubs/plugins.py +38 -0
- codegraph/plugins/swift/__init__.py +0 -0
- codegraph/plugins/swift/baseurl.py +109 -0
- codegraph/plugins/swift/exact.py +415 -0
- codegraph/plugins/swift/indexstore.py +209 -0
- codegraph/plugins/swift/packages.py +174 -0
- codegraph/plugins/swift/plugin.py +2890 -0
- codegraph/plugins/ts/__init__.py +0 -0
- codegraph/plugins/ts/baseurl.py +185 -0
- codegraph/plugins/ts/extractor/extract.mjs +2652 -0
- codegraph/plugins/ts/extractor/fw.mjs +685 -0
- codegraph/plugins/ts/extractor/package-lock.json +205 -0
- codegraph/plugins/ts/extractor/package.json +9 -0
- codegraph/plugins/ts/plugin.py +480 -0
- codegraph/plugins/tsweb/__init__.py +0 -0
- codegraph/plugins/tsweb/common.py +290 -0
- codegraph/plugins/tsweb/data.py +276 -0
- codegraph/presets/__init__.py +146 -0
- codegraph/presets/c_cpp.yaml +9 -0
- codegraph/presets/common.yaml +66 -0
- codegraph/presets/dart.yaml +9 -0
- codegraph/presets/django-ninja.yaml +15 -0
- codegraph/presets/django.yaml +25 -0
- codegraph/presets/djangorestframework.yaml +17 -0
- codegraph/presets/express.yaml +17 -0
- codegraph/presets/kotlin.yaml +11 -0
- codegraph/presets/laravel.yaml +40 -0
- codegraph/presets/nest.yaml +11 -0
- codegraph/presets/nextjs.yaml +15 -0
- codegraph/presets/nuxt.yaml +9 -0
- codegraph/presets/php.yaml +5 -0
- codegraph/presets/python.yaml +10 -0
- codegraph/presets/rust.yaml +5 -0
- codegraph/presets/swift.yaml +10 -0
- codegraph/presets/typescript.yaml +13 -0
- codegraph/process_runs.py +328 -0
- codegraph/protocols/__init__.py +299 -0
- codegraph/protocols/builtin.py +67 -0
- codegraph/protocols/matchers.py +144 -0
- codegraph/protocols/view.py +334 -0
- codegraph/query.py +2089 -0
- codegraph/realtime.py +260 -0
- codegraph/roundtrip.py +346 -0
- codegraph/routes.py +442 -0
- codegraph/starters.py +218 -0
- codegraph/tests_index.py +117 -0
- codegraph/viz/__init__.py +0 -0
- codegraph/viz/graph.py +369 -0
- codegraph/viz/server.py +198 -0
- codegraph/viz/static/app.css +148 -0
- codegraph/viz/static/app.js +1082 -0
- codegraph/viz/static/index.html +81 -0
- codegraph/viz/static/layered.js +237 -0
- codegraph/viz/static/vendor/VERSIONS.txt +4 -0
- codegraph/viz/static/vendor/cose-base.js +3214 -0
- codegraph/viz/static/vendor/cytoscape-fcose.js +1549 -0
- codegraph/viz/static/vendor/cytoscape.min.js +31 -0
- codegraph/viz/static/vendor/layout-base.js +5230 -0
- codegraph/viz/tools/package-lock.json +303 -0
- codegraph/viz/tools/package.json +7 -0
- codegraph/viz/tools/shoot.mjs +165 -0
- codegraph/xcode.py +251 -0
|
File without changes
|
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
"""Compile-time gates for systems code: Rust `#[cfg(...)]` predicates and C/C++ preprocessor conditions.
|
|
2
|
+
|
|
3
|
+
Two uses:
|
|
4
|
+
1. Always: every item / statement under a condition gets GATED_BY edges to `feature:<pkg>/<name>`, `cfg:<pred>`
|
|
5
|
+
(Rust) or `define:<MACRO>` (C/C++) nodes, with the attribute / directive line as evidence.
|
|
6
|
+
2. With a gate scenario (`index --gates FILE`, first scenario), conditions are evaluated three-valued
|
|
7
|
+
(true / false / unknown) and references made inside a region whose condition is *false* get edge.gate set,
|
|
8
|
+
exactly like the PHP feature-flag gating, so `reaches` lists them under GATED.
|
|
9
|
+
|
|
10
|
+
Scenario keys understood here (others are ignored, so one gates file can serve several languages):
|
|
11
|
+
features_on / features_off Cargo features ("fs" for any package, or "kv-core/fs")
|
|
12
|
+
cargo_features "default" | "none" | [list]: exactly these features (per package, plus what they
|
|
13
|
+
enable) are on; every other feature is off
|
|
14
|
+
cfg_true / cfg_false other cfg atoms: "unix", "windows", 'target_os="linux"', "test", "debug_assertions"
|
|
15
|
+
defines_on / defines_off C/C++ macros ("NAME" or "NAME=VALUE"); defines_on = defined, defines_off = undefined
|
|
16
|
+
"""
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import re
|
|
20
|
+
|
|
21
|
+
TRUE, FALSE, UNKNOWN = True, False, None
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
# ---------------------------------------------------------------- Rust cfg predicates
|
|
25
|
+
def parse_cfg(s: str):
|
|
26
|
+
"""'all(unix, not(feature = "x"))' -> ('all', [('atom','unix',None), ('not', ('atom','feature','x'))])."""
|
|
27
|
+
toks = re.findall(r'"(?:[^"\\]|\\.)*"|[A-Za-z_][\w:]*|[(),=]', s)
|
|
28
|
+
pos = [0]
|
|
29
|
+
|
|
30
|
+
def peek():
|
|
31
|
+
return toks[pos[0]] if pos[0] < len(toks) else None
|
|
32
|
+
|
|
33
|
+
def take():
|
|
34
|
+
t = peek()
|
|
35
|
+
pos[0] += 1
|
|
36
|
+
return t
|
|
37
|
+
|
|
38
|
+
def expr():
|
|
39
|
+
t = take()
|
|
40
|
+
if t is None:
|
|
41
|
+
return ("atom", "?", None)
|
|
42
|
+
if t in ("all", "any", "not") and peek() == "(":
|
|
43
|
+
take()
|
|
44
|
+
args = []
|
|
45
|
+
while peek() not in (")", None):
|
|
46
|
+
args.append(expr())
|
|
47
|
+
if peek() == ",":
|
|
48
|
+
take()
|
|
49
|
+
take()
|
|
50
|
+
if t == "not":
|
|
51
|
+
return ("not", args[0] if args else ("atom", "?", None))
|
|
52
|
+
return (t, args)
|
|
53
|
+
if peek() == "=":
|
|
54
|
+
take()
|
|
55
|
+
v = take() or '""'
|
|
56
|
+
return ("atom", t, v.strip('"'))
|
|
57
|
+
return ("atom", t, None)
|
|
58
|
+
|
|
59
|
+
try:
|
|
60
|
+
return expr()
|
|
61
|
+
except Exception:
|
|
62
|
+
return ("atom", s.strip(), None)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def cfg_atoms(p) -> list[tuple[str, str | None]]:
|
|
66
|
+
if p[0] == "atom":
|
|
67
|
+
return [(p[1], p[2])]
|
|
68
|
+
if p[0] == "not":
|
|
69
|
+
return cfg_atoms(p[1])
|
|
70
|
+
return [a for x in p[1] for a in cfg_atoms(x)]
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def atom_text(key: str, val: str | None) -> str:
|
|
74
|
+
return key if val is None else f'{key}="{val}"'
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class Scenario:
|
|
78
|
+
"""First gate scenario of the gates file, seen from native code."""
|
|
79
|
+
|
|
80
|
+
def __init__(self, sc: dict | None, packages: dict[str, dict] | None = None):
|
|
81
|
+
sc = sc or {}
|
|
82
|
+
self.name = sc.get("name")
|
|
83
|
+
self.raw = sc
|
|
84
|
+
self.features_on = set(sc.get("features_on") or [])
|
|
85
|
+
self.features_off = set(sc.get("features_off") or [])
|
|
86
|
+
self.cfg_true = set(sc.get("cfg_true") or [])
|
|
87
|
+
self.cfg_false = set(sc.get("cfg_false") or [])
|
|
88
|
+
self.defines = {}
|
|
89
|
+
for d in sc.get("defines_on") or []:
|
|
90
|
+
k, _, v = str(d).partition("=")
|
|
91
|
+
self.defines[k.strip()] = v.strip() or "1"
|
|
92
|
+
self.undefined = set(sc.get("defines_off") or [])
|
|
93
|
+
# cargo_features: compute the exact enabled set per package
|
|
94
|
+
self.exact: dict[str, set] | None = None
|
|
95
|
+
cf = sc.get("cargo_features")
|
|
96
|
+
if cf is not None and packages:
|
|
97
|
+
self.exact = {}
|
|
98
|
+
for pkg, feats in packages.items():
|
|
99
|
+
want = set(["default"] if cf == "default" else [] if cf == "none" else
|
|
100
|
+
[f.split("/", 1)[1] if "/" in f and f.split("/", 1)[0] == pkg else f for f in cf if "/" not in f or f.startswith(pkg + "/")])
|
|
101
|
+
on, todo = set(), list(want)
|
|
102
|
+
while todo:
|
|
103
|
+
f = todo.pop()
|
|
104
|
+
if f in on:
|
|
105
|
+
continue
|
|
106
|
+
on.add(f)
|
|
107
|
+
for g in feats.get(f, []):
|
|
108
|
+
if "/" not in g and not g.startswith("dep:"):
|
|
109
|
+
todo.append(g)
|
|
110
|
+
self.exact[pkg] = on
|
|
111
|
+
|
|
112
|
+
@property
|
|
113
|
+
def active(self) -> bool:
|
|
114
|
+
return bool(self.name)
|
|
115
|
+
|
|
116
|
+
# Rust
|
|
117
|
+
def feature(self, pkg: str, f: str):
|
|
118
|
+
if f"{pkg}/{f}" in self.features_on or f in self.features_on:
|
|
119
|
+
return TRUE
|
|
120
|
+
if f"{pkg}/{f}" in self.features_off or f in self.features_off:
|
|
121
|
+
return FALSE
|
|
122
|
+
if self.exact is not None and pkg in self.exact:
|
|
123
|
+
return f in self.exact[pkg]
|
|
124
|
+
return UNKNOWN
|
|
125
|
+
|
|
126
|
+
def eval_cfg(self, p, pkg: str):
|
|
127
|
+
k = p[0]
|
|
128
|
+
if k == "atom":
|
|
129
|
+
key, val = p[1], p[2]
|
|
130
|
+
if key == "feature" and val is not None:
|
|
131
|
+
return self.feature(pkg, val)
|
|
132
|
+
t = atom_text(key, val)
|
|
133
|
+
if t in self.cfg_true:
|
|
134
|
+
return TRUE
|
|
135
|
+
if t in self.cfg_false:
|
|
136
|
+
return FALSE
|
|
137
|
+
return UNKNOWN
|
|
138
|
+
if k == "not":
|
|
139
|
+
v = self.eval_cfg(p[1], pkg)
|
|
140
|
+
return None if v is None else (not v)
|
|
141
|
+
vals = [self.eval_cfg(x, pkg) for x in p[1]]
|
|
142
|
+
if k == "all":
|
|
143
|
+
if any(v is False for v in vals):
|
|
144
|
+
return FALSE
|
|
145
|
+
return TRUE if all(v is True for v in vals) else UNKNOWN
|
|
146
|
+
if any(v is True for v in vals):
|
|
147
|
+
return TRUE
|
|
148
|
+
return FALSE if vals and all(v is False for v in vals) else UNKNOWN
|
|
149
|
+
|
|
150
|
+
# C / C++
|
|
151
|
+
def eval_pp(self, expr: str):
|
|
152
|
+
return eval_pp(expr, self.defines, self.undefined)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
# ---------------------------------------------------------------- C preprocessor conditions
|
|
156
|
+
_PP_TOK = re.compile(r"\s*(defined|\|\||&&|==|!=|<=|>=|<<|>>|[!()<>+\-*/%]|0[xX][0-9a-fA-F]+[uUlL]*|\d+[uUlL]*|[A-Za-z_]\w*|.)")
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def pp_macros(expr: str) -> list[str]:
|
|
160
|
+
"""Macro names a condition depends on (identifiers, `defined X`), in order, without duplicates."""
|
|
161
|
+
out = []
|
|
162
|
+
for m in re.finditer(r"[A-Za-z_]\w*", expr):
|
|
163
|
+
n = m.group(0)
|
|
164
|
+
if n != "defined" and n not in out and not n.startswith("__has_"):
|
|
165
|
+
out.append(n)
|
|
166
|
+
return out
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def eval_pp(expr: str, defines: dict[str, str], undefined: set[str]):
|
|
170
|
+
"""Three-valued evaluation of a #if expression. Unknown macros make the result unknown (None)
|
|
171
|
+
unless the other operand decides it (0 && x, 1 || x)."""
|
|
172
|
+
toks = [t for t in _PP_TOK.findall(expr) if t.strip()]
|
|
173
|
+
pos = [0]
|
|
174
|
+
|
|
175
|
+
def peek():
|
|
176
|
+
return toks[pos[0]] if pos[0] < len(toks) else None
|
|
177
|
+
|
|
178
|
+
def take():
|
|
179
|
+
t = peek()
|
|
180
|
+
pos[0] += 1
|
|
181
|
+
return t
|
|
182
|
+
|
|
183
|
+
def num(t):
|
|
184
|
+
try:
|
|
185
|
+
return int(t.rstrip("uUlL"), 0)
|
|
186
|
+
except ValueError:
|
|
187
|
+
return None
|
|
188
|
+
|
|
189
|
+
def primary():
|
|
190
|
+
t = take()
|
|
191
|
+
if t is None:
|
|
192
|
+
return None
|
|
193
|
+
if t == "(":
|
|
194
|
+
v = lor()
|
|
195
|
+
if peek() == ")":
|
|
196
|
+
take()
|
|
197
|
+
return v
|
|
198
|
+
if t == "!":
|
|
199
|
+
v = primary()
|
|
200
|
+
return None if v is None else int(not v)
|
|
201
|
+
if t == "-":
|
|
202
|
+
v = primary()
|
|
203
|
+
return None if v is None else -v
|
|
204
|
+
if t == "defined":
|
|
205
|
+
paren = peek() == "("
|
|
206
|
+
if paren:
|
|
207
|
+
take()
|
|
208
|
+
name = take()
|
|
209
|
+
if paren and peek() == ")":
|
|
210
|
+
take()
|
|
211
|
+
if name in defines:
|
|
212
|
+
return 1
|
|
213
|
+
if name in undefined:
|
|
214
|
+
return 0
|
|
215
|
+
return None
|
|
216
|
+
n = num(t)
|
|
217
|
+
if n is not None:
|
|
218
|
+
return n
|
|
219
|
+
if re.match(r"[A-Za-z_]", t):
|
|
220
|
+
if peek() == "(": # function-like macro / __has_include(...): unknown
|
|
221
|
+
depth = 0
|
|
222
|
+
while peek() is not None:
|
|
223
|
+
x = take()
|
|
224
|
+
depth += {"(": 1, ")": -1}.get(x, 0)
|
|
225
|
+
if depth == 0:
|
|
226
|
+
break
|
|
227
|
+
return None
|
|
228
|
+
if t in defines:
|
|
229
|
+
return num(defines[t]) if num(defines[t]) is not None else None
|
|
230
|
+
if t in undefined:
|
|
231
|
+
return 0
|
|
232
|
+
return None
|
|
233
|
+
return None
|
|
234
|
+
|
|
235
|
+
def binop(sub, ops):
|
|
236
|
+
def f():
|
|
237
|
+
v = sub()
|
|
238
|
+
while peek() in ops:
|
|
239
|
+
op = take()
|
|
240
|
+
w = sub()
|
|
241
|
+
if v is None or w is None:
|
|
242
|
+
v = None
|
|
243
|
+
continue
|
|
244
|
+
v = {"==": int(v == w), "!=": int(v != w), "<": int(v < w), ">": int(v > w), "<=": int(v <= w),
|
|
245
|
+
">=": int(v >= w), "+": v + w, "-": v - w, "*": v * w, "/": (v // w if w else None),
|
|
246
|
+
"%": (v % w if w else None), "<<": v << w if 0 <= w < 64 else None, ">>": v >> w if 0 <= w < 64 else None}[op]
|
|
247
|
+
return v
|
|
248
|
+
return f
|
|
249
|
+
|
|
250
|
+
mul = binop(primary, ("*", "/", "%"))
|
|
251
|
+
add = binop(mul, ("+", "-"))
|
|
252
|
+
shift = binop(add, ("<<", ">>"))
|
|
253
|
+
rel = binop(shift, ("<", ">", "<=", ">="))
|
|
254
|
+
eq = binop(rel, ("==", "!="))
|
|
255
|
+
|
|
256
|
+
def land():
|
|
257
|
+
v = eq()
|
|
258
|
+
while peek() == "&&":
|
|
259
|
+
take()
|
|
260
|
+
w = eq()
|
|
261
|
+
v = 0 if (v == 0 or w == 0) else (None if v is None or w is None else 1)
|
|
262
|
+
return v
|
|
263
|
+
|
|
264
|
+
def lor():
|
|
265
|
+
v = land()
|
|
266
|
+
while peek() == "||":
|
|
267
|
+
take()
|
|
268
|
+
w = land()
|
|
269
|
+
v = 1 if (v not in (None, 0) or w not in (None, 0)) else (None if v is None or w is None else 0)
|
|
270
|
+
return v
|
|
271
|
+
|
|
272
|
+
try:
|
|
273
|
+
v = lor()
|
|
274
|
+
except Exception:
|
|
275
|
+
return UNKNOWN
|
|
276
|
+
return UNKNOWN if v is None else bool(v)
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def range_lookup(ranges: list[tuple[int, int, object]]):
|
|
280
|
+
"""ranges: (start_line, end_line, payload), 1-based inclusive. Returns f(line) -> list of payloads covering it
|
|
281
|
+
(outermost first)."""
|
|
282
|
+
ranges = sorted(ranges, key=lambda r: (r[0], -r[1]))
|
|
283
|
+
|
|
284
|
+
def f(line: int):
|
|
285
|
+
return [p for (a, b, p) in ranges if a <= line <= b]
|
|
286
|
+
return f
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
"""Running external SCIP indexers (rust-analyzer, scip-clang) with an on-disk cache.
|
|
2
|
+
|
|
3
|
+
The cache key covers the cache version, the indexer version, its arguments and (path, size, content hash) of every
|
|
4
|
+
relevant source file, so re-indexing an unchanged project reuses the previous SCIP file and any content change (even
|
|
5
|
+
one that keeps the size and mtime) re-runs the indexer. CODEGRAPH_NO_CACHE=1 disables the cache.
|
|
6
|
+
Output never goes into the indexed project: files live under <cache root>/scip/ (core/cache.py: $CODEGRAPH_CACHE,
|
|
7
|
+
else $XDG_CACHE_HOME/codegraph or ~/.cache/codegraph), named <name>-v<cache version>-<project key>-<key>.scip.
|
|
8
|
+
Concurrent processes indexing the same project share one indexer run (a per-key lock) and never see each other's
|
|
9
|
+
partial output (private temporary files, atomic os.replace).
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import contextlib
|
|
14
|
+
import hashlib
|
|
15
|
+
import os
|
|
16
|
+
import re
|
|
17
|
+
import secrets
|
|
18
|
+
import shutil
|
|
19
|
+
import subprocess
|
|
20
|
+
import time
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
try:
|
|
24
|
+
import fcntl
|
|
25
|
+
except ImportError: # Windows: no flock; runs stay safe through private temporary files, without the wait
|
|
26
|
+
fcntl = None
|
|
27
|
+
|
|
28
|
+
from ...core import cache, fsutil
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def cache_dir() -> Path:
|
|
32
|
+
return cache.subdir("scip", create=True)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def find_tool(env: str, names: list[str], extra_dirs: list[Path] = ()) -> str | None:
|
|
36
|
+
v = os.environ.get(env)
|
|
37
|
+
if v:
|
|
38
|
+
return v if (Path(v).exists() or shutil.which(v)) else None
|
|
39
|
+
for n in names:
|
|
40
|
+
p = shutil.which(n)
|
|
41
|
+
if p:
|
|
42
|
+
return p
|
|
43
|
+
for d in extra_dirs:
|
|
44
|
+
if (Path(d) / n).exists():
|
|
45
|
+
return str(Path(d) / n)
|
|
46
|
+
return None
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def tool_version(tool: str, args=("--version",)) -> str:
|
|
50
|
+
try:
|
|
51
|
+
r = subprocess.run([tool, *args], capture_output=True, text=True, timeout=60)
|
|
52
|
+
return (r.stdout or r.stderr).strip().splitlines()[0] if (r.stdout or r.stderr) else "?"
|
|
53
|
+
except Exception:
|
|
54
|
+
return "?"
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def fingerprint(root: Path, files: list[str], extra: str) -> str:
|
|
58
|
+
"""Cache key: cache version + `extra` (indexer version, config) + root + (path, size, content hash) per file."""
|
|
59
|
+
h = hashlib.sha256(f"cg-cache-v{fsutil.CACHE_VERSION}\0{extra}".encode())
|
|
60
|
+
h.update(str(root).encode())
|
|
61
|
+
for f in sorted(files):
|
|
62
|
+
h.update(f"{f}\0{fsutil.content_key(root / f)}\n".encode())
|
|
63
|
+
return h.hexdigest()[:20]
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _prune_legacy(name: str) -> None:
|
|
67
|
+
"""Drop SCIP files written under an older cache version (`<name>-<20 hex>.scip`): their keys can never match again."""
|
|
68
|
+
for old in cache_dir().glob(f"{name}-*.scip"):
|
|
69
|
+
if re.fullmatch(rf"{re.escape(name)}-[0-9a-f]{{20}}\.scip", old.name):
|
|
70
|
+
old.unlink(missing_ok=True)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@contextlib.contextmanager
|
|
74
|
+
def key_lock(name: str, key: str, timeout: float):
|
|
75
|
+
"""An exclusive per-key lock (`<name>-<key>.lock` in the cache dir), so concurrent processes indexing the same
|
|
76
|
+
project run the indexer once: the others wait, then find the cached result. Yields the seconds spent waiting, or
|
|
77
|
+
None when the lock could not be taken in `timeout` (or file locks are unavailable): the caller runs unlocked then."""
|
|
78
|
+
if fcntl is None:
|
|
79
|
+
yield None
|
|
80
|
+
return
|
|
81
|
+
path = cache_dir() / f"{name}-v{fsutil.CACHE_VERSION}-{key}.lock"
|
|
82
|
+
fd = os.open(path, os.O_RDWR | os.O_CREAT, 0o644)
|
|
83
|
+
t0, got = time.time(), False
|
|
84
|
+
try:
|
|
85
|
+
while True:
|
|
86
|
+
try:
|
|
87
|
+
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
88
|
+
got = True
|
|
89
|
+
break
|
|
90
|
+
except OSError:
|
|
91
|
+
if time.time() - t0 >= timeout:
|
|
92
|
+
break
|
|
93
|
+
time.sleep(0.2)
|
|
94
|
+
yield round(time.time() - t0, 2) if got else None
|
|
95
|
+
finally:
|
|
96
|
+
if got:
|
|
97
|
+
fcntl.flock(fd, fcntl.LOCK_UN)
|
|
98
|
+
os.close(fd)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def write_atomic(path: Path, text: str) -> None:
|
|
102
|
+
"""Write a file other processes may read at the same time: a private temporary file, then os.replace."""
|
|
103
|
+
if path.exists() and path.read_text() == text:
|
|
104
|
+
return
|
|
105
|
+
tmp = path.with_name(f"{path.name}.{os.getpid()}.{secrets.token_hex(4)}.tmp")
|
|
106
|
+
tmp.write_text(text)
|
|
107
|
+
os.replace(tmp, path)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _prune_tmp(name: str, max_age: float = 86400) -> None:
|
|
111
|
+
"""Drop temporary outputs a killed run left behind (older than a day)."""
|
|
112
|
+
now = time.time()
|
|
113
|
+
for t in cache_dir().glob(f"{name}-*.tmp.scip"):
|
|
114
|
+
try:
|
|
115
|
+
if now - t.stat().st_mtime > max_age:
|
|
116
|
+
t.unlink(missing_ok=True)
|
|
117
|
+
except OSError:
|
|
118
|
+
pass
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def run_cached(name: str, key: str, cmd: list[str], cwd: Path, out_path_arg: str | None, timeout: int,
|
|
122
|
+
out_flag_style: str = "space", env: dict | None = None,
|
|
123
|
+
notes: str | None = None, root: Path | None = None) -> tuple[Path | None, dict]:
|
|
124
|
+
"""Run `cmd` (which writes a SCIP file) unless a cached result exists. The output path is appended to `cmd`
|
|
125
|
+
as `<out_path_arg> <path>` (style 'space') or `<out_path_arg>=<path>` (style 'eq'). `env` replaces the child's
|
|
126
|
+
environment; output lines matching the `notes` regex are kept in info["notes"] (a build error far above the tail).
|
|
127
|
+
Safe for concurrent processes: one run per key at a time (key_lock; the others wait and get a cache hit), each
|
|
128
|
+
run writes its own temporary file and moves it into the cache atomically."""
|
|
129
|
+
_prune_legacy(name)
|
|
130
|
+
proj = root or cwd
|
|
131
|
+
key = f"{cache.root_key(proj)}-{key}" # the project's entries carry its key: `cg clean <root>` finds them
|
|
132
|
+
cache.note_project(proj)
|
|
133
|
+
out = cache_dir() / f"{name}-v{fsutil.CACHE_VERSION}-{key}.scip"
|
|
134
|
+
use_cache = os.environ.get("CODEGRAPH_NO_CACHE") != "1"
|
|
135
|
+
info = {"cache": "hit" if out.exists() and use_cache else "miss", "command": " ".join(cmd)}
|
|
136
|
+
if info["cache"] == "hit":
|
|
137
|
+
return out, info
|
|
138
|
+
with key_lock(name, key, timeout) as waited:
|
|
139
|
+
if waited:
|
|
140
|
+
info["lock_wait_seconds"] = waited
|
|
141
|
+
if use_cache and out.exists(): # another process indexed the same project while this one waited
|
|
142
|
+
info["cache"] = "hit"
|
|
143
|
+
return out, info
|
|
144
|
+
_prune_tmp(name)
|
|
145
|
+
return _run(cmd, cwd, out, out_path_arg, out_flag_style, timeout, info, env, notes)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _run(cmd, cwd, out: Path, out_path_arg, out_flag_style, timeout, info, env=None,
|
|
149
|
+
notes=None) -> tuple[Path | None, dict]:
|
|
150
|
+
tmp = out.with_name(f"{out.stem}.{os.getpid()}.{secrets.token_hex(4)}.tmp.scip")
|
|
151
|
+
full = list(cmd)
|
|
152
|
+
if out_path_arg:
|
|
153
|
+
full += [f"{out_path_arg}={tmp}"] if out_flag_style == "eq" else [out_path_arg, str(tmp)]
|
|
154
|
+
t0 = time.time()
|
|
155
|
+
try:
|
|
156
|
+
try:
|
|
157
|
+
r = subprocess.run(full, cwd=cwd, capture_output=True, text=True, timeout=timeout, env=env)
|
|
158
|
+
except subprocess.TimeoutExpired:
|
|
159
|
+
info["error"] = f"timeout after {timeout}s"
|
|
160
|
+
return None, info
|
|
161
|
+
except OSError as e:
|
|
162
|
+
info["error"] = str(e)
|
|
163
|
+
return None, info
|
|
164
|
+
info["seconds"] = round(time.time() - t0, 2)
|
|
165
|
+
tail = (r.stderr or "").strip().splitlines()[-8:]
|
|
166
|
+
info["stderr_tail"] = tail
|
|
167
|
+
if notes:
|
|
168
|
+
rx = re.compile(notes)
|
|
169
|
+
hits = [x.strip()[:300] for x in ((r.stdout or "") + "\n" + (r.stderr or "")).splitlines() if rx.search(x)]
|
|
170
|
+
if hits:
|
|
171
|
+
info["notes"] = list(dict.fromkeys(hits))[:6]
|
|
172
|
+
if r.returncode != 0 or not tmp.exists():
|
|
173
|
+
info["error"] = f"exit {r.returncode}" + ("" if tmp.exists() else ", no SCIP output written")
|
|
174
|
+
if r.returncode == 0 and out.exists(): # the cache file appeared meanwhile (unlocked fallback)
|
|
175
|
+
info.pop("error")
|
|
176
|
+
info["cache"] = "hit"
|
|
177
|
+
return out, info
|
|
178
|
+
return None, info
|
|
179
|
+
os.replace(tmp, out)
|
|
180
|
+
return out, info
|
|
181
|
+
finally:
|
|
182
|
+
if tmp.exists():
|
|
183
|
+
tmp.unlink(missing_ok=True)
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
"""SCIP reading helpers shared by the native-language plugins (Rust, C/C++).
|
|
2
|
+
|
|
3
|
+
Unlike the generic importer (plugins/scip/importer.py), the native plugins own their node ids: definitions are
|
|
4
|
+
matched to syntactic items by (file, line, name) so the SCIP layer and the tree-sitter layer agree. This module only
|
|
5
|
+
loads an index, merges duplicate documents (scip-clang emits one per translation unit that touches a header) and
|
|
6
|
+
parses symbol descriptors.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
import sys
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scip"))
|
|
16
|
+
import scip_pb2 # noqa: E402
|
|
17
|
+
|
|
18
|
+
DEFINITION = scip_pb2.SymbolRole.Definition
|
|
19
|
+
FORWARD_DEFINITION = getattr(scip_pb2.SymbolRole, "ForwardDefinition", 64)
|
|
20
|
+
WRITE_ACCESS = scip_pb2.SymbolRole.WriteAccess
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass
|
|
24
|
+
class Occ:
|
|
25
|
+
line: int # 0-based
|
|
26
|
+
col: int
|
|
27
|
+
end_col: int
|
|
28
|
+
symbol: str
|
|
29
|
+
roles: int
|
|
30
|
+
enclosing: tuple # (start_line, end_line) 0-based, or ()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass
|
|
34
|
+
class Doc:
|
|
35
|
+
path: str
|
|
36
|
+
occs: list[Occ] = field(default_factory=list)
|
|
37
|
+
symbols: dict = field(default_factory=dict) # symbol -> SymbolInformation
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass
|
|
41
|
+
class Index:
|
|
42
|
+
tool: str
|
|
43
|
+
version: str
|
|
44
|
+
docs: dict[str, Doc]
|
|
45
|
+
# symbol -> list of (path, Occ) definitions (scip-clang: also forward definitions/declarations)
|
|
46
|
+
defs: dict[str, list] = field(default_factory=dict)
|
|
47
|
+
occurrences: int = 0 # non-local occurrences read
|
|
48
|
+
positioned: int = 0 # ... of which carried a range cg can read
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def has_position(o) -> bool:
|
|
52
|
+
"""The occurrence carries a range in a form cg reads (packed `range` of 3 / 4 ints, or a typed range)."""
|
|
53
|
+
return len(o.range) in (3, 4) or o.HasField("single_line_range") or o.HasField("multi_line_range")
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def health_warning(name: str, occurrences: int, positioned: int, definitions: int | None = None,
|
|
57
|
+
matched: int | None = None, what: str = "the syntax layer's declarations") -> str | None:
|
|
58
|
+
"""A warning for an index cg could not use although it is not empty, else None. Reported in `cg coverage`
|
|
59
|
+
(`warnings`) and by `cg doctor --scip`, instead of an exact layer that silently adds nothing."""
|
|
60
|
+
if occurrences and not positioned:
|
|
61
|
+
return (f"SCIP index {name}: {occurrences} occurrences but none with a usable position (range missing or in a "
|
|
62
|
+
"form cg does not read): no edges could be placed; regenerate it, or report the indexer version")
|
|
63
|
+
if occurrences and definitions == 0:
|
|
64
|
+
return (f"SCIP index {name}: {occurrences} occurrences but 0 definitions (symbols cg maps to nodes): nothing "
|
|
65
|
+
"imported; check the indexer and that the index belongs to this project")
|
|
66
|
+
if definitions and matched == 0:
|
|
67
|
+
return (f"SCIP index {name}: 0 of {definitions} definitions matched {what}: the exact layer added no edges; "
|
|
68
|
+
"check that the index was built from these sources (same paths, same revision)")
|
|
69
|
+
return None
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _rng(r) -> tuple[int, int, int, int]:
|
|
73
|
+
r = list(r)
|
|
74
|
+
if len(r) == 3:
|
|
75
|
+
return r[0], r[1], r[0], r[2]
|
|
76
|
+
if len(r) == 4:
|
|
77
|
+
return r[0], r[1], r[2], r[3]
|
|
78
|
+
return (0, 0, 0, 0)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _typed(o, legacy: str, single: str, multi: str) -> tuple[int, int, int, int] | None:
|
|
82
|
+
"""An occurrence range: the legacy packed `range` / `enclosing_range`, else the typed one newer indexers write
|
|
83
|
+
(SCIP 0.9 `single_line_range` / `multi_line_range`, e.g. scip-java 0.13); None when absent."""
|
|
84
|
+
r = getattr(o, legacy)
|
|
85
|
+
if len(r):
|
|
86
|
+
return _rng(r)
|
|
87
|
+
if o.HasField(single):
|
|
88
|
+
t = getattr(o, single)
|
|
89
|
+
return t.line, t.start_character, t.line, t.end_character
|
|
90
|
+
if o.HasField(multi):
|
|
91
|
+
t = getattr(o, multi)
|
|
92
|
+
return t.start_line, t.start_character, t.end_line, t.end_character
|
|
93
|
+
return None
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def occ_range(o) -> tuple[int, int, int, int]:
|
|
97
|
+
return _typed(o, "range", "single_line_range", "multi_line_range") or (0, 0, 0, 0)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def occ_enclosing(o) -> tuple[int, int, int, int] | None:
|
|
101
|
+
return _typed(o, "enclosing_range", "single_line_enclosing_range", "multi_line_enclosing_range")
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def load(path: str | Path) -> Index:
|
|
105
|
+
idx = scip_pb2.Index()
|
|
106
|
+
idx.ParseFromString(Path(path).read_bytes())
|
|
107
|
+
docs: dict[str, Doc] = {}
|
|
108
|
+
seen: dict[str, set] = {}
|
|
109
|
+
out_occ = [0, 0]
|
|
110
|
+
for d in idx.documents:
|
|
111
|
+
doc = docs.get(d.relative_path)
|
|
112
|
+
if doc is None:
|
|
113
|
+
doc = docs[d.relative_path] = Doc(d.relative_path)
|
|
114
|
+
seen[d.relative_path] = set()
|
|
115
|
+
s = seen[d.relative_path]
|
|
116
|
+
for o in d.occurrences:
|
|
117
|
+
if o.symbol.startswith("local ") or not o.symbol:
|
|
118
|
+
continue
|
|
119
|
+
out_occ[0] += 1
|
|
120
|
+
if has_position(o):
|
|
121
|
+
out_occ[1] += 1
|
|
122
|
+
sl, sc, el, ec = occ_range(o)
|
|
123
|
+
key = (sl, sc, ec, o.symbol, o.symbol_roles)
|
|
124
|
+
if key in s:
|
|
125
|
+
continue
|
|
126
|
+
s.add(key)
|
|
127
|
+
enc = ()
|
|
128
|
+
er = occ_enclosing(o)
|
|
129
|
+
if er:
|
|
130
|
+
enc = (er[0], er[2])
|
|
131
|
+
doc.occs.append(Occ(sl, sc, ec, o.symbol, o.symbol_roles, enc))
|
|
132
|
+
for si in d.symbols:
|
|
133
|
+
if not si.symbol.startswith("local "):
|
|
134
|
+
doc.symbols.setdefault(si.symbol, si)
|
|
135
|
+
out = Index(idx.metadata.tool_info.name, idx.metadata.tool_info.version, docs, occurrences=out_occ[0],
|
|
136
|
+
positioned=out_occ[1])
|
|
137
|
+
for doc in docs.values():
|
|
138
|
+
doc.occs.sort(key=lambda o: (o.line, o.col))
|
|
139
|
+
for o in doc.occs:
|
|
140
|
+
if o.roles & DEFINITION:
|
|
141
|
+
out.defs.setdefault(o.symbol, []).append((doc.path, o))
|
|
142
|
+
return out
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
_DESC = re.compile(r"(`(?:[^`]|``)*`|[^\s/#.()\[\]:!`]+)?(/|#|\.|:|!|\(([^)]*)\)\.|\[)")
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def descriptors(symbol: str) -> tuple[str, list[tuple[str, str]]] | None:
|
|
149
|
+
"""'<scheme> <manager> <package> <version> <descriptors>' -> (package, [(name, suffix)]).
|
|
150
|
+
suffix: '/' namespace, '#' type, '.' term, '(' method, '!' macro, '[' type parameter / impl disambiguator."""
|
|
151
|
+
if symbol.startswith("local "):
|
|
152
|
+
return None
|
|
153
|
+
parts = symbol.split(" ", 4)
|
|
154
|
+
if len(parts) < 5:
|
|
155
|
+
return None
|
|
156
|
+
pkg, desc = parts[2], parts[4]
|
|
157
|
+
out = []
|
|
158
|
+
i = 0
|
|
159
|
+
while i < len(desc):
|
|
160
|
+
if desc[i] == "[": # [name] type parameter (rust-analyzer uses it for impl#[Type][Trait])
|
|
161
|
+
j = i + 1
|
|
162
|
+
depth = 1
|
|
163
|
+
while j < len(desc) and depth:
|
|
164
|
+
if desc[j] == "`":
|
|
165
|
+
k = desc.find("`", j + 1)
|
|
166
|
+
j = (k if k > 0 else len(desc) - 1) + 1
|
|
167
|
+
continue
|
|
168
|
+
depth += {"[": 1, "]": -1}.get(desc[j], 0)
|
|
169
|
+
j += 1
|
|
170
|
+
out.append((desc[i + 1:j - 1].strip("`"), "["))
|
|
171
|
+
i = j
|
|
172
|
+
continue
|
|
173
|
+
m = _DESC.match(desc, i)
|
|
174
|
+
if not m or m.end() == i:
|
|
175
|
+
break
|
|
176
|
+
name = (m.group(1) or "").strip("`").replace("``", "`")
|
|
177
|
+
suf = m.group(2)
|
|
178
|
+
if suf.startswith("("):
|
|
179
|
+
suf = "("
|
|
180
|
+
out.append((name, suf))
|
|
181
|
+
i = m.end()
|
|
182
|
+
return pkg, out
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def is_project_symbol(idx: Index, symbol: str) -> bool:
|
|
186
|
+
return symbol in idx.defs
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def native_warning(path, idx: Index, stats: dict) -> str | None:
|
|
190
|
+
"""health_warning for a native plugin's import (rust-analyzer, scip-clang): matched = definitions placed on a
|
|
191
|
+
syntax-layer item, the rest became synthetic nodes."""
|
|
192
|
+
m = stats.get("scip_defs_matched", 0)
|
|
193
|
+
return health_warning(Path(path).name, idx.occurrences, idx.positioned,
|
|
194
|
+
(m + stats.get("scip_defs_synthetic", 0)) or None, m, "the syntax layer's items")
|