codeanalyzer-python 1.2.0__py3-none-any.whl → 1.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codeanalyzer/__main__.py +29 -0
- codeanalyzer/artifacts/__init__.py +20 -0
- codeanalyzer/artifacts/config_keys.py +588 -0
- codeanalyzer/artifacts/config_use.py +597 -0
- codeanalyzer/artifacts/config_use_rules.yml +58 -0
- codeanalyzer/artifacts/dependencies.py +237 -0
- codeanalyzer/artifacts/discovery.py +167 -0
- codeanalyzer/artifacts/parsers.py +248 -0
- codeanalyzer/core.py +91 -0
- codeanalyzer/dataflow/builder.py +15 -1
- codeanalyzer/neo4j/project.py +192 -0
- codeanalyzer/neo4j/schema.py +57 -0
- codeanalyzer/options/options.py +5 -0
- codeanalyzer/schema/ids.py +21 -0
- codeanalyzer/schema/py_schema.py +118 -0
- codeanalyzer/syntactic_analysis/symbol_table_builder.py +11 -0
- {codeanalyzer_python-1.2.0.dist-info → codeanalyzer_python-1.3.0.dist-info}/METADATA +109 -5
- {codeanalyzer_python-1.2.0.dist-info → codeanalyzer_python-1.3.0.dist-info}/RECORD +22 -15
- {codeanalyzer_python-1.2.0.dist-info → codeanalyzer_python-1.3.0.dist-info}/WHEEL +0 -0
- {codeanalyzer_python-1.2.0.dist-info → codeanalyzer_python-1.3.0.dist-info}/entry_points.txt +0 -0
- {codeanalyzer_python-1.2.0.dist-info → codeanalyzer_python-1.3.0.dist-info}/licenses/LICENSE +0 -0
- {codeanalyzer_python-1.2.0.dist-info → codeanalyzer_python-1.3.0.dist-info}/licenses/NOTICE +0 -0
|
@@ -0,0 +1,597 @@
|
|
|
1
|
+
"""config_use detection + resolution (#162): mints `PY_USES_CONFIG` edges
|
|
2
|
+
between a call site's key argument and the `PyConfigKey` it reads, plus
|
|
3
|
+
first-class unresolved records for a key that never closes on a literal.
|
|
4
|
+
|
|
5
|
+
Three tiers, wired from core.py: `detect_config_reads` runs once, after
|
|
6
|
+
callee backfill and the config-keys extraction loop (both populate
|
|
7
|
+
substrate this depends on -- `BodyNode.callee` and `PyArtifact.config_keys`),
|
|
8
|
+
scanning every callable's `body{}` for a `call` node whose `callee` names a
|
|
9
|
+
`PyExternalSymbol` matching a shipped detector rule (`config_use_rules.yml`,
|
|
10
|
+
same load/validate idiom as `entrypoints/rules.py`). `resolve_uses` decodes
|
|
11
|
+
each matched call's key argument, resolves a string literal against
|
|
12
|
+
`PyArtifact.config_keys` per the rule's namespace preference (the literal
|
|
13
|
+
tier, built in directly), then threads whatever is still unresolved through
|
|
14
|
+
`tier_fns` in order -- core.py passes `[dataflow_intra_tier]` at `-a 3` and
|
|
15
|
+
`[dataflow_intra_tier, dataflow_interproc_tier]` at `-a 4` (#162 Task 3),
|
|
16
|
+
so a read resolved at a lower tier is never recomputed and `-a 2 ⊆ -a 3 ⊆
|
|
17
|
+
-a 4` holds by construction.
|
|
18
|
+
|
|
19
|
+
`dataflow_intra_tier` closes a non-literal key argument (`_Read.key_name`)
|
|
20
|
+
over its own callable's DDG: `_reaching_literal` finds every DDG edge for
|
|
21
|
+
that variable whose destination node's span *contains* the call's span --
|
|
22
|
+
not `== the call's own local id` as a first reading of the plan suggests,
|
|
23
|
+
because the CFG (and hence the DDG) is statement-level (`dataflow/cfg.py`)
|
|
24
|
+
while a call nested in `return`/an assignment gets its own, narrower body-key
|
|
25
|
+
span (`schema/l1_body.py`); containment is what actually finds the reaching
|
|
26
|
+
def for `return os.getenv(KEY)` as well as a bare `os.getenv(KEY)` statement
|
|
27
|
+
(where call and statement spans coincide) -- verified empirically against
|
|
28
|
+
both shapes before landing this. Each reaching def must slice+parse to
|
|
29
|
+
exactly one `Name = <str Constant>` Assign; multiple reaching defs must all
|
|
30
|
+
yield the *same* literal (spec caveat: identical duplicates count as closed).
|
|
31
|
+
`dataflow_interproc_tier` handles a key that names a *parameter* of its
|
|
32
|
+
enclosing callable: every call site targeting that callable (`call_graph`
|
|
33
|
+
join, then a `body` scan for the matching `callee` id -- never `param_in`,
|
|
34
|
+
controller ruling) must supply the same string literal at that parameter's
|
|
35
|
+
position, either directly (`PyCallArgument.value`) or by one non-recursive
|
|
36
|
+
hop of the same intra closure at the *caller*'s own call site (`visited`
|
|
37
|
+
seeded with the callee id guards the direct-self-recursion case). Both
|
|
38
|
+
tiers re-resolve a closed literal against `PyArtifact.config_keys` through
|
|
39
|
+
the same namespace-preference helper the literal tier uses, so a value that
|
|
40
|
+
closes but names no declared key still becomes `reason="undefined-key"`
|
|
41
|
+
rather than `"non-literal"`.
|
|
42
|
+
|
|
43
|
+
Rule matching is prefix-aware on `module`, not exact-equal: empirically,
|
|
44
|
+
`configparser.ConfigParser().get(...)` resolves (via the defuse linker) to
|
|
45
|
+
callee module `configparser.RawConfigParser` -- `get` is inherited from the
|
|
46
|
+
base class, not defined on `ConfigParser` itself -- so a strict `module ==
|
|
47
|
+
"configparser"` would never match the shipped rule. `module == rule.module
|
|
48
|
+
or module.startswith(rule.module + ".")` covers this without a per-rule
|
|
49
|
+
alias list.
|
|
50
|
+
|
|
51
|
+
Subscript caveat (controller ruling, verified empirically rather than
|
|
52
|
+
assumed): `os.environ["X"]` is an `ast.Subscript`, and
|
|
53
|
+
`_iter_calls_in_scope` (symbol_table_builder.py) only ever yields
|
|
54
|
+
`ast.Call` nodes -- a probe project with an `os.environ[...]` read produces
|
|
55
|
+
zero call body nodes for that statement
|
|
56
|
+
(`test_environ_subscript_is_not_lowered_to_a_call_node`,
|
|
57
|
+
test/test_config_use_literal.py). `os.getenv` and `os.environ.get` cover
|
|
58
|
+
the `[env]` namespace in v1; the subscript form is a recorded gap, not a
|
|
59
|
+
rule table entry -- there is no call node it could ever match.
|
|
60
|
+
"""
|
|
61
|
+
from __future__ import annotations
|
|
62
|
+
|
|
63
|
+
import ast
|
|
64
|
+
import json
|
|
65
|
+
from dataclasses import dataclass, replace
|
|
66
|
+
from pathlib import Path
|
|
67
|
+
from typing import Callable, Dict, List, Optional, Sequence, Set, Tuple
|
|
68
|
+
|
|
69
|
+
import yaml
|
|
70
|
+
|
|
71
|
+
from codeanalyzer.schema.ids import ordinal_id
|
|
72
|
+
from codeanalyzer.schema.py_schema import (
|
|
73
|
+
BodyNode, PyApplication, PyCallable, PyClass, PyConfigKey, PyConfigRead,
|
|
74
|
+
PyConfigUseEdge, PyModule,
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
_SHIPPED = Path(__file__).with_name("config_use_rules.yml")
|
|
78
|
+
_TOP_LEVEL_KEYS = {"version", "rules"}
|
|
79
|
+
_REQUIRED_RULE_KEYS = {"id", "module", "callable", "key_arg", "namespaces"}
|
|
80
|
+
_OPTIONAL_RULE_KEYS = {"kwarg"}
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class ConfigUseRulesError(Exception):
|
|
84
|
+
"""Raised for a malformed config_use_rules.yml. Never swallowed."""
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
@dataclass(frozen=True)
|
|
88
|
+
class Rule:
|
|
89
|
+
id: str
|
|
90
|
+
module: str
|
|
91
|
+
callable: str
|
|
92
|
+
key_arg: int
|
|
93
|
+
namespaces: Tuple[str, ...]
|
|
94
|
+
kwarg: Optional[str] = None # not yet actionable -- see _key_from_args
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
@dataclass(frozen=True)
|
|
98
|
+
class _Read:
|
|
99
|
+
"""One detector-matched call, before resolution."""
|
|
100
|
+
|
|
101
|
+
site: str # GLOBAL ordinal id: <callable-id>@<local-id>
|
|
102
|
+
# `callable_id`/`local_id`: the dataflow tiers' join point back to the
|
|
103
|
+
# enclosing callable's `.ddg`/`.body` (and, for the interproc tier, its
|
|
104
|
+
# `.parameters` and `call_graph` membership as a callee).
|
|
105
|
+
callable_id: str # the enclosing callable's can:// id
|
|
106
|
+
local_id: str # the call node's own LOCAL id within `callable_id`'s body
|
|
107
|
+
callee: str # external id
|
|
108
|
+
rule: Rule
|
|
109
|
+
key_literal: Optional[str] # decoded str, when the key arg is a str constant
|
|
110
|
+
key_name: Optional[str] # bare Name identifier, when the key arg is a Name
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
# A tier consumes the reads still unresolved after the ones before it, and
|
|
114
|
+
# returns (new edges, reads still unresolved after this tier). core.py wires
|
|
115
|
+
# `[dataflow_intra_tier]` at `-a 3` and `[dataflow_intra_tier,
|
|
116
|
+
# dataflow_interproc_tier]` at `-a 4`; the literal tier (built into
|
|
117
|
+
# resolve_uses) always runs first, regardless of level.
|
|
118
|
+
TierFn = Callable[[List[_Read], PyApplication], Tuple[List[PyConfigUseEdge], List[_Read]]]
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def load_rules(path: Path = _SHIPPED) -> List[Rule]:
|
|
122
|
+
try:
|
|
123
|
+
data = yaml.safe_load(path.read_text())
|
|
124
|
+
except FileNotFoundError as exc:
|
|
125
|
+
raise ConfigUseRulesError(f"rules file not found: {path}") from exc
|
|
126
|
+
except yaml.YAMLError as exc:
|
|
127
|
+
raise ConfigUseRulesError(f"{path}: invalid YAML: {exc}") from exc
|
|
128
|
+
if not isinstance(data, dict):
|
|
129
|
+
raise ConfigUseRulesError(f"{path}: top level must be a mapping")
|
|
130
|
+
unknown = sorted(set(data) - _TOP_LEVEL_KEYS)
|
|
131
|
+
if unknown:
|
|
132
|
+
raise ConfigUseRulesError(f"{path}: unknown top-level key(s): {', '.join(unknown)}")
|
|
133
|
+
raw_rules = data.get("rules") or []
|
|
134
|
+
if not isinstance(raw_rules, list):
|
|
135
|
+
raise ConfigUseRulesError(f"{path}: `rules` must be a list")
|
|
136
|
+
return [_parse_rule(raw, path) for raw in raw_rules]
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _parse_rule(raw: Dict, origin: Path) -> Rule:
|
|
140
|
+
if not isinstance(raw, dict):
|
|
141
|
+
raise ConfigUseRulesError(f"{origin}: rule entry must be a mapping: {raw!r}")
|
|
142
|
+
missing = _REQUIRED_RULE_KEYS - set(raw)
|
|
143
|
+
if missing:
|
|
144
|
+
raise ConfigUseRulesError(f"{origin}: rule {raw!r} missing {sorted(missing)}")
|
|
145
|
+
unknown = set(raw) - _REQUIRED_RULE_KEYS - _OPTIONAL_RULE_KEYS
|
|
146
|
+
if unknown:
|
|
147
|
+
raise ConfigUseRulesError(f"{origin}: rule {raw!r} has unknown key(s) {sorted(unknown)}")
|
|
148
|
+
namespaces = raw["namespaces"]
|
|
149
|
+
if not isinstance(namespaces, list) or not namespaces or not all(isinstance(n, str) for n in namespaces):
|
|
150
|
+
raise ConfigUseRulesError(
|
|
151
|
+
f"{origin}: rule {raw['id']!r} `namespaces` must be a non-empty list of strings"
|
|
152
|
+
)
|
|
153
|
+
kwarg = raw.get("kwarg")
|
|
154
|
+
if kwarg is not None and not isinstance(kwarg, str):
|
|
155
|
+
raise ConfigUseRulesError(f"{origin}: rule {raw['id']!r} `kwarg` must be a string")
|
|
156
|
+
return Rule(
|
|
157
|
+
id=raw["id"], module=raw["module"], callable=raw["callable"],
|
|
158
|
+
key_arg=int(raw["key_arg"]), namespaces=tuple(namespaces),
|
|
159
|
+
kwarg=kwarg,
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _rule_matches(rule: Rule, module: Optional[str], name: str) -> bool:
|
|
164
|
+
if name != rule.callable:
|
|
165
|
+
return False
|
|
166
|
+
mod = module or ""
|
|
167
|
+
return mod == rule.module or mod.startswith(rule.module + ".")
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _walk_callable_tree(c: PyCallable):
|
|
171
|
+
yield c
|
|
172
|
+
for ic in (c.callables or {}).values():
|
|
173
|
+
yield from _walk_callable_tree(ic)
|
|
174
|
+
for cl in (c.types or {}).values():
|
|
175
|
+
yield from _walk_class_tree(cl)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _walk_class_tree(cl: PyClass):
|
|
179
|
+
for m in (cl.callables or {}).values():
|
|
180
|
+
yield from _walk_callable_tree(m)
|
|
181
|
+
for ic in (cl.types or {}).values():
|
|
182
|
+
yield from _walk_class_tree(ic)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _walk_module_callables(mod: PyModule):
|
|
186
|
+
for fn in (mod.functions or {}).values():
|
|
187
|
+
yield from _walk_callable_tree(fn)
|
|
188
|
+
for cl in (mod.types or {}).values():
|
|
189
|
+
yield from _walk_class_tree(cl)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _walk_callables(app: PyApplication):
|
|
193
|
+
for mod in app.symbol_table.values():
|
|
194
|
+
yield from _walk_module_callables(mod)
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _index_callables(app: PyApplication) -> Dict[str, Tuple[PyCallable, str]]:
|
|
198
|
+
"""``callable id -> (callable, owning module's source)`` -- the dataflow
|
|
199
|
+
tiers' join point from a read's/call-graph's callable id back to the
|
|
200
|
+
`.ddg`/`.body` data (and the module `source` needed to slice a def's
|
|
201
|
+
span text) that the intra closure reads."""
|
|
202
|
+
return {
|
|
203
|
+
c.id: (c, mod.source)
|
|
204
|
+
for mod in app.symbol_table.values()
|
|
205
|
+
for c in _walk_module_callables(mod)
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _key_from_args(arguments, key_arg: int) -> Tuple[Optional[str], Optional[str]]:
|
|
210
|
+
"""`(key_literal, key_name)` from the call's key-position argument.
|
|
211
|
+
|
|
212
|
+
Only positional args reach `BodyNode.arguments` (symbol_table_builder.py
|
|
213
|
+
only walks `node.args`, never `node.keywords`) -- a `kwarg=`-only call
|
|
214
|
+
(e.g. `cp.get(section, option="x")`) has no substrate to read the key
|
|
215
|
+
from, and this returns `(None, None)` for it same as a missing position.
|
|
216
|
+
"""
|
|
217
|
+
if key_arg >= len(arguments):
|
|
218
|
+
return None, None
|
|
219
|
+
arg = arguments[key_arg]
|
|
220
|
+
literal = None
|
|
221
|
+
if arg.value is not None:
|
|
222
|
+
try:
|
|
223
|
+
decoded = json.loads(arg.value)
|
|
224
|
+
except json.JSONDecodeError:
|
|
225
|
+
decoded = None
|
|
226
|
+
if isinstance(decoded, str):
|
|
227
|
+
literal = decoded
|
|
228
|
+
return literal, arg.name
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def detect_config_reads(app: PyApplication, rules: Optional[Sequence[Rule]] = None) -> List[_Read]:
|
|
232
|
+
"""Every call body node whose resolved callee matches a detector rule,
|
|
233
|
+
sorted by site for deterministic downstream iteration."""
|
|
234
|
+
rules = load_rules() if rules is None else rules
|
|
235
|
+
reads: List[_Read] = []
|
|
236
|
+
for c in _walk_callables(app):
|
|
237
|
+
for local_id, node in (c.body or {}).items():
|
|
238
|
+
if node.kind != "call" or not node.callee:
|
|
239
|
+
continue
|
|
240
|
+
sym = app.external_symbols.get(node.callee)
|
|
241
|
+
if sym is None:
|
|
242
|
+
continue
|
|
243
|
+
for rule in rules:
|
|
244
|
+
if not _rule_matches(rule, sym.module, sym.name):
|
|
245
|
+
continue
|
|
246
|
+
key_literal, key_name = _key_from_args(node.arguments or [], rule.key_arg)
|
|
247
|
+
reads.append(_Read(
|
|
248
|
+
site=ordinal_id(c.id, local_id), callable_id=c.id, local_id=local_id,
|
|
249
|
+
callee=node.callee, rule=rule, key_literal=key_literal, key_name=key_name,
|
|
250
|
+
))
|
|
251
|
+
break # (module, callable) is unambiguous across the shipped rule set
|
|
252
|
+
reads.sort(key=lambda r: (r.site, r.rule.id))
|
|
253
|
+
return reads
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def _namespace_matches(literal: str, namespace: str, keys: List[PyConfigKey]) -> List[PyConfigKey]:
|
|
257
|
+
if namespace == "env":
|
|
258
|
+
matched = [k for k in keys if k.key == literal]
|
|
259
|
+
else: # ini/properties: the option name is the key's last dotted segment
|
|
260
|
+
matched = [k for k in keys if k.key == literal or k.key.endswith("." + literal)]
|
|
261
|
+
return sorted(matched, key=lambda k: k.id)
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _keys_by_namespace(app: PyApplication) -> Dict[str, List[PyConfigKey]]:
|
|
265
|
+
keys_by_namespace: Dict[str, List[PyConfigKey]] = {}
|
|
266
|
+
for art in app.artifacts.values():
|
|
267
|
+
for key in art.config_keys:
|
|
268
|
+
keys_by_namespace.setdefault(key.namespace, []).append(key)
|
|
269
|
+
return keys_by_namespace
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def _resolve_literal_against_keys(
|
|
273
|
+
rule: Rule, literal: str, keys_by_namespace: Dict[str, List[PyConfigKey]],
|
|
274
|
+
) -> List[PyConfigKey]:
|
|
275
|
+
"""The matched `PyConfigKey`s for `literal` under `rule`'s namespace
|
|
276
|
+
preference order -- the first namespace with >=1 match wins, shared by
|
|
277
|
+
the literal tier and both dataflow tiers so a closed-but-undeclared key
|
|
278
|
+
is `reason="undefined-key"` regardless of which tier closed it."""
|
|
279
|
+
for namespace in rule.namespaces:
|
|
280
|
+
matched = _namespace_matches(literal, namespace, keys_by_namespace.get(namespace, []))
|
|
281
|
+
if matched:
|
|
282
|
+
return matched
|
|
283
|
+
return []
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def resolve_uses(
|
|
287
|
+
reads: List[_Read], app: PyApplication, tier_fns: Sequence[TierFn] = (),
|
|
288
|
+
) -> Tuple[List[PyConfigUseEdge], List[PyConfigRead]]:
|
|
289
|
+
"""Literal tier (built in here) then any dataflow `tier_fns` (Task 3) over
|
|
290
|
+
what's still unresolved. Reads resolved at a lower tier are never
|
|
291
|
+
recomputed -- additive, so `-a 2 ⊆ -a 3 ⊆ -a 4` holds by construction."""
|
|
292
|
+
keys_by_namespace = _keys_by_namespace(app)
|
|
293
|
+
|
|
294
|
+
edges: List[PyConfigUseEdge] = []
|
|
295
|
+
unresolved: List[_Read] = []
|
|
296
|
+
for read in reads:
|
|
297
|
+
if read.key_literal is None:
|
|
298
|
+
unresolved.append(read)
|
|
299
|
+
continue
|
|
300
|
+
matched = _resolve_literal_against_keys(read.rule, read.key_literal, keys_by_namespace)
|
|
301
|
+
if not matched:
|
|
302
|
+
unresolved.append(read)
|
|
303
|
+
continue
|
|
304
|
+
for key in matched:
|
|
305
|
+
edges.append(PyConfigUseEdge(src=read.site, dst=key.id, prov=["literal"]))
|
|
306
|
+
|
|
307
|
+
for tier_fn in tier_fns:
|
|
308
|
+
new_edges, unresolved = tier_fn(unresolved, app)
|
|
309
|
+
edges.extend(new_edges)
|
|
310
|
+
|
|
311
|
+
# `prov` lists every tier attempted: the literal tier always runs (above);
|
|
312
|
+
# `tier_fns` non-empty means some dataflow tier(s) ran too -- the
|
|
313
|
+
# vocabulary is exactly "literal"/"dataflow" (no separate intra/interproc
|
|
314
|
+
# tag), so intra-only (-a 3) and intra+interproc (-a 4) both read the same.
|
|
315
|
+
attempted = ["literal"] + (["dataflow"] if tier_fns else [])
|
|
316
|
+
unresolved_records = [
|
|
317
|
+
PyConfigRead(
|
|
318
|
+
site=r.site, callee=r.callee, key=r.key_literal,
|
|
319
|
+
reason="undefined-key" if r.key_literal is not None else "non-literal",
|
|
320
|
+
prov=attempted,
|
|
321
|
+
)
|
|
322
|
+
for r in unresolved
|
|
323
|
+
]
|
|
324
|
+
edges.sort(key=lambda e: (e.src, e.dst))
|
|
325
|
+
unresolved_records.sort(key=lambda u: (u.site, u.reason, u.key or ""))
|
|
326
|
+
return edges, unresolved_records
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
# --- dataflow tiers (#162 Task 3) -------------------------------------------
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def _assign_literal(source: str, def_node: Optional[BodyNode]) -> Optional[str]:
|
|
333
|
+
"""The str constant a single reaching def closes on, or `None` if
|
|
334
|
+
`def_node` isn't exactly a single-target `Name = <str Constant>` Assign
|
|
335
|
+
(a formal-parameter binding has no span; a `for`-header or multi-target
|
|
336
|
+
assign fails to parse/shape-match -- both correctly never close).
|
|
337
|
+
Also rejected by design, not just incidentally: `ast.AnnAssign`
|
|
338
|
+
(`KEY: str = "X"`), `ast.AugAssign` (`KEY += "X"`), and tuple-unpack
|
|
339
|
+
(`KEY, other = ...`) -- none is a plain single-target `Name = <str
|
|
340
|
+
Constant>` Assign."""
|
|
341
|
+
if def_node is None or def_node.span is None:
|
|
342
|
+
return None
|
|
343
|
+
lo, hi = def_node.span.bytes
|
|
344
|
+
text = source.encode("utf-8")[lo:hi].decode("utf-8")
|
|
345
|
+
try:
|
|
346
|
+
parsed = ast.parse(text)
|
|
347
|
+
except SyntaxError:
|
|
348
|
+
return None
|
|
349
|
+
if len(parsed.body) != 1:
|
|
350
|
+
return None
|
|
351
|
+
stmt = parsed.body[0]
|
|
352
|
+
if not isinstance(stmt, ast.Assign) or len(stmt.targets) != 1 or not isinstance(stmt.targets[0], ast.Name):
|
|
353
|
+
return None
|
|
354
|
+
value = stmt.value
|
|
355
|
+
if isinstance(value, ast.Constant) and isinstance(value.value, str):
|
|
356
|
+
return value.value
|
|
357
|
+
return None
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
def _reaching_literal(c: PyCallable, source: str, use_local_id: str, var: str) -> Optional[str]:
|
|
361
|
+
"""The one string literal every DDG-reaching def of `var` at
|
|
362
|
+
`use_local_id` closes on -- `None` if nothing reaches, any reaching def
|
|
363
|
+
isn't a literal assignment, or reaching defs disagree (identical
|
|
364
|
+
duplicates count as one, per the spec caveat).
|
|
365
|
+
|
|
366
|
+
`use_local_id` is a `call` node's own local id, keyed by its own
|
|
367
|
+
(typically narrower) span -- but the CFG/DDG is statement-level
|
|
368
|
+
(`dataflow/cfg.py`), so a def's DDG-recorded *use* site is the
|
|
369
|
+
*enclosing statement's* local id, which only coincides with the call's
|
|
370
|
+
own id when the call is itself a bare expression statement. Matching by
|
|
371
|
+
span containment (the def's dst node's span contains the call's span)
|
|
372
|
+
covers both that coincidence and the common case of a call nested in a
|
|
373
|
+
`return`/assignment, without needing to special-case either shape.
|
|
374
|
+
|
|
375
|
+
Only `prov=["ssa"]` edges are consulted: at `-a 4`
|
|
376
|
+
`c.ddg` also carries `points-to` (may-alias widening) and `reaching-defs`
|
|
377
|
+
(SDG port-routing, #115) edges. An alias edge can connect this `var`'s
|
|
378
|
+
use to an unrelated attribute-write def -- `obj.attr = KEY` may-aliases
|
|
379
|
+
bare `KEY` itself, since an unsuffixed local's empty suffix is
|
|
380
|
+
oracle-compatible with any attribute path -- that is never a `Name =
|
|
381
|
+
<str Constant>` shape, so `_assign_literal` correctly refuses it; left
|
|
382
|
+
unfiltered, that refusal then kills a closure the ssa-only L3 set
|
|
383
|
+
resolved cleanly, breaking the `-a 3 ⊆ -a 4` monotonicity contract. A
|
|
384
|
+
bare local can only ever be rebound by its own name, so widening past
|
|
385
|
+
ssa here is never sound-adding, only noise.
|
|
386
|
+
"""
|
|
387
|
+
use_node = c.body.get(use_local_id)
|
|
388
|
+
if use_node is None or use_node.span is None:
|
|
389
|
+
return None
|
|
390
|
+
lo, hi = use_node.span.bytes
|
|
391
|
+
literals: Set[str] = set()
|
|
392
|
+
reached = False
|
|
393
|
+
for edge in c.ddg:
|
|
394
|
+
if edge.var != var or "ssa" not in (edge.prov or []):
|
|
395
|
+
continue
|
|
396
|
+
dst_node = c.body.get(edge.dst)
|
|
397
|
+
if dst_node is None or dst_node.span is None:
|
|
398
|
+
continue
|
|
399
|
+
d_lo, d_hi = dst_node.span.bytes
|
|
400
|
+
if not (d_lo <= lo and hi <= d_hi):
|
|
401
|
+
continue # this DDG use is some other reference to `var`, not this call's
|
|
402
|
+
reached = True
|
|
403
|
+
literal = _assign_literal(source, c.body.get(edge.src))
|
|
404
|
+
if literal is None:
|
|
405
|
+
return None # any non-closing reaching def kills resolution
|
|
406
|
+
literals.add(literal)
|
|
407
|
+
if not reached or len(literals) != 1:
|
|
408
|
+
return None
|
|
409
|
+
return next(iter(literals))
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
def dataflow_intra_tier(reads: List[_Read], app: PyApplication) -> Tuple[List[PyConfigUseEdge], List[_Read]]:
|
|
413
|
+
"""L3 tier: close a non-literal key argument over its own callable's DDG
|
|
414
|
+
(`_reaching_literal`), then resolve the closed literal against
|
|
415
|
+
`PyArtifact.config_keys` exactly like the literal tier does."""
|
|
416
|
+
index = _index_callables(app)
|
|
417
|
+
keys_by_namespace = _keys_by_namespace(app)
|
|
418
|
+
edges: List[PyConfigUseEdge] = []
|
|
419
|
+
unresolved: List[_Read] = []
|
|
420
|
+
for read in reads:
|
|
421
|
+
entry = index.get(read.callable_id)
|
|
422
|
+
if read.key_name is None or entry is None:
|
|
423
|
+
unresolved.append(read)
|
|
424
|
+
continue
|
|
425
|
+
c, source = entry
|
|
426
|
+
literal = _reaching_literal(c, source, read.local_id, read.key_name)
|
|
427
|
+
if literal is None:
|
|
428
|
+
unresolved.append(read)
|
|
429
|
+
continue
|
|
430
|
+
matched = _resolve_literal_against_keys(read.rule, literal, keys_by_namespace)
|
|
431
|
+
if not matched:
|
|
432
|
+
unresolved.append(replace(read, key_literal=literal))
|
|
433
|
+
continue
|
|
434
|
+
for key in matched:
|
|
435
|
+
edges.append(PyConfigUseEdge(src=read.site, dst=key.id, prov=["dataflow"]))
|
|
436
|
+
return edges, unresolved
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def _call_sites_targeting(
|
|
440
|
+
app: PyApplication, index: Dict[str, Tuple[PyCallable, str]], target_id: str,
|
|
441
|
+
) -> Tuple[List[Tuple[PyCallable, str, str, BodyNode]], bool]:
|
|
442
|
+
"""Every `(caller, caller_source, local_id, call_node)` whose `callee`
|
|
443
|
+
is `target_id` -- the call-graph join (which callables call it) narrowed
|
|
444
|
+
to the actual call-site body nodes (which arguments they pass), sorted
|
|
445
|
+
for deterministic iteration.
|
|
446
|
+
|
|
447
|
+
The second return value is completeness: False
|
|
448
|
+
when some `call_graph` caller of `target_id` can't be accounted for in
|
|
449
|
+
`sites` -- either it never resolves in `index` (a module-scope caller's
|
|
450
|
+
`src` is a `can://.../@external/<module>` id, #131 modeling --
|
|
451
|
+
`_index_callables` only ever holds declared callables) or it resolves
|
|
452
|
+
but contributes zero matching call-site body nodes. Either way, "every
|
|
453
|
+
site agrees" computed only over `sites` would silently speak for a
|
|
454
|
+
caller it never actually saw."""
|
|
455
|
+
caller_ids = sorted({e.src for e in app.call_graph if e.dst == target_id})
|
|
456
|
+
sites: List[Tuple[PyCallable, str, str, BodyNode]] = []
|
|
457
|
+
complete = True
|
|
458
|
+
for caller_id in caller_ids:
|
|
459
|
+
entry = index.get(caller_id)
|
|
460
|
+
if entry is None:
|
|
461
|
+
complete = False
|
|
462
|
+
continue
|
|
463
|
+
caller, source = entry
|
|
464
|
+
seen_here = False
|
|
465
|
+
for local_id, node in sorted((caller.body or {}).items()):
|
|
466
|
+
if node.kind == "call" and node.callee == target_id:
|
|
467
|
+
sites.append((caller, source, local_id, node))
|
|
468
|
+
seen_here = True
|
|
469
|
+
if not seen_here:
|
|
470
|
+
complete = False
|
|
471
|
+
return sites, complete
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
def _site_literal(
|
|
475
|
+
node: BodyNode, param_index: int, caller: PyCallable, source: str, local_id: str, visited: Set[str],
|
|
476
|
+
) -> Optional[str]:
|
|
477
|
+
"""This call site's contribution to closing the callee's parameter: the
|
|
478
|
+
str literal passed directly at `param_index`, or -- when that argument
|
|
479
|
+
is itself a bare Name -- one non-recursive hop of `_reaching_literal` at
|
|
480
|
+
the *caller*'s own call site (guarded by `visited` against the direct
|
|
481
|
+
self-recursive-call case). Only positional args reach `BodyNode.arguments`
|
|
482
|
+
(same substrate limitation `_key_from_args` documents), so a kwarg-only
|
|
483
|
+
or too-short call site contributes nothing -- `None`, same as any other
|
|
484
|
+
non-closing site, which is enough to leave the read unresolved."""
|
|
485
|
+
args = node.arguments or []
|
|
486
|
+
if param_index >= len(args):
|
|
487
|
+
return None
|
|
488
|
+
arg = args[param_index]
|
|
489
|
+
if arg.value is not None:
|
|
490
|
+
try:
|
|
491
|
+
decoded = json.loads(arg.value)
|
|
492
|
+
except json.JSONDecodeError:
|
|
493
|
+
return None
|
|
494
|
+
return decoded if isinstance(decoded, str) else None
|
|
495
|
+
if arg.name is not None and caller.id not in visited:
|
|
496
|
+
return _reaching_literal(caller, source, local_id, arg.name)
|
|
497
|
+
return None
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
_ENTRY_KINDS = {"entry", "formal_in"} # a param's implicit binding, never a rebind
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def _locally_redefined(c: PyCallable, use_local_id: str, var: str) -> bool:
|
|
504
|
+
"""True when some ssa DDG-reaching def of `var` at `use_local_id` is a
|
|
505
|
+
real statement -- not the callable's own implicit `@entry`/`@formal_in`
|
|
506
|
+
parameter binding.
|
|
507
|
+
|
|
508
|
+
An unshadowed parameter's only reaching def IS that implicit binding
|
|
509
|
+
(`access_paths.def_use_facts`: ENTRY defines every param as the
|
|
510
|
+
function's incoming state), so this is False for it -- the interproc
|
|
511
|
+
tier's caller-argument closure is safe. A parameter reassigned anywhere
|
|
512
|
+
on a path reaching the read -- even conditionally, even to a non-literal
|
|
513
|
+
value the intra tier can't close and so leaves `key_literal` at `None`
|
|
514
|
+
rather than "closed-but-undeclared" -- has at least one real statement
|
|
515
|
+
def here, and the caller's argument can no longer be assumed to be what
|
|
516
|
+
the read actually sees. Same span-containment reaching-def match
|
|
517
|
+
`_reaching_literal` uses, minus the literal-closing requirement."""
|
|
518
|
+
use_node = c.body.get(use_local_id)
|
|
519
|
+
if use_node is None or use_node.span is None:
|
|
520
|
+
return False
|
|
521
|
+
lo, hi = use_node.span.bytes
|
|
522
|
+
for edge in c.ddg:
|
|
523
|
+
if edge.var != var or "ssa" not in (edge.prov or []):
|
|
524
|
+
continue
|
|
525
|
+
dst_node = c.body.get(edge.dst)
|
|
526
|
+
if dst_node is None or dst_node.span is None:
|
|
527
|
+
continue
|
|
528
|
+
d_lo, d_hi = dst_node.span.bytes
|
|
529
|
+
if not (d_lo <= lo and hi <= d_hi):
|
|
530
|
+
continue
|
|
531
|
+
src_node = c.body.get(edge.src)
|
|
532
|
+
if src_node is not None and src_node.kind not in _ENTRY_KINDS:
|
|
533
|
+
return True
|
|
534
|
+
return False
|
|
535
|
+
|
|
536
|
+
|
|
537
|
+
def dataflow_interproc_tier(
|
|
538
|
+
reads: List[_Read], app: PyApplication,
|
|
539
|
+
) -> Tuple[List[PyConfigUseEdge], List[_Read]]:
|
|
540
|
+
"""L4 tier: a key that names a *parameter* of its enclosing callable
|
|
541
|
+
closes when the parameter is never locally redefined (`_locally_
|
|
542
|
+
redefined`) and every call site targeting that callable -- ALL of
|
|
543
|
+
them, `_call_sites_targeting`'s completeness flag -- supplies the
|
|
544
|
+
same string literal (directly, or via one hop of caller-side intra
|
|
545
|
+
closure) -- call-graph + caller argument values only, never `param_in`
|
|
546
|
+
traversal (controller ruling)."""
|
|
547
|
+
index = _index_callables(app)
|
|
548
|
+
keys_by_namespace = _keys_by_namespace(app)
|
|
549
|
+
edges: List[PyConfigUseEdge] = []
|
|
550
|
+
unresolved: List[_Read] = []
|
|
551
|
+
for read in reads:
|
|
552
|
+
entry = index.get(read.callable_id)
|
|
553
|
+
# `key_literal` already set means the intra tier closed this read to
|
|
554
|
+
# a literal that simply named no declared key (e.g. a parameter
|
|
555
|
+
# shadowed by a local reassignment the intra tier correctly traced
|
|
556
|
+
# instead) -- that is this read's real value; re-deriving one from
|
|
557
|
+
# the *callers'* arguments here would ignore the shadow and can
|
|
558
|
+
# misattribute a caller-supplied value to a name the callee never
|
|
559
|
+
# actually reads.
|
|
560
|
+
if read.key_name is None or read.key_literal is not None or entry is None:
|
|
561
|
+
unresolved.append(read)
|
|
562
|
+
continue
|
|
563
|
+
c, _source = entry
|
|
564
|
+
param_names = [p.name for p in c.parameters or []]
|
|
565
|
+
if read.key_name not in param_names:
|
|
566
|
+
unresolved.append(read)
|
|
567
|
+
continue
|
|
568
|
+
if _locally_redefined(c, read.local_id, read.key_name):
|
|
569
|
+
# Locally rebound on some path through the callable's own body --
|
|
570
|
+
# a caller's argument is not provably what the read sees.
|
|
571
|
+
unresolved.append(read)
|
|
572
|
+
continue
|
|
573
|
+
param_index = param_names.index(read.key_name)
|
|
574
|
+
sites, complete = _call_sites_targeting(app, index, c.id)
|
|
575
|
+
if not sites or not complete:
|
|
576
|
+
unresolved.append(read)
|
|
577
|
+
continue
|
|
578
|
+
visited = {c.id} # guards the direct self-recursive-call case
|
|
579
|
+
literals: Set[str] = set()
|
|
580
|
+
all_closed = True
|
|
581
|
+
for caller, source, local_id, node in sites:
|
|
582
|
+
literal = _site_literal(node, param_index, caller, source, local_id, visited)
|
|
583
|
+
if literal is None:
|
|
584
|
+
all_closed = False
|
|
585
|
+
break
|
|
586
|
+
literals.add(literal)
|
|
587
|
+
if not all_closed or len(literals) != 1:
|
|
588
|
+
unresolved.append(read)
|
|
589
|
+
continue
|
|
590
|
+
literal = next(iter(literals))
|
|
591
|
+
matched = _resolve_literal_against_keys(read.rule, literal, keys_by_namespace)
|
|
592
|
+
if not matched:
|
|
593
|
+
unresolved.append(replace(read, key_literal=literal))
|
|
594
|
+
continue
|
|
595
|
+
for key in matched:
|
|
596
|
+
edges.append(PyConfigUseEdge(src=read.site, dst=key.id, prov=["dataflow"]))
|
|
597
|
+
return edges, unresolved
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
version: 1
|
|
2
|
+
|
|
3
|
+
# Shipped config-use detector table (#162), spec §3. Each rule identifies a
|
|
4
|
+
# call whose resolved callee is `module.callable` (matched prefix-aware on
|
|
5
|
+
# `module` -- see config_use.py's module docstring for why) and which
|
|
6
|
+
# argument position carries the key. No user-extension flag yet (matches
|
|
7
|
+
# the artifact rules posture).
|
|
8
|
+
#
|
|
9
|
+
# `os.environ.__getitem__` (the `os.environ["X"]` subscript form) is
|
|
10
|
+
# deliberately absent: verified empirically that a subscript never lowers
|
|
11
|
+
# to a call body node, so there is no call this rule could ever match
|
|
12
|
+
# (config_use.py's module docstring has the finding).
|
|
13
|
+
rules:
|
|
14
|
+
- id: os.getenv
|
|
15
|
+
module: os
|
|
16
|
+
callable: getenv
|
|
17
|
+
key_arg: 0
|
|
18
|
+
namespaces: [env]
|
|
19
|
+
|
|
20
|
+
- id: os.environ.get
|
|
21
|
+
module: os.environ
|
|
22
|
+
callable: get
|
|
23
|
+
key_arg: 0
|
|
24
|
+
namespaces: [env]
|
|
25
|
+
|
|
26
|
+
- id: dotenv.get_key
|
|
27
|
+
module: dotenv
|
|
28
|
+
callable: get_key
|
|
29
|
+
key_arg: 1
|
|
30
|
+
namespaces: [env]
|
|
31
|
+
|
|
32
|
+
- id: configparser.get
|
|
33
|
+
module: configparser
|
|
34
|
+
callable: get
|
|
35
|
+
key_arg: 1
|
|
36
|
+
kwarg: option
|
|
37
|
+
namespaces: [ini, properties]
|
|
38
|
+
|
|
39
|
+
- id: configparser.getint
|
|
40
|
+
module: configparser
|
|
41
|
+
callable: getint
|
|
42
|
+
key_arg: 1
|
|
43
|
+
kwarg: option
|
|
44
|
+
namespaces: [ini, properties]
|
|
45
|
+
|
|
46
|
+
- id: configparser.getfloat
|
|
47
|
+
module: configparser
|
|
48
|
+
callable: getfloat
|
|
49
|
+
key_arg: 1
|
|
50
|
+
kwarg: option
|
|
51
|
+
namespaces: [ini, properties]
|
|
52
|
+
|
|
53
|
+
- id: configparser.getboolean
|
|
54
|
+
module: configparser
|
|
55
|
+
callable: getboolean
|
|
56
|
+
key_arg: 1
|
|
57
|
+
kwarg: option
|
|
58
|
+
namespaces: [ini, properties]
|