codeanalyzer-python 1.1.0__py3-none-any.whl → 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codeanalyzer/__main__.py +95 -123
- codeanalyzer/core.py +21 -45
- codeanalyzer/dataflow/access_paths.py +26 -4
- codeanalyzer/dataflow/builder.py +65 -1
- codeanalyzer/dataflow/identity.py +1 -1
- codeanalyzer/dataflow/pdg.py +7 -2
- codeanalyzer/dataflow/scc.py +1 -1
- codeanalyzer/entrypoints/__init__.py +3 -0
- codeanalyzer/entrypoints/detect.py +124 -0
- codeanalyzer/entrypoints/matching.py +182 -0
- codeanalyzer/entrypoints/pipeline.py +131 -0
- codeanalyzer/entrypoints/rules.py +159 -0
- codeanalyzer/entrypoints/rules.yml +88 -0
- codeanalyzer/neo4j/bolt.py +1 -1
- codeanalyzer/neo4j/project.py +85 -60
- codeanalyzer/neo4j/schema.py +35 -34
- codeanalyzer/options/__init__.py +2 -2
- codeanalyzer/options/options.py +2 -26
- codeanalyzer/schema/__init__.py +48 -0
- codeanalyzer/schema/l1_body.py +11 -1
- codeanalyzer/schema/l2_callees.py +29 -13
- codeanalyzer/schema/py_schema.py +95 -103
- codeanalyzer/semantic_analysis/call_graph.py +20 -4
- codeanalyzer/semantic_analysis/defuse_linker.py +1499 -0
- codeanalyzer/syntactic_analysis/symbol_table_builder.py +88 -3
- {codeanalyzer_python-1.1.0.dist-info → codeanalyzer_python-1.2.0.dist-info}/METADATA +36 -161
- {codeanalyzer_python-1.1.0.dist-info → codeanalyzer_python-1.2.0.dist-info}/RECORD +31 -30
- {codeanalyzer_python-1.1.0.dist-info → codeanalyzer_python-1.2.0.dist-info}/WHEEL +1 -1
- codeanalyzer/config/__init__.py +0 -3
- codeanalyzer/config/config.py +0 -8
- codeanalyzer/semantic_analysis/pycg/__init__.py +0 -20
- codeanalyzer/semantic_analysis/pycg/pycg_analysis.py +0 -1115
- codeanalyzer/semantic_analysis/pycg/pycg_exceptions.py +0 -23
- codeanalyzer/semantic_analysis/pycg/shard_planner.py +0 -401
- {codeanalyzer_python-1.1.0.dist-info → codeanalyzer_python-1.2.0.dist-info}/entry_points.txt +0 -0
- {codeanalyzer_python-1.1.0.dist-info → codeanalyzer_python-1.2.0.dist-info}/licenses/LICENSE +0 -0
- {codeanalyzer_python-1.1.0.dist-info → codeanalyzer_python-1.2.0.dist-info}/licenses/NOTICE +0 -0
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""Stage 0: which frameworks is this project actually using? (#27)
|
|
2
|
+
|
|
3
|
+
Gates every later stage, so a project without Celery never pays for Celery
|
|
4
|
+
rules and cannot false-positive on a locally-defined ``shared_task``. A
|
|
5
|
+
package counts as present if first-party source imports it OR the dependency
|
|
6
|
+
manifest names it -- either is sufficient, since an import may be dynamic.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Optional, Set
|
|
13
|
+
|
|
14
|
+
from codeanalyzer.entrypoints.rules import RuleSet
|
|
15
|
+
from codeanalyzer.schema.py_schema import PyApplication
|
|
16
|
+
|
|
17
|
+
_REQ = re.compile(r"^\s*['\"]?([A-Za-z0-9_.\-]+)")
|
|
18
|
+
_DEPS_START = re.compile(r"dependencies\s*=\s*\[")
|
|
19
|
+
_TABLE_HEADER = re.compile(r"(?m)^[ \t]*\[")
|
|
20
|
+
_PKG = re.compile(r"['\"]([A-Za-z0-9][A-Za-z0-9_.\-]*)")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def detected_frameworks(app: PyApplication, project_dir: Path, rules: RuleSet) -> Set[str]:
|
|
24
|
+
# `present` (imports, manifest names) and `detect:` values are both
|
|
25
|
+
# lowercased before comparison -- manifest names were already lowercased
|
|
26
|
+
# (PyPI/pip is case-insensitive) but imports and `detect:` were not, so
|
|
27
|
+
# a `detect: [Flask]` user rule silently never matched a `flask` import.
|
|
28
|
+
present = _imported_packages(app) | _manifest_packages(project_dir)
|
|
29
|
+
return {
|
|
30
|
+
name
|
|
31
|
+
for name, fw in rules.frameworks.items()
|
|
32
|
+
if any(pkg.lower() in present for pkg in (fw.detect or [name]))
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _imported_packages(app: PyApplication) -> Set[str]:
|
|
37
|
+
out: Set[str] = set()
|
|
38
|
+
for mod in app.symbol_table.values():
|
|
39
|
+
for imp in mod.imports or []:
|
|
40
|
+
# `from flask import Flask` puts the package in `module`, not `name`.
|
|
41
|
+
# Prefer `module`; fall back to `name` for a bare `import flask`.
|
|
42
|
+
spelling = (getattr(imp, "module", "") or getattr(imp, "name", "") or "")
|
|
43
|
+
spelling = spelling.lstrip(".")
|
|
44
|
+
if spelling:
|
|
45
|
+
out.add(spelling.split(".", 1)[0].lower())
|
|
46
|
+
return out
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _manifest_packages(project_dir: Path) -> Set[str]:
|
|
50
|
+
out: Set[str] = set()
|
|
51
|
+
pyproject = project_dir / "pyproject.toml"
|
|
52
|
+
if pyproject.exists():
|
|
53
|
+
# PEP 621 `[project] dependencies = [...]` -- single- or multi-line,
|
|
54
|
+
# possibly containing nested `[...]` extras (`celery[redis]`).
|
|
55
|
+
span = _deps_array_span(_strip_comments(pyproject.read_text()))
|
|
56
|
+
if span is not None:
|
|
57
|
+
for pm in _PKG.finditer(span):
|
|
58
|
+
out.add(pm.group(1).split("[", 1)[0].lower())
|
|
59
|
+
requirements = project_dir / "requirements.txt"
|
|
60
|
+
if requirements.exists():
|
|
61
|
+
for line in requirements.read_text().splitlines():
|
|
62
|
+
m = _REQ.match(line)
|
|
63
|
+
if m:
|
|
64
|
+
out.add(m.group(1).split("[", 1)[0].lower())
|
|
65
|
+
return out
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _strip_comments(text: str) -> str:
|
|
69
|
+
"""Drop everything from an unquoted ``#`` to end of line.
|
|
70
|
+
|
|
71
|
+
# ponytail: quote tracking resets each line, so a `#` inside a
|
|
72
|
+
# triple-quoted string spanning lines could be mis-stripped. TOML
|
|
73
|
+
# dependency arrays don't use those in practice; revisit if they do.
|
|
74
|
+
"""
|
|
75
|
+
out_lines = []
|
|
76
|
+
for line in text.splitlines():
|
|
77
|
+
in_str = None
|
|
78
|
+
cut = len(line)
|
|
79
|
+
for i, ch in enumerate(line):
|
|
80
|
+
if in_str:
|
|
81
|
+
if ch == in_str:
|
|
82
|
+
in_str = None
|
|
83
|
+
elif ch in ("'", '"'):
|
|
84
|
+
in_str = ch
|
|
85
|
+
elif ch == "#":
|
|
86
|
+
cut = i
|
|
87
|
+
break
|
|
88
|
+
out_lines.append(line[:cut])
|
|
89
|
+
return "\n".join(out_lines)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _deps_array_span(text: str) -> Optional[str]:
|
|
93
|
+
"""Return the contents between the `dependencies = [` and its matching
|
|
94
|
+
`]`, counting bracket depth so a nested `[...]` (extras, e.g.
|
|
95
|
+
`celery[redis]`) doesn't close the span early.
|
|
96
|
+
|
|
97
|
+
Bounded by the next TOML table header (a `[` starting a line): if the
|
|
98
|
+
array never closes before then, it's unterminated (truncated/corrupt
|
|
99
|
+
file) and this returns None rather than harvesting quoted strings out
|
|
100
|
+
of whatever table follows.
|
|
101
|
+
"""
|
|
102
|
+
m = _DEPS_START.search(text)
|
|
103
|
+
if not m:
|
|
104
|
+
return None
|
|
105
|
+
boundary = _TABLE_HEADER.search(text, m.end())
|
|
106
|
+
limit = boundary.start() if boundary else len(text)
|
|
107
|
+
depth = 1
|
|
108
|
+
in_str = None
|
|
109
|
+
i = m.end()
|
|
110
|
+
while i < limit and depth > 0:
|
|
111
|
+
ch = text[i]
|
|
112
|
+
if in_str:
|
|
113
|
+
if ch == in_str:
|
|
114
|
+
in_str = None
|
|
115
|
+
elif ch in ("'", '"'):
|
|
116
|
+
in_str = ch
|
|
117
|
+
elif ch == "[":
|
|
118
|
+
depth += 1
|
|
119
|
+
elif ch == "]":
|
|
120
|
+
depth -= 1
|
|
121
|
+
i += 1
|
|
122
|
+
if depth != 0:
|
|
123
|
+
return None
|
|
124
|
+
return text[m.end() : i - 1]
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
"""Stage 3: match rules against decorators and base classes (#27).
|
|
2
|
+
|
|
3
|
+
Matching is on ``PyDecorator.qualified_name`` -- never the written spelling --
|
|
4
|
+
so ``@route`` under ``from flask import route`` hits the same rule as
|
|
5
|
+
``@app.route``. An unresolved decorator (``qualified_name is None``) never
|
|
6
|
+
matches: under-approximate rather than guess.
|
|
7
|
+
|
|
8
|
+
Pattern grammar: ``{a,b}`` alternation (not nested) and a trailing/embedded
|
|
9
|
+
``*`` that matches module MEMBERS only -- it does not cross a ``.``, so
|
|
10
|
+
``rest_framework.viewsets.*`` matches ``ModelViewSet`` but not
|
|
11
|
+
``viewsets.mixins.ListModelMixin``. Everything else is literal.
|
|
12
|
+
``validate_pattern`` rejects anything outside this grammar (unbalanced or
|
|
13
|
+
nested ``{``) so a typo in a rules file is a load-time ``RulesError``
|
|
14
|
+
(enforced by ``rules.py``), never a crash mid-analysis.
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import ast
|
|
19
|
+
import re
|
|
20
|
+
from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Optional, Tuple
|
|
21
|
+
|
|
22
|
+
from codeanalyzer.schema.py_schema import PyEntrypoint
|
|
23
|
+
|
|
24
|
+
if TYPE_CHECKING:
|
|
25
|
+
from codeanalyzer.entrypoints.rules import BaseRule, DecoratorRule
|
|
26
|
+
|
|
27
|
+
# Dispatch names that are HTTP verbs. DRF's ViewSet dispatch names
|
|
28
|
+
# (list, retrieve, create, ...) are NOT verbs and must not be emitted as such.
|
|
29
|
+
_HTTP_VERBS = {"get", "post", "put", "patch", "delete", "head", "options"}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class PatternError(ValueError):
|
|
33
|
+
"""A ``match`` pattern outside the ``{a,b}`` / ``*`` grammar `_compile` handles."""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def validate_pattern(pattern: str) -> None:
|
|
37
|
+
"""Raise ``PatternError`` for unbalanced or nested ``{``."""
|
|
38
|
+
depth = 0
|
|
39
|
+
for ch in pattern:
|
|
40
|
+
if ch == "{":
|
|
41
|
+
depth += 1
|
|
42
|
+
if depth > 1:
|
|
43
|
+
raise PatternError(f"nested '{{' is not supported: {pattern!r}")
|
|
44
|
+
elif ch == "}":
|
|
45
|
+
depth -= 1
|
|
46
|
+
if depth < 0:
|
|
47
|
+
raise PatternError(f"unmatched '}}': {pattern!r}")
|
|
48
|
+
if depth != 0:
|
|
49
|
+
raise PatternError(f"unbalanced '{{': {pattern!r}")
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def match_pattern(pattern: str, qualified_name: Optional[str]) -> bool:
|
|
53
|
+
"""``{a,b}`` alternation and trailing ``*``; everything else is literal."""
|
|
54
|
+
if not qualified_name:
|
|
55
|
+
return False
|
|
56
|
+
return re.fullmatch(_compile(pattern), qualified_name) is not None
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _compile(pattern: str) -> str:
|
|
60
|
+
validate_pattern(pattern)
|
|
61
|
+
out, i = [], 0
|
|
62
|
+
while i < len(pattern):
|
|
63
|
+
ch = pattern[i]
|
|
64
|
+
if ch == "{":
|
|
65
|
+
j = pattern.index("}", i)
|
|
66
|
+
alts = pattern[i + 1 : j].split(",")
|
|
67
|
+
out.append("(?:" + "|".join(re.escape(a.strip()) for a in alts) + ")")
|
|
68
|
+
i = j + 1
|
|
69
|
+
elif ch == "*":
|
|
70
|
+
out.append(r"[^.\s]*")
|
|
71
|
+
i += 1
|
|
72
|
+
else:
|
|
73
|
+
out.append(re.escape(ch))
|
|
74
|
+
i += 1
|
|
75
|
+
return "".join(out)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _literal(text: Optional[str]) -> Any:
|
|
79
|
+
"""Best-effort: decorator arguments are unparsed source fragments."""
|
|
80
|
+
if text is None:
|
|
81
|
+
return None
|
|
82
|
+
try:
|
|
83
|
+
return ast.literal_eval(text)
|
|
84
|
+
except (ValueError, SyntaxError):
|
|
85
|
+
return None
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _route_of(dec, spec: Optional[Dict[str, Any]]) -> Optional[str]:
|
|
89
|
+
if not spec or spec.get("from") != "positional":
|
|
90
|
+
return None
|
|
91
|
+
args = dec.positional_arguments or []
|
|
92
|
+
idx = int(spec.get("index", 0))
|
|
93
|
+
if idx >= len(args):
|
|
94
|
+
return None
|
|
95
|
+
value = _literal(args[idx])
|
|
96
|
+
return value if isinstance(value, str) else None
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _methods_of(dec, spec: Optional[Dict[str, Any]]) -> List[str]:
|
|
100
|
+
if not spec:
|
|
101
|
+
return []
|
|
102
|
+
source = spec.get("from")
|
|
103
|
+
if source == "match_suffix":
|
|
104
|
+
verb = (dec.qualified_name or "").rsplit(".", 1)[-1]
|
|
105
|
+
return [verb.upper()]
|
|
106
|
+
if source == "keyword":
|
|
107
|
+
raw = (dec.keyword_arguments or {}).get(spec.get("name", ""))
|
|
108
|
+
value = _literal(raw)
|
|
109
|
+
if isinstance(value, (list, tuple)):
|
|
110
|
+
return [str(v).upper() for v in value]
|
|
111
|
+
return [str(v).upper() for v in (spec.get("default") or [])]
|
|
112
|
+
return []
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def entrypoints_from_decorators(
|
|
116
|
+
node, framework: str, rules: Iterable["DecoratorRule"]
|
|
117
|
+
) -> List[PyEntrypoint]:
|
|
118
|
+
out: List[PyEntrypoint] = []
|
|
119
|
+
for dec in getattr(node, "decorators", []) or []:
|
|
120
|
+
for rule in rules:
|
|
121
|
+
if not match_pattern(rule.match, dec.qualified_name):
|
|
122
|
+
continue
|
|
123
|
+
out.append(
|
|
124
|
+
PyEntrypoint(
|
|
125
|
+
framework=framework,
|
|
126
|
+
confidence=rule.confidence,
|
|
127
|
+
rule=rule.id,
|
|
128
|
+
ruleset=rule.origin,
|
|
129
|
+
evidence=dec.qualified_name,
|
|
130
|
+
route=_route_of(dec, rule.route),
|
|
131
|
+
http_methods=_methods_of(dec, rule.methods),
|
|
132
|
+
)
|
|
133
|
+
)
|
|
134
|
+
return out
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def entrypoints_from_bases(
|
|
138
|
+
cls,
|
|
139
|
+
framework: str,
|
|
140
|
+
rules: Iterable["BaseRule"],
|
|
141
|
+
resolve: Callable[[str], Optional[str]],
|
|
142
|
+
) -> Tuple[List[PyEntrypoint], Dict[str, List[PyEntrypoint]]]:
|
|
143
|
+
"""Records for a routed class and for the methods the framework dispatches.
|
|
144
|
+
|
|
145
|
+
``resolve`` maps a written base-class name to its resolved qualified name
|
|
146
|
+
(identity when already qualified). Dispatch names are intersected with the
|
|
147
|
+
methods the class actually defines, so a ``ListView`` with only ``get``
|
|
148
|
+
gains no phantom ``post`` entrypoint.
|
|
149
|
+
"""
|
|
150
|
+
class_eps: List[PyEntrypoint] = []
|
|
151
|
+
method_eps: Dict[str, List[PyEntrypoint]] = {}
|
|
152
|
+
|
|
153
|
+
for rule in rules:
|
|
154
|
+
if not any(
|
|
155
|
+
match_pattern(rule.match, resolve(b) or b) for b in (cls.base_classes or [])
|
|
156
|
+
):
|
|
157
|
+
continue
|
|
158
|
+
class_eps.append(
|
|
159
|
+
PyEntrypoint(
|
|
160
|
+
framework=framework,
|
|
161
|
+
confidence=rule.confidence,
|
|
162
|
+
rule=rule.id,
|
|
163
|
+
ruleset=rule.origin,
|
|
164
|
+
evidence=cls.signature,
|
|
165
|
+
)
|
|
166
|
+
)
|
|
167
|
+
defined = set((cls.callables or {}).keys())
|
|
168
|
+
for name in rule.dispatch:
|
|
169
|
+
if name not in defined:
|
|
170
|
+
continue
|
|
171
|
+
method_eps.setdefault(name, []).append(
|
|
172
|
+
PyEntrypoint(
|
|
173
|
+
framework=framework,
|
|
174
|
+
confidence=rule.confidence,
|
|
175
|
+
rule=f"{rule.id}.dispatch",
|
|
176
|
+
ruleset=rule.origin,
|
|
177
|
+
evidence=cls.signature,
|
|
178
|
+
http_methods=[name.upper()] if name in _HTTP_VERBS else [],
|
|
179
|
+
via=cls.id or None,
|
|
180
|
+
)
|
|
181
|
+
)
|
|
182
|
+
return class_eps, method_eps
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"""Entrypoint detection: a post-pass over the built L1 symbol table (#27).
|
|
2
|
+
|
|
3
|
+
Runs AFTER the symbol table exists so every view reference resolves as a
|
|
4
|
+
lookup against ids that already exist. Additive metadata: a failure here
|
|
5
|
+
loses flags, never the analysis.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Dict, Iterable, Iterator
|
|
11
|
+
|
|
12
|
+
from codeanalyzer.entrypoints.detect import detected_frameworks
|
|
13
|
+
from codeanalyzer.entrypoints.matching import entrypoints_from_bases, entrypoints_from_decorators
|
|
14
|
+
from codeanalyzer.entrypoints.rules import RuleSet, load_rules
|
|
15
|
+
from codeanalyzer.schema.py_schema import PyApplication, PyCallable, PyClass, PyModule
|
|
16
|
+
from codeanalyzer.utils import logger
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def detect_entrypoints(
|
|
20
|
+
app: PyApplication, project_dir: Path, rule_paths: Iterable[Path] = ()
|
|
21
|
+
) -> None:
|
|
22
|
+
"""Populate ``entrypoints`` on every callable and class, in place.
|
|
23
|
+
|
|
24
|
+
Loading the rules is a CONFIGURATION step, not a detection step: a
|
|
25
|
+
malformed user rules file is a hard error that must stop the run before
|
|
26
|
+
analysis starts, so ``load_rules`` runs outside (and before) the
|
|
27
|
+
try/except below. Everything after that -- the actual framework
|
|
28
|
+
detection -- is best-effort and must never abort the analysis.
|
|
29
|
+
"""
|
|
30
|
+
rules = load_rules(rule_paths)
|
|
31
|
+
try:
|
|
32
|
+
_run_stages(app, project_dir, rules)
|
|
33
|
+
except Exception as exc: # noqa: BLE001 - additive pass must never abort analysis
|
|
34
|
+
logger.warning("entrypoint detection failed: %s", exc)
|
|
35
|
+
app.entrypoint_report.errors.append(str(exc))
|
|
36
|
+
_derive_flags(app)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _run_stages(app: PyApplication, project_dir: Path, rules: RuleSet) -> None:
|
|
40
|
+
"""Stages 0-4. Stage 0 (framework detection) and Stage 3 (decorator and
|
|
41
|
+
base-class matching) exist so far.
|
|
42
|
+
|
|
43
|
+
Base-class resolution needs the OWNING MODULE's import table (a written
|
|
44
|
+
base like ``APIView`` only resolves via that module's own
|
|
45
|
+
``from rest_framework.views import APIView``), so this walks module by
|
|
46
|
+
module rather than the whole app flat, building one resolver per module.
|
|
47
|
+
|
|
48
|
+
Clears every node's ``entrypoints`` first: on a warm cache,
|
|
49
|
+
``_build_symbol_table`` reuses the SAME cached ``PyModule``/``PyCallable``
|
|
50
|
+
objects when a file is unchanged, so without this clear a second run
|
|
51
|
+
would ``extend`` onto records already written by the first run and
|
|
52
|
+
duplicate them. A single full clear up front (rather than clearing each
|
|
53
|
+
node as it's visited) avoids wiping ``entrypoints_from_bases`` records
|
|
54
|
+
that ``_walk_module`` writes onto a method before visiting that method
|
|
55
|
+
directly.
|
|
56
|
+
"""
|
|
57
|
+
for node in _walk(app):
|
|
58
|
+
node.entrypoints = []
|
|
59
|
+
|
|
60
|
+
app.entrypoint_report.rulesets = list(rules.rulesets)
|
|
61
|
+
frameworks = detected_frameworks(app, project_dir, rules)
|
|
62
|
+
app.entrypoint_report.frameworks_detected = sorted(frameworks)
|
|
63
|
+
|
|
64
|
+
names = sorted(frameworks)
|
|
65
|
+
for mod in app.symbol_table.values():
|
|
66
|
+
resolve = _base_resolver(mod)
|
|
67
|
+
for node in _walk_module(mod):
|
|
68
|
+
for name in names:
|
|
69
|
+
fw = rules.frameworks[name]
|
|
70
|
+
node.entrypoints.extend(entrypoints_from_decorators(node, name, fw.decorators))
|
|
71
|
+
if isinstance(node, PyClass) and fw.bases:
|
|
72
|
+
class_eps, method_eps = entrypoints_from_bases(
|
|
73
|
+
node, name, fw.bases, resolve
|
|
74
|
+
)
|
|
75
|
+
node.entrypoints.extend(class_eps)
|
|
76
|
+
for method_name, eps in method_eps.items():
|
|
77
|
+
target = (node.callables or {}).get(method_name)
|
|
78
|
+
if target is not None:
|
|
79
|
+
target.entrypoints.extend(eps)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _base_resolver(mod: PyModule):
|
|
83
|
+
"""A per-module ``resolve`` callable for ``entrypoints_from_bases``, built
|
|
84
|
+
from the module's own import table -- exact data already on the node,
|
|
85
|
+
never a Jedi guess. Covers ``from x.y import Z[ as W]`` and
|
|
86
|
+
``import x.y[ as z]``, plus a dotted base (``views.APIView``) whose head
|
|
87
|
+
is the imported name. A base the import table has no mapping for is
|
|
88
|
+
returned unchanged -- under-approximate rather than guess.
|
|
89
|
+
"""
|
|
90
|
+
aliases: Dict[str, str] = {}
|
|
91
|
+
for imp in mod.imports or []:
|
|
92
|
+
original = imp.alias or imp.name
|
|
93
|
+
aliases[imp.name] = imp.module if imp.module == original else f"{imp.module}.{original}"
|
|
94
|
+
|
|
95
|
+
def resolve(written: str) -> str:
|
|
96
|
+
head, _, rest = written.partition(".")
|
|
97
|
+
target = aliases.get(head)
|
|
98
|
+
return f"{target}.{rest}" if target and rest else (target or written)
|
|
99
|
+
|
|
100
|
+
return resolve
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _derive_flags(app: PyApplication) -> None:
|
|
104
|
+
for node in _walk(app):
|
|
105
|
+
node.is_entrypoint = bool(node.entrypoints)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _walk_module(mod: PyModule) -> Iterator[object]:
|
|
109
|
+
def walk_callable(c: PyCallable) -> Iterator[object]:
|
|
110
|
+
yield c
|
|
111
|
+
for inner in (c.callables or {}).values():
|
|
112
|
+
yield from walk_callable(inner)
|
|
113
|
+
for cls in (c.types or {}).values():
|
|
114
|
+
yield from walk_class(cls)
|
|
115
|
+
|
|
116
|
+
def walk_class(k: PyClass) -> Iterator[object]:
|
|
117
|
+
yield k
|
|
118
|
+
for m in (k.callables or {}).values():
|
|
119
|
+
yield from walk_callable(m)
|
|
120
|
+
for inner in (k.types or {}).values():
|
|
121
|
+
yield from walk_class(inner)
|
|
122
|
+
|
|
123
|
+
for fn in (mod.functions or {}).values():
|
|
124
|
+
yield from walk_callable(fn)
|
|
125
|
+
for cls in (mod.types or {}).values():
|
|
126
|
+
yield from walk_class(cls)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _walk(app: PyApplication) -> Iterator[object]:
|
|
130
|
+
for mod in app.symbol_table.values():
|
|
131
|
+
yield from _walk_module(mod)
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
"""Loading and merging of entrypoint rules (#27).
|
|
2
|
+
|
|
3
|
+
The shipped ``rules.yml`` covers known frameworks; users extend it with
|
|
4
|
+
``--entrypoint-rules``. User rules merge additively and may ``disable:`` a
|
|
5
|
+
shipped rule by id. A malformed user file is a hard error before analysis
|
|
6
|
+
starts -- silently skipping it would let someone ship rules they believe
|
|
7
|
+
are live.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any, Dict, Iterable, List, Optional
|
|
14
|
+
|
|
15
|
+
import yaml
|
|
16
|
+
|
|
17
|
+
from codeanalyzer.entrypoints.matching import PatternError, validate_pattern
|
|
18
|
+
|
|
19
|
+
_SHIPPED = Path(__file__).with_name("rules.yml")
|
|
20
|
+
_CONFIDENCE = {"declared", "certain", "heuristic"}
|
|
21
|
+
# `declared:` (readers) and per-framework routing engines are real spec
|
|
22
|
+
# blocks (Units 4-5) not implemented yet; they are deliberately absent here
|
|
23
|
+
# rather than accepted-and-ignored, so a user file using them fails loudly
|
|
24
|
+
# instead of loading clean and doing nothing.
|
|
25
|
+
_TOP_LEVEL_KEYS = {"version", "frameworks", "disable"}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class RulesError(Exception):
|
|
29
|
+
"""Raised for a malformed rules file. Never swallowed."""
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass
|
|
33
|
+
class DecoratorRule:
|
|
34
|
+
id: str
|
|
35
|
+
match: str
|
|
36
|
+
confidence: str = "certain"
|
|
37
|
+
route: Optional[Dict[str, Any]] = None
|
|
38
|
+
methods: Optional[Dict[str, Any]] = None
|
|
39
|
+
origin: str = "shipped"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass
|
|
43
|
+
class BaseRule:
|
|
44
|
+
id: str
|
|
45
|
+
match: str
|
|
46
|
+
confidence: str = "certain"
|
|
47
|
+
transitive: bool = False
|
|
48
|
+
dispatch: List[str] = field(default_factory=list)
|
|
49
|
+
origin: str = "shipped"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass
|
|
53
|
+
class Framework:
|
|
54
|
+
name: str
|
|
55
|
+
detect: List[str] = field(default_factory=list)
|
|
56
|
+
decorators: List[DecoratorRule] = field(default_factory=list)
|
|
57
|
+
bases: List[BaseRule] = field(default_factory=list)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@dataclass
|
|
61
|
+
class RuleSet:
|
|
62
|
+
frameworks: Dict[str, Framework] = field(default_factory=dict)
|
|
63
|
+
rulesets: List[str] = field(default_factory=list)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def load_rules(user_paths: Iterable[Path] = ()) -> RuleSet:
|
|
67
|
+
out = RuleSet()
|
|
68
|
+
_merge(out, _read(_SHIPPED), "shipped")
|
|
69
|
+
for p in user_paths:
|
|
70
|
+
_merge(out, _read(Path(p)), f"user:{p}")
|
|
71
|
+
return out
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _read(path: Path) -> Dict[str, Any]:
|
|
75
|
+
try:
|
|
76
|
+
data = yaml.safe_load(path.read_text())
|
|
77
|
+
except FileNotFoundError as exc:
|
|
78
|
+
raise RulesError(f"rules file not found: {path}") from exc
|
|
79
|
+
except yaml.YAMLError as exc:
|
|
80
|
+
raise RulesError(f"{path}: invalid YAML: {exc}") from exc
|
|
81
|
+
if not isinstance(data, dict):
|
|
82
|
+
raise RulesError(f"{path}: top level must be a mapping")
|
|
83
|
+
return data
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _merge(out: RuleSet, data: Dict[str, Any], origin: str) -> None:
|
|
87
|
+
unknown = sorted(set(data) - _TOP_LEVEL_KEYS)
|
|
88
|
+
if unknown:
|
|
89
|
+
raise RulesError(f"{origin}: unknown top-level key(s): {', '.join(unknown)}")
|
|
90
|
+
out.rulesets.append(origin)
|
|
91
|
+
disabled = set(_disable_list(data, origin))
|
|
92
|
+
frameworks = data.get("frameworks") or {}
|
|
93
|
+
if not isinstance(frameworks, dict):
|
|
94
|
+
raise RulesError(f"{origin}: `frameworks` must be a mapping")
|
|
95
|
+
|
|
96
|
+
for name, body in frameworks.items():
|
|
97
|
+
if not isinstance(body, dict):
|
|
98
|
+
raise RulesError(f"{origin}: framework `{name}` must be a mapping")
|
|
99
|
+
fw = out.frameworks.setdefault(name, Framework(name=name))
|
|
100
|
+
fw.detect = sorted(set(fw.detect) | set(body.get("detect") or []))
|
|
101
|
+
for raw in body.get("decorators") or []:
|
|
102
|
+
fw.decorators.append(_decorator_rule(raw, origin))
|
|
103
|
+
for raw in body.get("bases") or []:
|
|
104
|
+
fw.bases.append(_base_rule(raw, origin))
|
|
105
|
+
|
|
106
|
+
for fw in out.frameworks.values():
|
|
107
|
+
fw.decorators = [r for r in fw.decorators if r.id not in disabled]
|
|
108
|
+
fw.bases = [r for r in fw.bases if r.id not in disabled]
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _disable_list(data: Dict[str, Any], origin: str) -> List[str]:
|
|
112
|
+
raw = data.get("disable") or []
|
|
113
|
+
if not isinstance(raw, list) or not all(isinstance(x, str) for x in raw):
|
|
114
|
+
raise RulesError(f"{origin}: `disable` must be a list of rule id strings")
|
|
115
|
+
return raw
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _require(raw: Dict[str, Any], key: str, origin: str) -> Any:
|
|
119
|
+
if key not in raw:
|
|
120
|
+
raise RulesError(f"{origin}: rule {raw!r} is missing `{key}`")
|
|
121
|
+
return raw[key]
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _confidence(raw: Dict[str, Any], origin: str) -> str:
|
|
125
|
+
c = raw.get("confidence", "certain")
|
|
126
|
+
if c not in _CONFIDENCE:
|
|
127
|
+
raise RulesError(f"{origin}: confidence must be one of {sorted(_CONFIDENCE)}, got {c!r}")
|
|
128
|
+
return c
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _match(raw: Dict[str, Any], origin: str) -> str:
|
|
132
|
+
match = _require(raw, "match", origin)
|
|
133
|
+
try:
|
|
134
|
+
validate_pattern(match)
|
|
135
|
+
except PatternError as exc:
|
|
136
|
+
raise RulesError(f"{origin}: rule {raw.get('id', raw)!r}: {exc}") from exc
|
|
137
|
+
return match
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _decorator_rule(raw: Dict[str, Any], origin: str) -> DecoratorRule:
|
|
141
|
+
return DecoratorRule(
|
|
142
|
+
id=_require(raw, "id", origin),
|
|
143
|
+
match=_match(raw, origin),
|
|
144
|
+
confidence=_confidence(raw, origin),
|
|
145
|
+
route=raw.get("route"),
|
|
146
|
+
methods=raw.get("methods"),
|
|
147
|
+
origin=origin,
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _base_rule(raw: Dict[str, Any], origin: str) -> BaseRule:
|
|
152
|
+
return BaseRule(
|
|
153
|
+
id=_require(raw, "id", origin),
|
|
154
|
+
match=_match(raw, origin),
|
|
155
|
+
confidence=_confidence(raw, origin),
|
|
156
|
+
transitive=bool(raw.get("transitive", False)),
|
|
157
|
+
dispatch=list(raw.get("dispatch") or []),
|
|
158
|
+
origin=origin,
|
|
159
|
+
)
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
version: 1
|
|
2
|
+
|
|
3
|
+
frameworks:
|
|
4
|
+
flask:
|
|
5
|
+
detect: [flask]
|
|
6
|
+
decorators:
|
|
7
|
+
# Flask 3's Flask/Blueprint decorators are all inherited from one base
|
|
8
|
+
# (flask.sansio.scaffold.Scaffold); matching is on Jedi's resolved
|
|
9
|
+
# DEFINITION path, not the public `flask.Flask`/`flask.Blueprint`
|
|
10
|
+
# spelling, so one rule now covers both call sites.
|
|
11
|
+
- id: flask.route
|
|
12
|
+
match: "flask.sansio.scaffold.Scaffold.route"
|
|
13
|
+
route: {from: positional, index: 0}
|
|
14
|
+
methods: {from: keyword, name: methods, default: [GET]}
|
|
15
|
+
- id: flask.bp-verb
|
|
16
|
+
match: "flask.sansio.scaffold.Scaffold.{get,post,put,delete,patch}"
|
|
17
|
+
route: {from: positional, index: 0}
|
|
18
|
+
methods: {from: match_suffix}
|
|
19
|
+
bases:
|
|
20
|
+
- id: flask.methodview
|
|
21
|
+
match: "flask.views.MethodView"
|
|
22
|
+
transitive: true
|
|
23
|
+
dispatch: [get, post, put, delete, patch]
|
|
24
|
+
|
|
25
|
+
fastapi:
|
|
26
|
+
detect: [fastapi]
|
|
27
|
+
decorators:
|
|
28
|
+
# FastAPI's own get/post/... are defined directly on the FastAPI class
|
|
29
|
+
# (fastapi/applications.py); APIRouter's are a distinct class
|
|
30
|
+
# (fastapi/routing.py) -- Jedi resolves each to its own module, so
|
|
31
|
+
# these stay two rules.
|
|
32
|
+
- id: fastapi.verb
|
|
33
|
+
match: "fastapi.applications.FastAPI.{get,post,put,delete,patch,head,options}"
|
|
34
|
+
route: {from: positional, index: 0}
|
|
35
|
+
methods: {from: match_suffix}
|
|
36
|
+
- id: fastapi.router-verb
|
|
37
|
+
match: "fastapi.routing.APIRouter.{get,post,put,delete,patch}"
|
|
38
|
+
route: {from: positional, index: 0}
|
|
39
|
+
methods: {from: match_suffix}
|
|
40
|
+
- id: fastapi.websocket
|
|
41
|
+
match: "fastapi.applications.FastAPI.websocket"
|
|
42
|
+
route: {from: positional, index: 0}
|
|
43
|
+
|
|
44
|
+
celery:
|
|
45
|
+
detect: [celery]
|
|
46
|
+
decorators:
|
|
47
|
+
- id: celery.shared-task
|
|
48
|
+
match: "celery.app.shared_task"
|
|
49
|
+
- id: celery.task
|
|
50
|
+
match: "celery.app.base.Celery.task"
|
|
51
|
+
|
|
52
|
+
click:
|
|
53
|
+
detect: [click, typer]
|
|
54
|
+
decorators:
|
|
55
|
+
- id: click.command
|
|
56
|
+
match: "click.decorators.{command,group}"
|
|
57
|
+
- id: typer.command
|
|
58
|
+
match: "typer.main.Typer.command"
|
|
59
|
+
|
|
60
|
+
drf:
|
|
61
|
+
detect: [rest_framework]
|
|
62
|
+
decorators:
|
|
63
|
+
- id: drf.api-view
|
|
64
|
+
match: "rest_framework.decorators.api_view"
|
|
65
|
+
- id: drf.action
|
|
66
|
+
match: "rest_framework.decorators.action"
|
|
67
|
+
bases:
|
|
68
|
+
- id: drf.apiview
|
|
69
|
+
match: "rest_framework.views.APIView"
|
|
70
|
+
transitive: true
|
|
71
|
+
dispatch: [get, post, put, patch, delete, head, options]
|
|
72
|
+
- id: drf.viewset
|
|
73
|
+
match: "rest_framework.viewsets.*"
|
|
74
|
+
transitive: true
|
|
75
|
+
dispatch: [list, retrieve, create, update, partial_update, destroy]
|
|
76
|
+
|
|
77
|
+
django:
|
|
78
|
+
detect: [django]
|
|
79
|
+
bases:
|
|
80
|
+
# Base-class matching resolves against the module's own import table
|
|
81
|
+
# (the WRITTEN spelling, e.g. `from django.views.generic import
|
|
82
|
+
# ListView`), never Jedi's definition path -- so the public
|
|
83
|
+
# `django.views.generic.*` path is correct here, unlike the decorator
|
|
84
|
+
# rules above.
|
|
85
|
+
- id: django.cbv
|
|
86
|
+
match: "django.views.generic.*"
|
|
87
|
+
transitive: true
|
|
88
|
+
dispatch: [get, post, put, patch, delete, head, options]
|
codeanalyzer/neo4j/bolt.py
CHANGED