patchsight 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- patchsight/__init__.py +5 -0
- patchsight/cli.py +130 -0
- patchsight/db.py +221 -0
- patchsight/eval/__init__.py +1 -0
- patchsight/eval/confidence.py +169 -0
- patchsight/eval/metrics.py +134 -0
- patchsight/eval/parity.py +96 -0
- patchsight/eval/safety.py +54 -0
- patchsight/names.py +76 -0
- patchsight/pipeline/__init__.py +4 -0
- patchsight/pipeline/__main__.py +863 -0
- patchsight/pipeline/benchmarks/__init__.py +1 -0
- patchsight/pipeline/benchmarks/bundle.py +108 -0
- patchsight/pipeline/benchmarks/changedfuncs.py +275 -0
- patchsight/pipeline/benchmarks/dunlap.py +135 -0
- patchsight/pipeline/benchmarks/extraction.py +242 -0
- patchsight/pipeline/benchmarks/replay.py +187 -0
- patchsight/pipeline/benchmarks/splits.py +121 -0
- patchsight/pipeline/build_db.py +227 -0
- patchsight/pipeline/diffs/__init__.py +1 -0
- patchsight/pipeline/diffs/commits.py +109 -0
- patchsight/pipeline/diffs/github.py +177 -0
- patchsight/pipeline/diffs/hunks.py +120 -0
- patchsight/pipeline/diffs/mapping.py +203 -0
- patchsight/pipeline/diffs/paths.py +127 -0
- patchsight/pipeline/extract/__init__.py +1 -0
- patchsight/pipeline/extract/extractor.py +156 -0
- patchsight/pipeline/extract/prompt.py +219 -0
- patchsight/pipeline/extract/schema.py +40 -0
- patchsight/pipeline/extract/validate.py +81 -0
- patchsight/pipeline/fix_sample.py +384 -0
- patchsight/pipeline/http.py +143 -0
- patchsight/pipeline/ingest/__init__.py +1 -0
- patchsight/pipeline/ingest/aliases.py +76 -0
- patchsight/pipeline/ingest/ghsa_repo.py +80 -0
- patchsight/pipeline/ingest/osv.py +253 -0
- patchsight/pipeline/ingest/references.py +189 -0
- patchsight/pipeline/ingest/snapshot.py +146 -0
- patchsight/pipeline/llm/__init__.py +1 -0
- patchsight/pipeline/llm/client.py +288 -0
- patchsight/pipeline/llm/config.py +68 -0
- patchsight/pipeline/llm/tokens.py +74 -0
- patchsight/pipeline/pypi.py +64 -0
- patchsight/pipeline/week1.py +314 -0
- patchsight/py.typed +0 -0
- patchsight/scan/__init__.py +4 -0
- patchsight/scan/ast_scanner.py +270 -0
- patchsight/scan/dbcache.py +119 -0
- patchsight/scan/lockfiles/__init__.py +1 -0
- patchsight/scan/lockfiles/requirements.py +116 -0
- patchsight/scan/osv_client.py +80 -0
- patchsight/scan/report.py +90 -0
- patchsight/scan/run.py +43 -0
- patchsight/scan/verdicts.py +178 -0
- patchsight-0.1.0.dist-info/METADATA +104 -0
- patchsight-0.1.0.dist-info/RECORD +59 -0
- patchsight-0.1.0.dist-info/WHEEL +4 -0
- patchsight-0.1.0.dist-info/entry_points.txt +3 -0
- patchsight-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
"""Week 1 data gate: measure the PyPI advisory data before building on it (proposal §15).
|
|
2
|
+
|
|
3
|
+
Three questions decide the plan:
|
|
4
|
+
|
|
5
|
+
1. What share of PyPI vulnerabilities link a fix commit? Under about 40%, the fix-commit
|
|
6
|
+
locator agent becomes essential.
|
|
7
|
+
2. What share already carry symbol data? This is the project's opening statistic.
|
|
8
|
+
3. How many new vulnerabilities arrive per day? This sizes the nightly LLM budget.
|
|
9
|
+
|
|
10
|
+
Counts are per vulnerability (alias group) unless stated, because one vulnerability often
|
|
11
|
+
appears as both a GHSA and a PYSEC record.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
import math
|
|
15
|
+
import statistics
|
|
16
|
+
from collections import Counter
|
|
17
|
+
from collections.abc import Iterable, Sequence
|
|
18
|
+
from datetime import date, datetime, timedelta
|
|
19
|
+
|
|
20
|
+
from pydantic import BaseModel, ConfigDict
|
|
21
|
+
|
|
22
|
+
from patchsight.pipeline.ingest.aliases import AliasGroup, alias_groups, first_published
|
|
23
|
+
from patchsight.pipeline.ingest.ghsa_repo import GhsaScan
|
|
24
|
+
from patchsight.pipeline.ingest.osv import Advisory, Excluded, Quarantined
|
|
25
|
+
|
|
26
|
+
GATE_THRESHOLD = 0.40
|
|
27
|
+
GITHUB_KINDS = frozenset({"github_commit", "github_pull", "github_compare"})
|
|
28
|
+
COMMIT_KINDS = frozenset({"github_commit", "gitlab_commit", "bitbucket_commit", "other_commit"})
|
|
29
|
+
# Proposal §7: ~6,000 input tokens per advisory against a 200,000-token daily cap per model.
|
|
30
|
+
TOKENS_PER_ADVISORY = 6_000
|
|
31
|
+
DAILY_TOKEN_CAP = 200_000
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class Share(BaseModel):
|
|
35
|
+
model_config = ConfigDict(frozen=True)
|
|
36
|
+
|
|
37
|
+
count: int
|
|
38
|
+
total: int
|
|
39
|
+
share: float
|
|
40
|
+
ci95_low: float
|
|
41
|
+
ci95_high: float
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class DailyRate(BaseModel):
|
|
45
|
+
model_config = ConfigDict(frozen=True)
|
|
46
|
+
|
|
47
|
+
days: int
|
|
48
|
+
total: int
|
|
49
|
+
mean: float
|
|
50
|
+
median: float
|
|
51
|
+
p90: float
|
|
52
|
+
max: int
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class Week1Report(BaseModel):
|
|
56
|
+
as_of: date
|
|
57
|
+
records_total: int
|
|
58
|
+
records_by_source: dict[str, int]
|
|
59
|
+
excluded: dict[str, int]
|
|
60
|
+
quarantined: int
|
|
61
|
+
quarantine_reasons: dict[str, int]
|
|
62
|
+
advisories_in_scope: int
|
|
63
|
+
in_scope_by_source: dict[str, int]
|
|
64
|
+
vulnerabilities: int
|
|
65
|
+
packages: int
|
|
66
|
+
fix_link_any: Share
|
|
67
|
+
fix_link_github: Share
|
|
68
|
+
fix_link_github_commit: Share
|
|
69
|
+
fix_link_strong: Share
|
|
70
|
+
fix_link_by_kind: dict[str, int]
|
|
71
|
+
records_with_fix_link_any: Share
|
|
72
|
+
symbol_data_osv: Share
|
|
73
|
+
symbol_data_osv_ids: tuple[str, ...]
|
|
74
|
+
ghsa_repo: GhsaScan | None
|
|
75
|
+
new_per_day_365: DailyRate
|
|
76
|
+
new_per_day_90: DailyRate
|
|
77
|
+
new_with_github_link_per_day_365: DailyRate
|
|
78
|
+
modified_last_7_days: int
|
|
79
|
+
modified_last_30_days: int
|
|
80
|
+
advisories_per_day_per_model: int
|
|
81
|
+
gate_metric: Share
|
|
82
|
+
gate_passed: bool
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def wilson(count: int, total: int, z: float = 1.959964) -> Share:
|
|
86
|
+
"""A proportion with its Wilson score 95% interval."""
|
|
87
|
+
if total == 0:
|
|
88
|
+
return Share(count=0, total=0, share=0.0, ci95_low=0.0, ci95_high=0.0)
|
|
89
|
+
p = count / total
|
|
90
|
+
denominator = 1 + z * z / total
|
|
91
|
+
centre = (p + z * z / (2 * total)) / denominator
|
|
92
|
+
margin = z * math.sqrt(p * (1 - p) / total + z * z / (4 * total * total)) / denominator
|
|
93
|
+
return Share(
|
|
94
|
+
count=count,
|
|
95
|
+
total=total,
|
|
96
|
+
share=p,
|
|
97
|
+
# The Wilson interval always contains p; clamp away floating-point error at 0 and 1.
|
|
98
|
+
ci95_low=max(0.0, min(p, centre - margin)),
|
|
99
|
+
ci95_high=min(1.0, max(p, centre + margin)),
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def daily_rate(days_seen: Iterable[date], start: date, end: date) -> DailyRate:
|
|
104
|
+
"""Per-day counts over ``[start, end)``, including days with no arrivals."""
|
|
105
|
+
counts = Counter(d for d in days_seen if start <= d < end)
|
|
106
|
+
span = (end - start).days
|
|
107
|
+
series = [counts.get(start + timedelta(days=i), 0) for i in range(span)]
|
|
108
|
+
ordered = sorted(series)
|
|
109
|
+
p90 = ordered[min(len(ordered) - 1, math.ceil(0.9 * len(ordered)) - 1)] if ordered else 0
|
|
110
|
+
return DailyRate(
|
|
111
|
+
days=span,
|
|
112
|
+
total=sum(series),
|
|
113
|
+
mean=statistics.fmean(series) if series else 0.0,
|
|
114
|
+
median=statistics.median(series) if series else 0.0,
|
|
115
|
+
p90=float(p90),
|
|
116
|
+
max=max(series, default=0),
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _group_has(group: AliasGroup, by_id: dict[str, Advisory], kinds: frozenset[str]) -> bool:
|
|
121
|
+
return any(c.kind in kinds for i in group.advisory_ids for c in by_id[i].fix_candidates)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _group_has_strong(group: AliasGroup, by_id: dict[str, Advisory]) -> bool:
|
|
125
|
+
"""A FIX-typed reference or a GIT range ``fixed`` commit, not just a WEB link."""
|
|
126
|
+
return any(
|
|
127
|
+
c.origin == "git_range" or c.reference_type == "FIX"
|
|
128
|
+
for i in group.advisory_ids
|
|
129
|
+
for c in by_id[i].fix_candidates
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def build_report(
|
|
134
|
+
results: Sequence[Advisory | Excluded | Quarantined],
|
|
135
|
+
as_of: datetime,
|
|
136
|
+
ghsa_scan: GhsaScan | None = None,
|
|
137
|
+
) -> Week1Report:
|
|
138
|
+
advisories = [r for r in results if isinstance(r, Advisory)]
|
|
139
|
+
excluded = [r for r in results if isinstance(r, Excluded)]
|
|
140
|
+
quarantined = [r for r in results if isinstance(r, Quarantined)]
|
|
141
|
+
by_id = {a.id: a for a in advisories}
|
|
142
|
+
groups = alias_groups(advisories)
|
|
143
|
+
total_groups = len(groups)
|
|
144
|
+
|
|
145
|
+
def share(predicate_kinds: frozenset[str]) -> Share:
|
|
146
|
+
return wilson(sum(_group_has(g, by_id, predicate_kinds) for g in groups), total_groups)
|
|
147
|
+
|
|
148
|
+
all_kinds = GITHUB_KINDS | COMMIT_KINDS | {"gitlab_merge_request"}
|
|
149
|
+
kind_counts: Counter[str] = Counter()
|
|
150
|
+
for group in groups:
|
|
151
|
+
kinds = {c.kind for i in group.advisory_ids for c in by_id[i].fix_candidates}
|
|
152
|
+
kind_counts.update(kinds)
|
|
153
|
+
|
|
154
|
+
symbol_groups = [g for g in groups if any(by_id[i].symbol_hints for i in g.advisory_ids)]
|
|
155
|
+
end = as_of.date()
|
|
156
|
+
published = {g.key: first_published(g, by_id) for g in groups}
|
|
157
|
+
new_days = [d.date() for d in published.values() if d is not None]
|
|
158
|
+
github_days = [
|
|
159
|
+
d.date()
|
|
160
|
+
for g in groups
|
|
161
|
+
if (d := published[g.key]) is not None and _group_has(g, by_id, GITHUB_KINDS)
|
|
162
|
+
]
|
|
163
|
+
gate_metric = share(GITHUB_KINDS)
|
|
164
|
+
return Week1Report(
|
|
165
|
+
as_of=end,
|
|
166
|
+
records_total=len(results),
|
|
167
|
+
records_by_source=dict(
|
|
168
|
+
sorted(
|
|
169
|
+
Counter(
|
|
170
|
+
r.id.split("-", 1)[0] if not isinstance(r, Quarantined) else "unreadable"
|
|
171
|
+
for r in results
|
|
172
|
+
).items()
|
|
173
|
+
)
|
|
174
|
+
),
|
|
175
|
+
excluded=dict(sorted(Counter(r.reason for r in excluded).items())),
|
|
176
|
+
quarantined=len(quarantined),
|
|
177
|
+
quarantine_reasons=dict(Counter(q.reason for q in quarantined).most_common(10)),
|
|
178
|
+
advisories_in_scope=len(advisories),
|
|
179
|
+
in_scope_by_source=dict(sorted(Counter(a.source for a in advisories).items())),
|
|
180
|
+
vulnerabilities=total_groups,
|
|
181
|
+
packages=len({p.name for a in advisories for p in a.packages}),
|
|
182
|
+
fix_link_any=share(all_kinds),
|
|
183
|
+
fix_link_github=gate_metric,
|
|
184
|
+
fix_link_github_commit=share(frozenset({"github_commit"})),
|
|
185
|
+
fix_link_strong=wilson(sum(_group_has_strong(g, by_id) for g in groups), total_groups),
|
|
186
|
+
fix_link_by_kind=dict(sorted(kind_counts.items())),
|
|
187
|
+
records_with_fix_link_any=wilson(
|
|
188
|
+
sum(bool(a.fix_candidates) for a in advisories), len(advisories)
|
|
189
|
+
),
|
|
190
|
+
symbol_data_osv=wilson(len(symbol_groups), total_groups),
|
|
191
|
+
symbol_data_osv_ids=tuple(i for g in symbol_groups for i in g.advisory_ids),
|
|
192
|
+
ghsa_repo=ghsa_scan,
|
|
193
|
+
new_per_day_365=daily_rate(new_days, end - timedelta(days=365), end),
|
|
194
|
+
new_per_day_90=daily_rate(new_days, end - timedelta(days=90), end),
|
|
195
|
+
new_with_github_link_per_day_365=daily_rate(github_days, end - timedelta(days=365), end),
|
|
196
|
+
modified_last_7_days=sum(a.modified >= as_of - timedelta(days=7) for a in advisories),
|
|
197
|
+
modified_last_30_days=sum(a.modified >= as_of - timedelta(days=30) for a in advisories),
|
|
198
|
+
advisories_per_day_per_model=DAILY_TOKEN_CAP // TOKENS_PER_ADVISORY,
|
|
199
|
+
gate_metric=gate_metric,
|
|
200
|
+
gate_passed=gate_metric.share >= GATE_THRESHOLD,
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _pct(s: Share) -> str:
|
|
205
|
+
# Small shares need more decimals, or a handful of cases reads as "0.0%".
|
|
206
|
+
fmt = ".2%" if 0 < s.share < 0.01 else ".1%"
|
|
207
|
+
ci = f"95% CI {s.ci95_low:{fmt}} to {s.ci95_high:{fmt}}"
|
|
208
|
+
return f"{s.share:{fmt}} ({s.count:,} / {s.total:,}; {ci})"
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _rate(r: DailyRate) -> str:
|
|
212
|
+
stats = f"mean {r.mean:.1f}, median {r.median:.0f}, p90 {r.p90:.0f}, max {r.max}"
|
|
213
|
+
return f"{stats} ({r.total:,} in {r.days} days)"
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def render_markdown(report: Week1Report, manifest_lines: Sequence[str]) -> str:
|
|
217
|
+
gate = "PASS" if report.gate_passed else "FAIL"
|
|
218
|
+
lines = [
|
|
219
|
+
"# Week 1 data gate",
|
|
220
|
+
"",
|
|
221
|
+
f"Measured on the OSV PyPI snapshot of {report.as_of.isoformat()}. "
|
|
222
|
+
"Generated by `python -m patchsight.pipeline week1-report`; do not edit by hand.",
|
|
223
|
+
"",
|
|
224
|
+
"## Sources",
|
|
225
|
+
"",
|
|
226
|
+
*manifest_lines,
|
|
227
|
+
"",
|
|
228
|
+
"Advisory data: OSV.dev (CC BY 4.0), aggregating the GitHub Advisory Database "
|
|
229
|
+
"(CC BY 4.0) and the PyPA Advisory Database (CC BY 4.0). This report contains "
|
|
230
|
+
"counts derived from that data, not the data itself.",
|
|
231
|
+
"",
|
|
232
|
+
"## Gate result",
|
|
233
|
+
"",
|
|
234
|
+
f"**{gate}.** Share of PyPI vulnerabilities with at least one GitHub fix link "
|
|
235
|
+
f"(commit, pull request or tag comparison): {_pct(report.gate_metric)}. "
|
|
236
|
+
f"Threshold: {GATE_THRESHOLD:.0%}. Below it, the fix-commit locator agent becomes "
|
|
237
|
+
"essential (proposal §15).",
|
|
238
|
+
"",
|
|
239
|
+
"This counts *candidate* links, so it is an upper bound. Only "
|
|
240
|
+
f"{_pct(report.fix_link_strong)} carry an explicit fix signal (a FIX-typed "
|
|
241
|
+
"reference or a GIT-range fixed commit); the rest are untyped links to commits or "
|
|
242
|
+
"pull requests. How many candidates really are the fix is measured when the diffs "
|
|
243
|
+
"are fetched.",
|
|
244
|
+
"",
|
|
245
|
+
"## Scope",
|
|
246
|
+
"",
|
|
247
|
+
f"- Records in the dump: {report.records_total:,} "
|
|
248
|
+
f"({', '.join(f'{k} {v:,}' for k, v in report.records_by_source.items())})",
|
|
249
|
+
f"- Excluded: {', '.join(f'{k} {v:,}' for k, v in report.excluded.items())}",
|
|
250
|
+
f"- Quarantined as malformed: {report.quarantined:,}",
|
|
251
|
+
f"- In-scope advisories: {report.advisories_in_scope:,} "
|
|
252
|
+
f"({', '.join(f'{k} {v:,}' for k, v in report.in_scope_by_source.items())})",
|
|
253
|
+
f"- Vulnerabilities after alias grouping: **{report.vulnerabilities:,}**, "
|
|
254
|
+
f"across {report.packages:,} packages",
|
|
255
|
+
"",
|
|
256
|
+
"Malicious-package reports (`MAL-*`) are excluded: the whole package is malware, "
|
|
257
|
+
"so there is no fix commit and no vulnerable function to find.",
|
|
258
|
+
"",
|
|
259
|
+
"## 1. Fix-commit links",
|
|
260
|
+
"",
|
|
261
|
+
"Per vulnerability (alias group):",
|
|
262
|
+
"",
|
|
263
|
+
f"- Any fix link, any host: {_pct(report.fix_link_any)}",
|
|
264
|
+
f"- GitHub commit, PR or comparison (**gate metric**): {_pct(report.fix_link_github)}",
|
|
265
|
+
f"- GitHub commit link specifically: {_pct(report.fix_link_github_commit)}",
|
|
266
|
+
f"- Strong signal only (FIX-typed reference or GIT-range fixed commit): "
|
|
267
|
+
f"{_pct(report.fix_link_strong)}",
|
|
268
|
+
f"- Per record rather than per vulnerability, any host: "
|
|
269
|
+
f"{_pct(report.records_with_fix_link_any)}",
|
|
270
|
+
"",
|
|
271
|
+
"Vulnerabilities by kind of link present (one vulnerability can have several):",
|
|
272
|
+
"",
|
|
273
|
+
"| Kind | Vulnerabilities |",
|
|
274
|
+
"|---|---|",
|
|
275
|
+
*(f"| `{k}` | {v:,} |" for k, v in report.fix_link_by_kind.items()),
|
|
276
|
+
"",
|
|
277
|
+
"A link is a *candidate*: whether it is on the package's own repository and is "
|
|
278
|
+
"really the fix is checked when the diff is fetched. For reference, 78% of Go "
|
|
279
|
+
"reports in Dunlap et al. had a patch link.",
|
|
280
|
+
"",
|
|
281
|
+
"## 2. Existing symbol data (the opening statistic)",
|
|
282
|
+
"",
|
|
283
|
+
f"- In the OSV dump (`ecosystem_specific.imports` or `affected_functions`): "
|
|
284
|
+
f"{_pct(report.symbol_data_osv)}: {', '.join(report.symbol_data_osv_ids) or 'none'}",
|
|
285
|
+
]
|
|
286
|
+
if report.ghsa_repo is not None:
|
|
287
|
+
g = report.ghsa_repo
|
|
288
|
+
lines.append(
|
|
289
|
+
f"- In `github/advisory-database` itself: {len(g.with_symbol_data):,} of "
|
|
290
|
+
f"{g.found:,} PyPI GHSA files checked carry symbol data "
|
|
291
|
+
f"({len(g.missing)} expected file(s) not found at the publication-month path)"
|
|
292
|
+
)
|
|
293
|
+
lines += [
|
|
294
|
+
"",
|
|
295
|
+
"## 3. Arrival rate",
|
|
296
|
+
"",
|
|
297
|
+
"New vulnerabilities per day, by first publication across the alias group:",
|
|
298
|
+
"",
|
|
299
|
+
f"- Last 365 days: {_rate(report.new_per_day_365)}",
|
|
300
|
+
f"- Last 90 days: {_rate(report.new_per_day_90)}",
|
|
301
|
+
"- Last 365 days, with a GitHub fix link: "
|
|
302
|
+
f"{_rate(report.new_with_github_link_per_day_365)}",
|
|
303
|
+
"",
|
|
304
|
+
f"Re-extraction pressure (lower bound, since each record shows only its latest "
|
|
305
|
+
f"modification): {report.modified_last_7_days:,} in-scope advisories modified in "
|
|
306
|
+
f"the last 7 days, {report.modified_last_30_days:,} in the last 30.",
|
|
307
|
+
"",
|
|
308
|
+
f"Budget check: at about {TOKENS_PER_ADVISORY:,} input tokens per advisory and a "
|
|
309
|
+
f"{DAILY_TOKEN_CAP:,}-token daily cap, one model handles about "
|
|
310
|
+
f"{report.advisories_per_day_per_model} advisories a day. These caps are reported "
|
|
311
|
+
"by third parties and must be confirmed in the provider console.",
|
|
312
|
+
"",
|
|
313
|
+
]
|
|
314
|
+
return "\n".join(lines)
|
patchsight/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
"""Find which imported names user code references, with Python's ``ast`` module. No LLM.
|
|
2
|
+
|
|
3
|
+
Two passes per file:
|
|
4
|
+
|
|
5
|
+
1. **Imports** build an alias table, wherever they appear in the file. ``import yaml as y``
|
|
6
|
+
makes ``y`` mean ``yaml``, and ``from yaml import load as l`` makes ``l`` mean
|
|
7
|
+
``yaml.load``.
|
|
8
|
+
2. **References**: attribute chains and names in load context are resolved through the
|
|
9
|
+
table to full dotted names such as ``yaml.load``. Only the longest chain is recorded,
|
|
10
|
+
so ``y.constructor.FullConstructor`` yields one reference, not three.
|
|
11
|
+
|
|
12
|
+
Uses the scanner cannot resolve are recorded as *dynamic* and make a verdict UNKNOWN:
|
|
13
|
+
|
|
14
|
+
* ``getattr(module, <computed name>)`` (a string constant is resolved normally);
|
|
15
|
+
* ``importlib.import_module(...)`` and ``__import__(...)``;
|
|
16
|
+
* a module object used as a plain value (passed, stored or returned);
|
|
17
|
+
* ``from package import *``.
|
|
18
|
+
|
|
19
|
+
The analysis is flow-insensitive, so an import anywhere counts for the whole file. That
|
|
20
|
+
can over-report references, which only raises an alert's priority.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
import ast
|
|
24
|
+
from collections.abc import Iterator
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
|
|
27
|
+
from pydantic import BaseModel, ConfigDict
|
|
28
|
+
|
|
29
|
+
SKIP_DIRS = frozenset(
|
|
30
|
+
{
|
|
31
|
+
".git",
|
|
32
|
+
".hg",
|
|
33
|
+
".venv",
|
|
34
|
+
"venv",
|
|
35
|
+
"env",
|
|
36
|
+
".tox",
|
|
37
|
+
".nox",
|
|
38
|
+
"node_modules",
|
|
39
|
+
"site-packages",
|
|
40
|
+
"__pycache__",
|
|
41
|
+
"build",
|
|
42
|
+
"dist",
|
|
43
|
+
".mypy_cache",
|
|
44
|
+
".pytest_cache",
|
|
45
|
+
}
|
|
46
|
+
)
|
|
47
|
+
_DYNAMIC_IMPORTERS = ("__import__", "importlib.import_module")
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class Location(BaseModel):
|
|
51
|
+
model_config = ConfigDict(frozen=True)
|
|
52
|
+
|
|
53
|
+
path: str
|
|
54
|
+
line: int
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class Reference(BaseModel):
|
|
58
|
+
model_config = ConfigDict(frozen=True)
|
|
59
|
+
|
|
60
|
+
name: str
|
|
61
|
+
location: Location
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class DynamicUse(BaseModel):
|
|
65
|
+
model_config = ConfigDict(frozen=True)
|
|
66
|
+
|
|
67
|
+
root: str
|
|
68
|
+
reason: str
|
|
69
|
+
location: Location
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class Usage(BaseModel):
|
|
73
|
+
"""Everything the scanner found in a source tree."""
|
|
74
|
+
|
|
75
|
+
imports: dict[str, tuple[Location, ...]] = {}
|
|
76
|
+
references: tuple[Reference, ...] = ()
|
|
77
|
+
dynamic: tuple[DynamicUse, ...] = ()
|
|
78
|
+
unparsable: dict[str, str] = {}
|
|
79
|
+
files: int = 0
|
|
80
|
+
|
|
81
|
+
def imported(self, names: tuple[str, ...]) -> list[Location]:
|
|
82
|
+
"""Where any of ``names``, or a submodule of one, is imported."""
|
|
83
|
+
hits = [
|
|
84
|
+
location
|
|
85
|
+
for module, locations in self.imports.items()
|
|
86
|
+
if any(module == n or module.startswith(f"{n}.") for n in names)
|
|
87
|
+
for location in locations
|
|
88
|
+
]
|
|
89
|
+
return sorted(set(hits), key=lambda loc: (loc.path, loc.line))
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def python_files(root: Path) -> Iterator[Path]:
|
|
93
|
+
if root.is_file():
|
|
94
|
+
if root.suffix == ".py":
|
|
95
|
+
yield root
|
|
96
|
+
return
|
|
97
|
+
for path in sorted(root.rglob("*.py")):
|
|
98
|
+
if not SKIP_DIRS.intersection(path.relative_to(root).parts[:-1]):
|
|
99
|
+
yield path
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
class _File:
|
|
103
|
+
"""Scan state for one file."""
|
|
104
|
+
|
|
105
|
+
def __init__(self, display: str) -> None:
|
|
106
|
+
self.display = display
|
|
107
|
+
self.aliases: dict[str, str] = {}
|
|
108
|
+
self.module_aliases: set[str] = set()
|
|
109
|
+
self.imports: dict[str, list[Location]] = {}
|
|
110
|
+
self.references: list[Reference] = []
|
|
111
|
+
self.dynamic: list[DynamicUse] = []
|
|
112
|
+
|
|
113
|
+
def loc(self, node: ast.AST) -> Location:
|
|
114
|
+
return Location(path=self.display, line=getattr(node, "lineno", 0))
|
|
115
|
+
|
|
116
|
+
def collect_imports(self, tree: ast.AST) -> None:
|
|
117
|
+
for node in ast.walk(tree):
|
|
118
|
+
if isinstance(node, ast.Import):
|
|
119
|
+
for alias in node.names:
|
|
120
|
+
self.imports.setdefault(alias.name, []).append(self.loc(node))
|
|
121
|
+
local = alias.asname or alias.name.split(".")[0]
|
|
122
|
+
self.aliases[local] = alias.name if alias.asname else local
|
|
123
|
+
self.module_aliases.add(local)
|
|
124
|
+
elif isinstance(node, ast.ImportFrom) and node.module and not node.level:
|
|
125
|
+
self.imports.setdefault(node.module, []).append(self.loc(node))
|
|
126
|
+
for alias in node.names:
|
|
127
|
+
if alias.name == "*":
|
|
128
|
+
self.dynamic.append(
|
|
129
|
+
DynamicUse(
|
|
130
|
+
root=node.module.split(".")[0],
|
|
131
|
+
reason=f"from {node.module} import *",
|
|
132
|
+
location=self.loc(node),
|
|
133
|
+
)
|
|
134
|
+
)
|
|
135
|
+
else:
|
|
136
|
+
self.aliases[alias.asname or alias.name] = f"{node.module}.{alias.name}"
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
class _References(ast.NodeVisitor):
|
|
140
|
+
def __init__(self, state: _File) -> None:
|
|
141
|
+
self.state = state
|
|
142
|
+
self.covered: set[int] = set() # nodes already part of a recorded chain
|
|
143
|
+
|
|
144
|
+
def visit_Import(self, node: ast.Import) -> None:
|
|
145
|
+
return
|
|
146
|
+
|
|
147
|
+
def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
|
|
148
|
+
return
|
|
149
|
+
|
|
150
|
+
@staticmethod
|
|
151
|
+
def _chain(node: ast.expr) -> tuple[list[str], list[ast.expr]] | None:
|
|
152
|
+
parts: list[str] = []
|
|
153
|
+
inner: list[ast.expr] = []
|
|
154
|
+
while isinstance(node, ast.Attribute):
|
|
155
|
+
parts.append(node.attr)
|
|
156
|
+
node = node.value
|
|
157
|
+
inner.append(node)
|
|
158
|
+
if isinstance(node, ast.Name):
|
|
159
|
+
parts.append(node.id)
|
|
160
|
+
return parts[::-1], inner
|
|
161
|
+
return None
|
|
162
|
+
|
|
163
|
+
def _resolve(self, parts: list[str]) -> str | None:
|
|
164
|
+
target = self.state.aliases.get(parts[0])
|
|
165
|
+
return ".".join([target, *parts[1:]]) if target else None
|
|
166
|
+
|
|
167
|
+
def visit_Attribute(self, node: ast.Attribute) -> None:
|
|
168
|
+
if id(node) not in self.covered and (found := self._chain(node)) is not None:
|
|
169
|
+
parts, inner = found
|
|
170
|
+
if (target := self._resolve(parts)) is not None:
|
|
171
|
+
self.state.references.append(Reference(name=target, location=self.state.loc(node)))
|
|
172
|
+
self.covered.update(id(n) for n in inner)
|
|
173
|
+
self.generic_visit(node)
|
|
174
|
+
|
|
175
|
+
def visit_Name(self, node: ast.Name) -> None:
|
|
176
|
+
if id(node) in self.covered or not isinstance(node.ctx, ast.Load):
|
|
177
|
+
return
|
|
178
|
+
target = self.state.aliases.get(node.id)
|
|
179
|
+
if target is None:
|
|
180
|
+
return
|
|
181
|
+
if node.id in self.state.module_aliases:
|
|
182
|
+
self.state.dynamic.append(
|
|
183
|
+
DynamicUse(
|
|
184
|
+
root=target.split(".")[0],
|
|
185
|
+
reason=f"module {target} used as a value",
|
|
186
|
+
location=self.state.loc(node),
|
|
187
|
+
)
|
|
188
|
+
)
|
|
189
|
+
else:
|
|
190
|
+
self.state.references.append(Reference(name=target, location=self.state.loc(node)))
|
|
191
|
+
|
|
192
|
+
def visit_Call(self, node: ast.Call) -> None:
|
|
193
|
+
function = node.func
|
|
194
|
+
found = self._chain(function) if isinstance(function, ast.Attribute | ast.Name) else None
|
|
195
|
+
called = self._resolve(found[0]) if found else None
|
|
196
|
+
if isinstance(function, ast.Name) and function.id == "__import__":
|
|
197
|
+
called = "__import__"
|
|
198
|
+
elif isinstance(function, ast.Name) and function.id == "getattr":
|
|
199
|
+
called = "getattr"
|
|
200
|
+
|
|
201
|
+
if called in _DYNAMIC_IMPORTERS and node.args:
|
|
202
|
+
first = node.args[0]
|
|
203
|
+
target = (
|
|
204
|
+
first.value
|
|
205
|
+
if isinstance(first, ast.Constant) and isinstance(first.value, str)
|
|
206
|
+
else "?"
|
|
207
|
+
)
|
|
208
|
+
self.state.dynamic.append(
|
|
209
|
+
DynamicUse(
|
|
210
|
+
root=target.split(".")[0],
|
|
211
|
+
reason=f"{called}({target!r})",
|
|
212
|
+
location=self.state.loc(node),
|
|
213
|
+
)
|
|
214
|
+
)
|
|
215
|
+
if target != "?":
|
|
216
|
+
self.state.imports.setdefault(target, []).append(self.state.loc(node))
|
|
217
|
+
elif called == "getattr" and len(node.args) >= 2 and isinstance(node.args[0], ast.Name):
|
|
218
|
+
base = self.state.aliases.get(node.args[0].id)
|
|
219
|
+
if base is not None:
|
|
220
|
+
self.covered.add(id(node.args[0]))
|
|
221
|
+
attribute = node.args[1]
|
|
222
|
+
if isinstance(attribute, ast.Constant) and isinstance(attribute.value, str):
|
|
223
|
+
self.state.references.append(
|
|
224
|
+
Reference(name=f"{base}.{attribute.value}", location=self.state.loc(node))
|
|
225
|
+
)
|
|
226
|
+
else:
|
|
227
|
+
self.state.dynamic.append(
|
|
228
|
+
DynamicUse(
|
|
229
|
+
root=base.split(".")[0],
|
|
230
|
+
reason=f"getattr({node.args[0].id}, <computed>)",
|
|
231
|
+
location=self.state.loc(node),
|
|
232
|
+
)
|
|
233
|
+
)
|
|
234
|
+
self.generic_visit(node)
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def scan(root: Path) -> Usage:
|
|
238
|
+
"""Scan every Python file under ``root``, skipping virtualenvs, builds and VCS dirs."""
|
|
239
|
+
imports: dict[str, list[Location]] = {}
|
|
240
|
+
references: list[Reference] = []
|
|
241
|
+
dynamic: list[DynamicUse] = []
|
|
242
|
+
unparsable: dict[str, str] = {}
|
|
243
|
+
files = 0
|
|
244
|
+
base = root if root.is_dir() else root.parent
|
|
245
|
+
for path in python_files(root):
|
|
246
|
+
files += 1
|
|
247
|
+
display = path.relative_to(base).as_posix()
|
|
248
|
+
source = path.read_text(encoding="utf-8", errors="replace")
|
|
249
|
+
try:
|
|
250
|
+
tree = ast.parse(source, filename=display)
|
|
251
|
+
except (SyntaxError, ValueError):
|
|
252
|
+
unparsable[display] = source[:200_000]
|
|
253
|
+
continue
|
|
254
|
+
state = _File(display)
|
|
255
|
+
state.collect_imports(tree)
|
|
256
|
+
_References(state).visit(tree)
|
|
257
|
+
for module, locations in state.imports.items():
|
|
258
|
+
imports.setdefault(module, []).extend(locations)
|
|
259
|
+
references.extend(state.references)
|
|
260
|
+
dynamic.extend(state.dynamic)
|
|
261
|
+
return Usage(
|
|
262
|
+
imports={
|
|
263
|
+
m: tuple(sorted(set(v), key=lambda loc: (loc.path, loc.line)))
|
|
264
|
+
for m, v in imports.items()
|
|
265
|
+
},
|
|
266
|
+
references=tuple(references),
|
|
267
|
+
dynamic=tuple(dynamic),
|
|
268
|
+
unparsable=unparsable,
|
|
269
|
+
files=files,
|
|
270
|
+
)
|