patchsight 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. patchsight/__init__.py +5 -0
  2. patchsight/cli.py +130 -0
  3. patchsight/db.py +221 -0
  4. patchsight/eval/__init__.py +1 -0
  5. patchsight/eval/confidence.py +169 -0
  6. patchsight/eval/metrics.py +134 -0
  7. patchsight/eval/parity.py +96 -0
  8. patchsight/eval/safety.py +54 -0
  9. patchsight/names.py +76 -0
  10. patchsight/pipeline/__init__.py +4 -0
  11. patchsight/pipeline/__main__.py +863 -0
  12. patchsight/pipeline/benchmarks/__init__.py +1 -0
  13. patchsight/pipeline/benchmarks/bundle.py +108 -0
  14. patchsight/pipeline/benchmarks/changedfuncs.py +275 -0
  15. patchsight/pipeline/benchmarks/dunlap.py +135 -0
  16. patchsight/pipeline/benchmarks/extraction.py +242 -0
  17. patchsight/pipeline/benchmarks/replay.py +187 -0
  18. patchsight/pipeline/benchmarks/splits.py +121 -0
  19. patchsight/pipeline/build_db.py +227 -0
  20. patchsight/pipeline/diffs/__init__.py +1 -0
  21. patchsight/pipeline/diffs/commits.py +109 -0
  22. patchsight/pipeline/diffs/github.py +177 -0
  23. patchsight/pipeline/diffs/hunks.py +120 -0
  24. patchsight/pipeline/diffs/mapping.py +203 -0
  25. patchsight/pipeline/diffs/paths.py +127 -0
  26. patchsight/pipeline/extract/__init__.py +1 -0
  27. patchsight/pipeline/extract/extractor.py +156 -0
  28. patchsight/pipeline/extract/prompt.py +219 -0
  29. patchsight/pipeline/extract/schema.py +40 -0
  30. patchsight/pipeline/extract/validate.py +81 -0
  31. patchsight/pipeline/fix_sample.py +384 -0
  32. patchsight/pipeline/http.py +143 -0
  33. patchsight/pipeline/ingest/__init__.py +1 -0
  34. patchsight/pipeline/ingest/aliases.py +76 -0
  35. patchsight/pipeline/ingest/ghsa_repo.py +80 -0
  36. patchsight/pipeline/ingest/osv.py +253 -0
  37. patchsight/pipeline/ingest/references.py +189 -0
  38. patchsight/pipeline/ingest/snapshot.py +146 -0
  39. patchsight/pipeline/llm/__init__.py +1 -0
  40. patchsight/pipeline/llm/client.py +288 -0
  41. patchsight/pipeline/llm/config.py +68 -0
  42. patchsight/pipeline/llm/tokens.py +74 -0
  43. patchsight/pipeline/pypi.py +64 -0
  44. patchsight/pipeline/week1.py +314 -0
  45. patchsight/py.typed +0 -0
  46. patchsight/scan/__init__.py +4 -0
  47. patchsight/scan/ast_scanner.py +270 -0
  48. patchsight/scan/dbcache.py +119 -0
  49. patchsight/scan/lockfiles/__init__.py +1 -0
  50. patchsight/scan/lockfiles/requirements.py +116 -0
  51. patchsight/scan/osv_client.py +80 -0
  52. patchsight/scan/report.py +90 -0
  53. patchsight/scan/run.py +43 -0
  54. patchsight/scan/verdicts.py +178 -0
  55. patchsight-0.1.0.dist-info/METADATA +104 -0
  56. patchsight-0.1.0.dist-info/RECORD +59 -0
  57. patchsight-0.1.0.dist-info/WHEEL +4 -0
  58. patchsight-0.1.0.dist-info/entry_points.txt +3 -0
  59. patchsight-0.1.0.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,314 @@
1
+ """Week 1 data gate: measure the PyPI advisory data before building on it (proposal §15).
2
+
3
+ Three questions decide the plan:
4
+
5
+ 1. What share of PyPI vulnerabilities link a fix commit? Under about 40%, the fix-commit
6
+ locator agent becomes essential.
7
+ 2. What share already carry symbol data? This is the project's opening statistic.
8
+ 3. How many new vulnerabilities arrive per day? This sizes the nightly LLM budget.
9
+
10
+ Counts are per vulnerability (alias group) unless stated, because one vulnerability often
11
+ appears as both a GHSA and a PYSEC record.
12
+ """
13
+
14
+ import math
15
+ import statistics
16
+ from collections import Counter
17
+ from collections.abc import Iterable, Sequence
18
+ from datetime import date, datetime, timedelta
19
+
20
+ from pydantic import BaseModel, ConfigDict
21
+
22
+ from patchsight.pipeline.ingest.aliases import AliasGroup, alias_groups, first_published
23
+ from patchsight.pipeline.ingest.ghsa_repo import GhsaScan
24
+ from patchsight.pipeline.ingest.osv import Advisory, Excluded, Quarantined
25
+
26
+ GATE_THRESHOLD = 0.40
27
+ GITHUB_KINDS = frozenset({"github_commit", "github_pull", "github_compare"})
28
+ COMMIT_KINDS = frozenset({"github_commit", "gitlab_commit", "bitbucket_commit", "other_commit"})
29
+ # Proposal §7: ~6,000 input tokens per advisory against a 200,000-token daily cap per model.
30
+ TOKENS_PER_ADVISORY = 6_000
31
+ DAILY_TOKEN_CAP = 200_000
32
+
33
+
34
+ class Share(BaseModel):
35
+ model_config = ConfigDict(frozen=True)
36
+
37
+ count: int
38
+ total: int
39
+ share: float
40
+ ci95_low: float
41
+ ci95_high: float
42
+
43
+
44
+ class DailyRate(BaseModel):
45
+ model_config = ConfigDict(frozen=True)
46
+
47
+ days: int
48
+ total: int
49
+ mean: float
50
+ median: float
51
+ p90: float
52
+ max: int
53
+
54
+
55
+ class Week1Report(BaseModel):
56
+ as_of: date
57
+ records_total: int
58
+ records_by_source: dict[str, int]
59
+ excluded: dict[str, int]
60
+ quarantined: int
61
+ quarantine_reasons: dict[str, int]
62
+ advisories_in_scope: int
63
+ in_scope_by_source: dict[str, int]
64
+ vulnerabilities: int
65
+ packages: int
66
+ fix_link_any: Share
67
+ fix_link_github: Share
68
+ fix_link_github_commit: Share
69
+ fix_link_strong: Share
70
+ fix_link_by_kind: dict[str, int]
71
+ records_with_fix_link_any: Share
72
+ symbol_data_osv: Share
73
+ symbol_data_osv_ids: tuple[str, ...]
74
+ ghsa_repo: GhsaScan | None
75
+ new_per_day_365: DailyRate
76
+ new_per_day_90: DailyRate
77
+ new_with_github_link_per_day_365: DailyRate
78
+ modified_last_7_days: int
79
+ modified_last_30_days: int
80
+ advisories_per_day_per_model: int
81
+ gate_metric: Share
82
+ gate_passed: bool
83
+
84
+
85
+ def wilson(count: int, total: int, z: float = 1.959964) -> Share:
86
+ """A proportion with its Wilson score 95% interval."""
87
+ if total == 0:
88
+ return Share(count=0, total=0, share=0.0, ci95_low=0.0, ci95_high=0.0)
89
+ p = count / total
90
+ denominator = 1 + z * z / total
91
+ centre = (p + z * z / (2 * total)) / denominator
92
+ margin = z * math.sqrt(p * (1 - p) / total + z * z / (4 * total * total)) / denominator
93
+ return Share(
94
+ count=count,
95
+ total=total,
96
+ share=p,
97
+ # The Wilson interval always contains p; clamp away floating-point error at 0 and 1.
98
+ ci95_low=max(0.0, min(p, centre - margin)),
99
+ ci95_high=min(1.0, max(p, centre + margin)),
100
+ )
101
+
102
+
103
+ def daily_rate(days_seen: Iterable[date], start: date, end: date) -> DailyRate:
104
+ """Per-day counts over ``[start, end)``, including days with no arrivals."""
105
+ counts = Counter(d for d in days_seen if start <= d < end)
106
+ span = (end - start).days
107
+ series = [counts.get(start + timedelta(days=i), 0) for i in range(span)]
108
+ ordered = sorted(series)
109
+ p90 = ordered[min(len(ordered) - 1, math.ceil(0.9 * len(ordered)) - 1)] if ordered else 0
110
+ return DailyRate(
111
+ days=span,
112
+ total=sum(series),
113
+ mean=statistics.fmean(series) if series else 0.0,
114
+ median=statistics.median(series) if series else 0.0,
115
+ p90=float(p90),
116
+ max=max(series, default=0),
117
+ )
118
+
119
+
120
+ def _group_has(group: AliasGroup, by_id: dict[str, Advisory], kinds: frozenset[str]) -> bool:
121
+ return any(c.kind in kinds for i in group.advisory_ids for c in by_id[i].fix_candidates)
122
+
123
+
124
+ def _group_has_strong(group: AliasGroup, by_id: dict[str, Advisory]) -> bool:
125
+ """A FIX-typed reference or a GIT range ``fixed`` commit, not just a WEB link."""
126
+ return any(
127
+ c.origin == "git_range" or c.reference_type == "FIX"
128
+ for i in group.advisory_ids
129
+ for c in by_id[i].fix_candidates
130
+ )
131
+
132
+
133
+ def build_report(
134
+ results: Sequence[Advisory | Excluded | Quarantined],
135
+ as_of: datetime,
136
+ ghsa_scan: GhsaScan | None = None,
137
+ ) -> Week1Report:
138
+ advisories = [r for r in results if isinstance(r, Advisory)]
139
+ excluded = [r for r in results if isinstance(r, Excluded)]
140
+ quarantined = [r for r in results if isinstance(r, Quarantined)]
141
+ by_id = {a.id: a for a in advisories}
142
+ groups = alias_groups(advisories)
143
+ total_groups = len(groups)
144
+
145
+ def share(predicate_kinds: frozenset[str]) -> Share:
146
+ return wilson(sum(_group_has(g, by_id, predicate_kinds) for g in groups), total_groups)
147
+
148
+ all_kinds = GITHUB_KINDS | COMMIT_KINDS | {"gitlab_merge_request"}
149
+ kind_counts: Counter[str] = Counter()
150
+ for group in groups:
151
+ kinds = {c.kind for i in group.advisory_ids for c in by_id[i].fix_candidates}
152
+ kind_counts.update(kinds)
153
+
154
+ symbol_groups = [g for g in groups if any(by_id[i].symbol_hints for i in g.advisory_ids)]
155
+ end = as_of.date()
156
+ published = {g.key: first_published(g, by_id) for g in groups}
157
+ new_days = [d.date() for d in published.values() if d is not None]
158
+ github_days = [
159
+ d.date()
160
+ for g in groups
161
+ if (d := published[g.key]) is not None and _group_has(g, by_id, GITHUB_KINDS)
162
+ ]
163
+ gate_metric = share(GITHUB_KINDS)
164
+ return Week1Report(
165
+ as_of=end,
166
+ records_total=len(results),
167
+ records_by_source=dict(
168
+ sorted(
169
+ Counter(
170
+ r.id.split("-", 1)[0] if not isinstance(r, Quarantined) else "unreadable"
171
+ for r in results
172
+ ).items()
173
+ )
174
+ ),
175
+ excluded=dict(sorted(Counter(r.reason for r in excluded).items())),
176
+ quarantined=len(quarantined),
177
+ quarantine_reasons=dict(Counter(q.reason for q in quarantined).most_common(10)),
178
+ advisories_in_scope=len(advisories),
179
+ in_scope_by_source=dict(sorted(Counter(a.source for a in advisories).items())),
180
+ vulnerabilities=total_groups,
181
+ packages=len({p.name for a in advisories for p in a.packages}),
182
+ fix_link_any=share(all_kinds),
183
+ fix_link_github=gate_metric,
184
+ fix_link_github_commit=share(frozenset({"github_commit"})),
185
+ fix_link_strong=wilson(sum(_group_has_strong(g, by_id) for g in groups), total_groups),
186
+ fix_link_by_kind=dict(sorted(kind_counts.items())),
187
+ records_with_fix_link_any=wilson(
188
+ sum(bool(a.fix_candidates) for a in advisories), len(advisories)
189
+ ),
190
+ symbol_data_osv=wilson(len(symbol_groups), total_groups),
191
+ symbol_data_osv_ids=tuple(i for g in symbol_groups for i in g.advisory_ids),
192
+ ghsa_repo=ghsa_scan,
193
+ new_per_day_365=daily_rate(new_days, end - timedelta(days=365), end),
194
+ new_per_day_90=daily_rate(new_days, end - timedelta(days=90), end),
195
+ new_with_github_link_per_day_365=daily_rate(github_days, end - timedelta(days=365), end),
196
+ modified_last_7_days=sum(a.modified >= as_of - timedelta(days=7) for a in advisories),
197
+ modified_last_30_days=sum(a.modified >= as_of - timedelta(days=30) for a in advisories),
198
+ advisories_per_day_per_model=DAILY_TOKEN_CAP // TOKENS_PER_ADVISORY,
199
+ gate_metric=gate_metric,
200
+ gate_passed=gate_metric.share >= GATE_THRESHOLD,
201
+ )
202
+
203
+
204
+ def _pct(s: Share) -> str:
205
+ # Small shares need more decimals, or a handful of cases reads as "0.0%".
206
+ fmt = ".2%" if 0 < s.share < 0.01 else ".1%"
207
+ ci = f"95% CI {s.ci95_low:{fmt}} to {s.ci95_high:{fmt}}"
208
+ return f"{s.share:{fmt}} ({s.count:,} / {s.total:,}; {ci})"
209
+
210
+
211
+ def _rate(r: DailyRate) -> str:
212
+ stats = f"mean {r.mean:.1f}, median {r.median:.0f}, p90 {r.p90:.0f}, max {r.max}"
213
+ return f"{stats} ({r.total:,} in {r.days} days)"
214
+
215
+
216
+ def render_markdown(report: Week1Report, manifest_lines: Sequence[str]) -> str:
217
+ gate = "PASS" if report.gate_passed else "FAIL"
218
+ lines = [
219
+ "# Week 1 data gate",
220
+ "",
221
+ f"Measured on the OSV PyPI snapshot of {report.as_of.isoformat()}. "
222
+ "Generated by `python -m patchsight.pipeline week1-report`; do not edit by hand.",
223
+ "",
224
+ "## Sources",
225
+ "",
226
+ *manifest_lines,
227
+ "",
228
+ "Advisory data: OSV.dev (CC BY 4.0), aggregating the GitHub Advisory Database "
229
+ "(CC BY 4.0) and the PyPA Advisory Database (CC BY 4.0). This report contains "
230
+ "counts derived from that data, not the data itself.",
231
+ "",
232
+ "## Gate result",
233
+ "",
234
+ f"**{gate}.** Share of PyPI vulnerabilities with at least one GitHub fix link "
235
+ f"(commit, pull request or tag comparison): {_pct(report.gate_metric)}. "
236
+ f"Threshold: {GATE_THRESHOLD:.0%}. Below it, the fix-commit locator agent becomes "
237
+ "essential (proposal §15).",
238
+ "",
239
+ "This counts *candidate* links, so it is an upper bound. Only "
240
+ f"{_pct(report.fix_link_strong)} carry an explicit fix signal (a FIX-typed "
241
+ "reference or a GIT-range fixed commit); the rest are untyped links to commits or "
242
+ "pull requests. How many candidates really are the fix is measured when the diffs "
243
+ "are fetched.",
244
+ "",
245
+ "## Scope",
246
+ "",
247
+ f"- Records in the dump: {report.records_total:,} "
248
+ f"({', '.join(f'{k} {v:,}' for k, v in report.records_by_source.items())})",
249
+ f"- Excluded: {', '.join(f'{k} {v:,}' for k, v in report.excluded.items())}",
250
+ f"- Quarantined as malformed: {report.quarantined:,}",
251
+ f"- In-scope advisories: {report.advisories_in_scope:,} "
252
+ f"({', '.join(f'{k} {v:,}' for k, v in report.in_scope_by_source.items())})",
253
+ f"- Vulnerabilities after alias grouping: **{report.vulnerabilities:,}**, "
254
+ f"across {report.packages:,} packages",
255
+ "",
256
+ "Malicious-package reports (`MAL-*`) are excluded: the whole package is malware, "
257
+ "so there is no fix commit and no vulnerable function to find.",
258
+ "",
259
+ "## 1. Fix-commit links",
260
+ "",
261
+ "Per vulnerability (alias group):",
262
+ "",
263
+ f"- Any fix link, any host: {_pct(report.fix_link_any)}",
264
+ f"- GitHub commit, PR or comparison (**gate metric**): {_pct(report.fix_link_github)}",
265
+ f"- GitHub commit link specifically: {_pct(report.fix_link_github_commit)}",
266
+ f"- Strong signal only (FIX-typed reference or GIT-range fixed commit): "
267
+ f"{_pct(report.fix_link_strong)}",
268
+ f"- Per record rather than per vulnerability, any host: "
269
+ f"{_pct(report.records_with_fix_link_any)}",
270
+ "",
271
+ "Vulnerabilities by kind of link present (one vulnerability can have several):",
272
+ "",
273
+ "| Kind | Vulnerabilities |",
274
+ "|---|---|",
275
+ *(f"| `{k}` | {v:,} |" for k, v in report.fix_link_by_kind.items()),
276
+ "",
277
+ "A link is a *candidate*: whether it is on the package's own repository and is "
278
+ "really the fix is checked when the diff is fetched. For reference, 78% of Go "
279
+ "reports in Dunlap et al. had a patch link.",
280
+ "",
281
+ "## 2. Existing symbol data (the opening statistic)",
282
+ "",
283
+ f"- In the OSV dump (`ecosystem_specific.imports` or `affected_functions`): "
284
+ f"{_pct(report.symbol_data_osv)}: {', '.join(report.symbol_data_osv_ids) or 'none'}",
285
+ ]
286
+ if report.ghsa_repo is not None:
287
+ g = report.ghsa_repo
288
+ lines.append(
289
+ f"- In `github/advisory-database` itself: {len(g.with_symbol_data):,} of "
290
+ f"{g.found:,} PyPI GHSA files checked carry symbol data "
291
+ f"({len(g.missing)} expected file(s) not found at the publication-month path)"
292
+ )
293
+ lines += [
294
+ "",
295
+ "## 3. Arrival rate",
296
+ "",
297
+ "New vulnerabilities per day, by first publication across the alias group:",
298
+ "",
299
+ f"- Last 365 days: {_rate(report.new_per_day_365)}",
300
+ f"- Last 90 days: {_rate(report.new_per_day_90)}",
301
+ "- Last 365 days, with a GitHub fix link: "
302
+ f"{_rate(report.new_with_github_link_per_day_365)}",
303
+ "",
304
+ f"Re-extraction pressure (lower bound, since each record shows only its latest "
305
+ f"modification): {report.modified_last_7_days:,} in-scope advisories modified in "
306
+ f"the last 7 days, {report.modified_last_30_days:,} in the last 30.",
307
+ "",
308
+ f"Budget check: at about {TOKENS_PER_ADVISORY:,} input tokens per advisory and a "
309
+ f"{DAILY_TOKEN_CAP:,}-token daily cap, one model handles about "
310
+ f"{report.advisories_per_day_per_model} advisories a day. These caps are reported "
311
+ "by third parties and must be confirmed in the provider console.",
312
+ "",
313
+ ]
314
+ return "\n".join(lines)
patchsight/py.typed ADDED
File without changes
@@ -0,0 +1,4 @@
1
+ """Scan path: lockfile parsing, OSV lookup, database join and static usage analysis.
2
+
3
+ No LLM and no server. Must not import ``patchsight.pipeline`` or any LLM client.
4
+ """
@@ -0,0 +1,270 @@
1
+ """Find which imported names user code references, with Python's ``ast`` module. No LLM.
2
+
3
+ Two passes per file:
4
+
5
+ 1. **Imports** build an alias table, wherever they appear in the file. ``import yaml as y``
6
+ makes ``y`` mean ``yaml``, and ``from yaml import load as l`` makes ``l`` mean
7
+ ``yaml.load``.
8
+ 2. **References**: attribute chains and names in load context are resolved through the
9
+ table to full dotted names such as ``yaml.load``. Only the longest chain is recorded,
10
+ so ``y.constructor.FullConstructor`` yields one reference, not three.
11
+
12
+ Uses the scanner cannot resolve are recorded as *dynamic* and make a verdict UNKNOWN:
13
+
14
+ * ``getattr(module, <computed name>)`` (a string constant is resolved normally);
15
+ * ``importlib.import_module(...)`` and ``__import__(...)``;
16
+ * a module object used as a plain value (passed, stored or returned);
17
+ * ``from package import *``.
18
+
19
+ The analysis is flow-insensitive, so an import anywhere counts for the whole file. That
20
+ can over-report references, which only raises an alert's priority.
21
+ """
22
+
23
+ import ast
24
+ from collections.abc import Iterator
25
+ from pathlib import Path
26
+
27
+ from pydantic import BaseModel, ConfigDict
28
+
29
+ SKIP_DIRS = frozenset(
30
+ {
31
+ ".git",
32
+ ".hg",
33
+ ".venv",
34
+ "venv",
35
+ "env",
36
+ ".tox",
37
+ ".nox",
38
+ "node_modules",
39
+ "site-packages",
40
+ "__pycache__",
41
+ "build",
42
+ "dist",
43
+ ".mypy_cache",
44
+ ".pytest_cache",
45
+ }
46
+ )
47
+ _DYNAMIC_IMPORTERS = ("__import__", "importlib.import_module")
48
+
49
+
50
+ class Location(BaseModel):
51
+ model_config = ConfigDict(frozen=True)
52
+
53
+ path: str
54
+ line: int
55
+
56
+
57
+ class Reference(BaseModel):
58
+ model_config = ConfigDict(frozen=True)
59
+
60
+ name: str
61
+ location: Location
62
+
63
+
64
+ class DynamicUse(BaseModel):
65
+ model_config = ConfigDict(frozen=True)
66
+
67
+ root: str
68
+ reason: str
69
+ location: Location
70
+
71
+
72
+ class Usage(BaseModel):
73
+ """Everything the scanner found in a source tree."""
74
+
75
+ imports: dict[str, tuple[Location, ...]] = {}
76
+ references: tuple[Reference, ...] = ()
77
+ dynamic: tuple[DynamicUse, ...] = ()
78
+ unparsable: dict[str, str] = {}
79
+ files: int = 0
80
+
81
+ def imported(self, names: tuple[str, ...]) -> list[Location]:
82
+ """Where any of ``names``, or a submodule of one, is imported."""
83
+ hits = [
84
+ location
85
+ for module, locations in self.imports.items()
86
+ if any(module == n or module.startswith(f"{n}.") for n in names)
87
+ for location in locations
88
+ ]
89
+ return sorted(set(hits), key=lambda loc: (loc.path, loc.line))
90
+
91
+
92
+ def python_files(root: Path) -> Iterator[Path]:
93
+ if root.is_file():
94
+ if root.suffix == ".py":
95
+ yield root
96
+ return
97
+ for path in sorted(root.rglob("*.py")):
98
+ if not SKIP_DIRS.intersection(path.relative_to(root).parts[:-1]):
99
+ yield path
100
+
101
+
102
+ class _File:
103
+ """Scan state for one file."""
104
+
105
+ def __init__(self, display: str) -> None:
106
+ self.display = display
107
+ self.aliases: dict[str, str] = {}
108
+ self.module_aliases: set[str] = set()
109
+ self.imports: dict[str, list[Location]] = {}
110
+ self.references: list[Reference] = []
111
+ self.dynamic: list[DynamicUse] = []
112
+
113
+ def loc(self, node: ast.AST) -> Location:
114
+ return Location(path=self.display, line=getattr(node, "lineno", 0))
115
+
116
+ def collect_imports(self, tree: ast.AST) -> None:
117
+ for node in ast.walk(tree):
118
+ if isinstance(node, ast.Import):
119
+ for alias in node.names:
120
+ self.imports.setdefault(alias.name, []).append(self.loc(node))
121
+ local = alias.asname or alias.name.split(".")[0]
122
+ self.aliases[local] = alias.name if alias.asname else local
123
+ self.module_aliases.add(local)
124
+ elif isinstance(node, ast.ImportFrom) and node.module and not node.level:
125
+ self.imports.setdefault(node.module, []).append(self.loc(node))
126
+ for alias in node.names:
127
+ if alias.name == "*":
128
+ self.dynamic.append(
129
+ DynamicUse(
130
+ root=node.module.split(".")[0],
131
+ reason=f"from {node.module} import *",
132
+ location=self.loc(node),
133
+ )
134
+ )
135
+ else:
136
+ self.aliases[alias.asname or alias.name] = f"{node.module}.{alias.name}"
137
+
138
+
139
+ class _References(ast.NodeVisitor):
140
+ def __init__(self, state: _File) -> None:
141
+ self.state = state
142
+ self.covered: set[int] = set() # nodes already part of a recorded chain
143
+
144
+ def visit_Import(self, node: ast.Import) -> None:
145
+ return
146
+
147
+ def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
148
+ return
149
+
150
+ @staticmethod
151
+ def _chain(node: ast.expr) -> tuple[list[str], list[ast.expr]] | None:
152
+ parts: list[str] = []
153
+ inner: list[ast.expr] = []
154
+ while isinstance(node, ast.Attribute):
155
+ parts.append(node.attr)
156
+ node = node.value
157
+ inner.append(node)
158
+ if isinstance(node, ast.Name):
159
+ parts.append(node.id)
160
+ return parts[::-1], inner
161
+ return None
162
+
163
+ def _resolve(self, parts: list[str]) -> str | None:
164
+ target = self.state.aliases.get(parts[0])
165
+ return ".".join([target, *parts[1:]]) if target else None
166
+
167
+ def visit_Attribute(self, node: ast.Attribute) -> None:
168
+ if id(node) not in self.covered and (found := self._chain(node)) is not None:
169
+ parts, inner = found
170
+ if (target := self._resolve(parts)) is not None:
171
+ self.state.references.append(Reference(name=target, location=self.state.loc(node)))
172
+ self.covered.update(id(n) for n in inner)
173
+ self.generic_visit(node)
174
+
175
+ def visit_Name(self, node: ast.Name) -> None:
176
+ if id(node) in self.covered or not isinstance(node.ctx, ast.Load):
177
+ return
178
+ target = self.state.aliases.get(node.id)
179
+ if target is None:
180
+ return
181
+ if node.id in self.state.module_aliases:
182
+ self.state.dynamic.append(
183
+ DynamicUse(
184
+ root=target.split(".")[0],
185
+ reason=f"module {target} used as a value",
186
+ location=self.state.loc(node),
187
+ )
188
+ )
189
+ else:
190
+ self.state.references.append(Reference(name=target, location=self.state.loc(node)))
191
+
192
+ def visit_Call(self, node: ast.Call) -> None:
193
+ function = node.func
194
+ found = self._chain(function) if isinstance(function, ast.Attribute | ast.Name) else None
195
+ called = self._resolve(found[0]) if found else None
196
+ if isinstance(function, ast.Name) and function.id == "__import__":
197
+ called = "__import__"
198
+ elif isinstance(function, ast.Name) and function.id == "getattr":
199
+ called = "getattr"
200
+
201
+ if called in _DYNAMIC_IMPORTERS and node.args:
202
+ first = node.args[0]
203
+ target = (
204
+ first.value
205
+ if isinstance(first, ast.Constant) and isinstance(first.value, str)
206
+ else "?"
207
+ )
208
+ self.state.dynamic.append(
209
+ DynamicUse(
210
+ root=target.split(".")[0],
211
+ reason=f"{called}({target!r})",
212
+ location=self.state.loc(node),
213
+ )
214
+ )
215
+ if target != "?":
216
+ self.state.imports.setdefault(target, []).append(self.state.loc(node))
217
+ elif called == "getattr" and len(node.args) >= 2 and isinstance(node.args[0], ast.Name):
218
+ base = self.state.aliases.get(node.args[0].id)
219
+ if base is not None:
220
+ self.covered.add(id(node.args[0]))
221
+ attribute = node.args[1]
222
+ if isinstance(attribute, ast.Constant) and isinstance(attribute.value, str):
223
+ self.state.references.append(
224
+ Reference(name=f"{base}.{attribute.value}", location=self.state.loc(node))
225
+ )
226
+ else:
227
+ self.state.dynamic.append(
228
+ DynamicUse(
229
+ root=base.split(".")[0],
230
+ reason=f"getattr({node.args[0].id}, <computed>)",
231
+ location=self.state.loc(node),
232
+ )
233
+ )
234
+ self.generic_visit(node)
235
+
236
+
237
+ def scan(root: Path) -> Usage:
238
+ """Scan every Python file under ``root``, skipping virtualenvs, builds and VCS dirs."""
239
+ imports: dict[str, list[Location]] = {}
240
+ references: list[Reference] = []
241
+ dynamic: list[DynamicUse] = []
242
+ unparsable: dict[str, str] = {}
243
+ files = 0
244
+ base = root if root.is_dir() else root.parent
245
+ for path in python_files(root):
246
+ files += 1
247
+ display = path.relative_to(base).as_posix()
248
+ source = path.read_text(encoding="utf-8", errors="replace")
249
+ try:
250
+ tree = ast.parse(source, filename=display)
251
+ except (SyntaxError, ValueError):
252
+ unparsable[display] = source[:200_000]
253
+ continue
254
+ state = _File(display)
255
+ state.collect_imports(tree)
256
+ _References(state).visit(tree)
257
+ for module, locations in state.imports.items():
258
+ imports.setdefault(module, []).extend(locations)
259
+ references.extend(state.references)
260
+ dynamic.extend(state.dynamic)
261
+ return Usage(
262
+ imports={
263
+ m: tuple(sorted(set(v), key=lambda loc: (loc.path, loc.line)))
264
+ for m, v in imports.items()
265
+ },
266
+ references=tuple(references),
267
+ dynamic=tuple(dynamic),
268
+ unparsable=unparsable,
269
+ files=files,
270
+ )