pi-python-helper 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +60 -0
- package/CONTRIBUTING.md +77 -0
- package/LICENSE +17 -0
- package/README.md +172 -0
- package/SECURITY.md +24 -0
- package/docs/compatibility.md +64 -0
- package/docs/tools.md +518 -0
- package/extensions/index.ts +28 -0
- package/extensions/shared.ts +48 -0
- package/extensions/tools/dependencies.ts +203 -0
- package/extensions/tools/environment.ts +171 -0
- package/extensions/tools/testing.ts +350 -0
- package/extensions/tools/validation.ts +344 -0
- package/helpers/scan_project.py +777 -0
- package/package.json +73 -0
- package/skills/python-development/SKILL.md +45 -0
- package/src/build/commands.ts +65 -0
- package/src/build/discover.ts +98 -0
- package/src/build/failure.ts +452 -0
- package/src/build/pytest.ts +118 -0
- package/src/build/selection.ts +138 -0
- package/src/build/staleness.ts +117 -0
- package/src/core/result.ts +102 -0
- package/src/core/runner.ts +96 -0
- package/src/core/safety.ts +181 -0
- package/src/core/version.ts +20 -0
- package/src/dependencies/plan.ts +398 -0
- package/src/environment/discovery.ts +166 -0
- package/src/environment/tools.ts +141 -0
- package/src/project/conformance.ts +343 -0
- package/src/project/inspect.ts +351 -0
- package/src/project/installed.ts +225 -0
- package/src/project/paths.ts +74 -0
- package/src/project/root.ts +72 -0
- package/src/project/scanner.ts +243 -0
- package/src/validation/bundle.ts +65 -0
- package/src/validation/evidence.ts +33 -0
- package/src/validation/tdd.ts +62 -0
|
@@ -0,0 +1,777 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Read-only Python project scanner used by pi-python-helper.
|
|
3
|
+
|
|
4
|
+
The extension shells out to this script because two of its questions cannot be
|
|
5
|
+
answered reliably from the outside:
|
|
6
|
+
|
|
7
|
+
* which imports a project actually uses (needs `ast`, not text search), and
|
|
8
|
+
* what `pyproject.toml` and `uv.lock` declare (needs a real TOML parser).
|
|
9
|
+
|
|
10
|
+
Protocol
|
|
11
|
+
--------
|
|
12
|
+
A JSON request is read from stdin and exactly one JSON document is written to
|
|
13
|
+
stdout. Human-readable diagnostics go to stderr. The script never writes to the
|
|
14
|
+
project and never imports project code, so it is safe to run against an
|
|
15
|
+
uninstalled checkout.
|
|
16
|
+
|
|
17
|
+
usage: scan_project.py [--mode MODE] [--root DIR] [--max-files N]
|
|
18
|
+
[--help] [--version]
|
|
19
|
+
|
|
20
|
+
Exit codes:
|
|
21
|
+
0 the scan completed and a result document was written to stdout
|
|
22
|
+
1 an unexpected failure occurred (also reported as JSON on stdout)
|
|
23
|
+
2 the arguments or the request were invalid
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import argparse
|
|
29
|
+
import ast
|
|
30
|
+
import json
|
|
31
|
+
import os
|
|
32
|
+
import re
|
|
33
|
+
import sys
|
|
34
|
+
from pathlib import Path
|
|
35
|
+
|
|
36
|
+
# Bumped whenever the request or the result document changes shape, so the
|
|
37
|
+
# caller can refuse to interpret a document it does not understand.
|
|
38
|
+
SCANNER_VERSION = 1
|
|
39
|
+
|
|
40
|
+
KNOWN_SECTIONS = ("environment", "manifest", "imports")
|
|
41
|
+
EXIT_OK = 0
|
|
42
|
+
EXIT_FAILURE = 1
|
|
43
|
+
EXIT_USAGE = 2
|
|
44
|
+
|
|
45
|
+
EXCLUDED_DIRS = {
|
|
46
|
+
".git",
|
|
47
|
+
".hg",
|
|
48
|
+
".svn",
|
|
49
|
+
".venv",
|
|
50
|
+
"venv",
|
|
51
|
+
".tox",
|
|
52
|
+
".nox",
|
|
53
|
+
".eggs",
|
|
54
|
+
".mypy_cache",
|
|
55
|
+
".ruff_cache",
|
|
56
|
+
".pytest_cache",
|
|
57
|
+
".hypothesis",
|
|
58
|
+
"node_modules",
|
|
59
|
+
"build",
|
|
60
|
+
"dist",
|
|
61
|
+
"__pycache__",
|
|
62
|
+
"site-packages",
|
|
63
|
+
".idea",
|
|
64
|
+
".vscode",
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
# Small fallback so the tool still classifies imports on Python < 3.10, where
|
|
68
|
+
# `sys.stdlib_module_names` does not exist. Only common modules are listed; an
|
|
69
|
+
# unknown name is reported as unclassified rather than assumed third-party.
|
|
70
|
+
FALLBACK_STDLIB = {
|
|
71
|
+
"abc", "argparse", "ast", "asyncio", "base64", "collections", "concurrent",
|
|
72
|
+
"contextlib", "copy", "csv", "ctypes", "dataclasses", "datetime", "decimal",
|
|
73
|
+
"difflib", "email", "enum", "errno", "faulthandler", "fnmatch", "fractions",
|
|
74
|
+
"functools", "gc", "getpass", "glob", "gzip", "hashlib", "heapq", "hmac",
|
|
75
|
+
"html", "http", "importlib", "inspect", "io", "ipaddress", "itertools",
|
|
76
|
+
"json", "keyword", "linecache", "locale", "logging", "lzma", "math",
|
|
77
|
+
"multiprocessing", "operator", "os", "pathlib", "pickle", "pkgutil",
|
|
78
|
+
"platform", "plistlib", "pprint", "profile", "pstats", "queue", "random",
|
|
79
|
+
"re", "secrets", "select", "shelve", "shlex", "shutil", "signal", "site",
|
|
80
|
+
"smtplib", "socket", "sqlite3", "ssl", "stat", "statistics", "string",
|
|
81
|
+
"struct", "subprocess", "sys", "tarfile", "tempfile", "textwrap",
|
|
82
|
+
"threading", "time", "timeit", "token", "tokenize", "traceback", "tracemalloc",
|
|
83
|
+
"types", "typing", "unittest", "urllib", "uuid", "venv", "warnings",
|
|
84
|
+
"weakref", "webbrowser", "xml", "zipfile", "zlib", "zoneinfo", "__future__",
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
NAME_RE = re.compile(r"^\s*([A-Za-z0-9][A-Za-z0-9._-]*)")
|
|
88
|
+
NORMALIZE_RE = re.compile(r"[-_.]+")
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def normalize(name: str) -> str:
|
|
92
|
+
"""PEP 503 name normalization: case and -/_/. are not significant."""
|
|
93
|
+
return NORMALIZE_RE.sub("-", name).strip().lower()
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def toml_module():
|
|
97
|
+
try:
|
|
98
|
+
import tomllib # type: ignore[import-not-found]
|
|
99
|
+
return tomllib, None
|
|
100
|
+
except ModuleNotFoundError:
|
|
101
|
+
pass
|
|
102
|
+
try:
|
|
103
|
+
import tomli # type: ignore[import-not-found]
|
|
104
|
+
return tomli, None
|
|
105
|
+
except ModuleNotFoundError:
|
|
106
|
+
return None, "no TOML parser available (needs Python 3.11+ or the tomli package)"
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def load_toml(path: Path):
|
|
110
|
+
module, error = toml_module()
|
|
111
|
+
if module is None:
|
|
112
|
+
return None, error
|
|
113
|
+
try:
|
|
114
|
+
with open(path, "rb") as handle:
|
|
115
|
+
return module.load(handle), None
|
|
116
|
+
except Exception as exc: # malformed TOML is a diagnostic, not a crash
|
|
117
|
+
return None, f"failed to parse {path.name}: {exc}"
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def parse_requirement(raw):
|
|
121
|
+
"""Minimal PEP 508 split into name/specifier/extras/marker."""
|
|
122
|
+
if not isinstance(raw, str) or not raw.strip():
|
|
123
|
+
return None
|
|
124
|
+
text = raw.strip()
|
|
125
|
+
marker = None
|
|
126
|
+
if ";" in text:
|
|
127
|
+
text, marker = text.split(";", 1)
|
|
128
|
+
extras = []
|
|
129
|
+
if "[" in text:
|
|
130
|
+
head, rest = text.split("[", 1)
|
|
131
|
+
extras_text, _, tail = rest.partition("]")
|
|
132
|
+
extras = [item.strip() for item in extras_text.split(",") if item.strip()]
|
|
133
|
+
text = head + tail
|
|
134
|
+
match = NAME_RE.match(text)
|
|
135
|
+
if not match:
|
|
136
|
+
return None
|
|
137
|
+
name = match.group(1)
|
|
138
|
+
return {
|
|
139
|
+
"raw": raw,
|
|
140
|
+
"name": name,
|
|
141
|
+
"normalized": normalize(name),
|
|
142
|
+
"specifier": text[match.end():].strip(),
|
|
143
|
+
"extras": extras,
|
|
144
|
+
"marker": marker.strip() if marker else None,
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def parse_requirement_list(entries):
|
|
149
|
+
if not isinstance(entries, list):
|
|
150
|
+
return []
|
|
151
|
+
parsed = []
|
|
152
|
+
for entry in entries:
|
|
153
|
+
if not isinstance(entry, str):
|
|
154
|
+
continue # dependency-groups also allows {include-group = "..."}
|
|
155
|
+
item = parse_requirement(entry)
|
|
156
|
+
if item:
|
|
157
|
+
parsed.append(item)
|
|
158
|
+
return parsed
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def safe_is_file(path: Path) -> bool:
|
|
162
|
+
"""Stat without raising.
|
|
163
|
+
|
|
164
|
+
A single unreadable directory (for example a root-owned `/tmp` sibling)
|
|
165
|
+
must never abort the whole scan, so every probe degrades to False.
|
|
166
|
+
"""
|
|
167
|
+
try:
|
|
168
|
+
return path.is_file()
|
|
169
|
+
except OSError:
|
|
170
|
+
return False
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def safe_is_dir(path: Path) -> bool:
|
|
174
|
+
try:
|
|
175
|
+
return path.is_dir()
|
|
176
|
+
except OSError:
|
|
177
|
+
return False
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def safe_iterdir(path: Path) -> list:
|
|
181
|
+
try:
|
|
182
|
+
return sorted(path.iterdir())
|
|
183
|
+
except OSError:
|
|
184
|
+
return []
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def contains_python(path: Path, limit: int = 400) -> bool:
|
|
188
|
+
"""True when the directory actually holds Python code.
|
|
189
|
+
|
|
190
|
+
A directory under `src/` is only an importable module when it contains
|
|
191
|
+
Python. Without this check a TypeScript or JavaScript tree that happens to
|
|
192
|
+
live under `src/` would be reported as a Python package. The walk is bounded
|
|
193
|
+
so a deep tree cannot make layout detection expensive.
|
|
194
|
+
"""
|
|
195
|
+
if safe_is_file(path / "__init__.py"):
|
|
196
|
+
return True
|
|
197
|
+
seen = 0
|
|
198
|
+
stack = [path]
|
|
199
|
+
while stack and seen < limit:
|
|
200
|
+
current = stack.pop()
|
|
201
|
+
for entry in safe_iterdir(current):
|
|
202
|
+
seen += 1
|
|
203
|
+
if entry.name in EXCLUDED_DIRS:
|
|
204
|
+
continue
|
|
205
|
+
if entry.is_dir():
|
|
206
|
+
stack.append(entry)
|
|
207
|
+
elif entry.is_file() and entry.suffix == ".py":
|
|
208
|
+
return True
|
|
209
|
+
return False
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def detect_layout(root: Path) -> tuple[str, list[str]]:
|
|
213
|
+
"""Return the layout kind and the top-level importable module names."""
|
|
214
|
+
modules: list[str] = []
|
|
215
|
+
src = root / "src"
|
|
216
|
+
if safe_is_dir(src):
|
|
217
|
+
layout = "src"
|
|
218
|
+
for entry in safe_iterdir(src):
|
|
219
|
+
if entry.is_dir() and entry.name not in EXCLUDED_DIRS and contains_python(entry):
|
|
220
|
+
modules.append(entry.name)
|
|
221
|
+
elif entry.is_file() and entry.suffix == ".py" and entry.stem != "__init__":
|
|
222
|
+
modules.append(entry.stem)
|
|
223
|
+
else:
|
|
224
|
+
layout = "flat"
|
|
225
|
+
for entry in safe_iterdir(root):
|
|
226
|
+
if entry.name in EXCLUDED_DIRS:
|
|
227
|
+
continue
|
|
228
|
+
if entry.is_dir() and safe_is_file(entry / "__init__.py"):
|
|
229
|
+
modules.append(entry.name)
|
|
230
|
+
elif entry.is_file() and entry.suffix == ".py":
|
|
231
|
+
modules.append(entry.stem)
|
|
232
|
+
return layout, sorted(set(modules))
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def scan_manifests(root: Path) -> dict:
|
|
236
|
+
result: dict = {
|
|
237
|
+
"pyprojectPath": None,
|
|
238
|
+
"name": None,
|
|
239
|
+
"version": None,
|
|
240
|
+
"requiresPython": None,
|
|
241
|
+
"description": None,
|
|
242
|
+
"license": None,
|
|
243
|
+
"dependencies": [],
|
|
244
|
+
"optionalDependencies": {},
|
|
245
|
+
"dependencyGroups": {},
|
|
246
|
+
"buildBackend": None,
|
|
247
|
+
"buildRequires": [],
|
|
248
|
+
"entryPoints": [],
|
|
249
|
+
"toolConfiguration": {},
|
|
250
|
+
"layout": None,
|
|
251
|
+
"modules": [],
|
|
252
|
+
"legacySetupPy": False,
|
|
253
|
+
"legacySetupCfg": False,
|
|
254
|
+
"requirementsFiles": [],
|
|
255
|
+
"uvWorkspaceMembers": [],
|
|
256
|
+
"uvSources": [],
|
|
257
|
+
"warnings": [],
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
layout, modules = detect_layout(root)
|
|
261
|
+
result["layout"] = layout
|
|
262
|
+
result["modules"] = modules
|
|
263
|
+
|
|
264
|
+
project_name = None
|
|
265
|
+
pyproject = root / "pyproject.toml"
|
|
266
|
+
if pyproject.is_file():
|
|
267
|
+
result["pyprojectPath"] = str(pyproject)
|
|
268
|
+
data, error = load_toml(pyproject)
|
|
269
|
+
if error:
|
|
270
|
+
result["warnings"].append(error)
|
|
271
|
+
result["tomlError"] = error
|
|
272
|
+
elif data is not None:
|
|
273
|
+
project = data.get("project") if isinstance(data.get("project"), dict) else {}
|
|
274
|
+
project_name = project.get("name")
|
|
275
|
+
result["name"] = project_name
|
|
276
|
+
result["version"] = project.get("version")
|
|
277
|
+
result["requiresPython"] = project.get("requires-python")
|
|
278
|
+
result["description"] = project.get("description")
|
|
279
|
+
license_value = project.get("license")
|
|
280
|
+
if isinstance(license_value, dict):
|
|
281
|
+
license_value = license_value.get("text") or license_value.get("file")
|
|
282
|
+
result["license"] = license_value
|
|
283
|
+
result["dependencies"] = parse_requirement_list(project.get("dependencies"))
|
|
284
|
+
optional = project.get("optional-dependencies")
|
|
285
|
+
if isinstance(optional, dict):
|
|
286
|
+
result["optionalDependencies"] = {
|
|
287
|
+
key: parse_requirement_list(value) for key, value in optional.items()
|
|
288
|
+
}
|
|
289
|
+
scripts = project.get("scripts")
|
|
290
|
+
gui_scripts = project.get("gui-scripts")
|
|
291
|
+
for table in (scripts, gui_scripts):
|
|
292
|
+
if isinstance(table, dict):
|
|
293
|
+
result["entryPoints"].extend(sorted(table.keys()))
|
|
294
|
+
build_system = data.get("build-system")
|
|
295
|
+
if isinstance(build_system, dict):
|
|
296
|
+
result["buildBackend"] = build_system.get("build-backend")
|
|
297
|
+
result["buildRequires"] = [
|
|
298
|
+
item.get("name")
|
|
299
|
+
for item in parse_requirement_list(build_system.get("requires"))
|
|
300
|
+
]
|
|
301
|
+
groups = data.get("dependency-groups")
|
|
302
|
+
if isinstance(groups, dict):
|
|
303
|
+
result["dependencyGroups"] = {
|
|
304
|
+
key: parse_requirement_list(value) for key, value in groups.items()
|
|
305
|
+
}
|
|
306
|
+
tool = data.get("tool") if isinstance(data.get("tool"), dict) else {}
|
|
307
|
+
result["toolConfiguration"] = {
|
|
308
|
+
key: key in tool
|
|
309
|
+
for key in ("ruff", "mypy", "pytest", "coverage", "pyright", "ty", "hatch")
|
|
310
|
+
}
|
|
311
|
+
uv_table = tool.get("uv") if isinstance(tool.get("uv"), dict) else {}
|
|
312
|
+
workspace = uv_table.get("workspace") if isinstance(uv_table.get("workspace"), dict) else {}
|
|
313
|
+
members = workspace.get("members")
|
|
314
|
+
result["uvWorkspaceMembers"] = [m for m in members if isinstance(m, str)] if isinstance(members, list) else []
|
|
315
|
+
sources = uv_table.get("sources")
|
|
316
|
+
result["uvSources"] = sorted(sources.keys()) if isinstance(sources, dict) else []
|
|
317
|
+
else:
|
|
318
|
+
result["warnings"].append("pyproject.toml was not found at the project root.")
|
|
319
|
+
|
|
320
|
+
result["legacySetupPy"] = safe_is_file(root / "setup.py")
|
|
321
|
+
result["legacySetupCfg"] = safe_is_file(root / "setup.cfg")
|
|
322
|
+
result["requirementsFiles"] = sorted(
|
|
323
|
+
entry.name
|
|
324
|
+
for entry in safe_iterdir(root)
|
|
325
|
+
if entry.name.startswith("requirements")
|
|
326
|
+
and entry.name.endswith(".txt")
|
|
327
|
+
and safe_is_file(entry)
|
|
328
|
+
)
|
|
329
|
+
if project_name:
|
|
330
|
+
result["importName"] = normalize(project_name).replace("-", "_")
|
|
331
|
+
return result
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def parse_lock_dependencies(entry) -> list:
|
|
335
|
+
"""Edges of a locked package, keeping the marker that guards each one.
|
|
336
|
+
|
|
337
|
+
uv writes platform and version conditions here (for example
|
|
338
|
+
`{ name = "colorama", marker = "sys_platform == 'win32'" }`). A locked
|
|
339
|
+
package that is only ever referenced behind a marker must not be reported as
|
|
340
|
+
missing from the environment on a platform where the marker is false.
|
|
341
|
+
"""
|
|
342
|
+
raw = entry.get("dependencies")
|
|
343
|
+
if not isinstance(raw, list):
|
|
344
|
+
return []
|
|
345
|
+
edges = []
|
|
346
|
+
for dependency in raw:
|
|
347
|
+
if isinstance(dependency, str):
|
|
348
|
+
name = dependency
|
|
349
|
+
marker = None
|
|
350
|
+
elif isinstance(dependency, dict) and isinstance(dependency.get("name"), str):
|
|
351
|
+
name = dependency["name"]
|
|
352
|
+
marker = dependency.get("marker")
|
|
353
|
+
else:
|
|
354
|
+
continue
|
|
355
|
+
edges.append(
|
|
356
|
+
{
|
|
357
|
+
"name": name,
|
|
358
|
+
"normalized": normalize(name),
|
|
359
|
+
"marker": marker.strip() if isinstance(marker, str) and marker.strip() else None,
|
|
360
|
+
}
|
|
361
|
+
)
|
|
362
|
+
return edges
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def scan_lock(root: Path) -> dict:
|
|
366
|
+
uv_lock = root / "uv.lock"
|
|
367
|
+
result: dict = {
|
|
368
|
+
"path": str(uv_lock) if uv_lock.is_file() else None,
|
|
369
|
+
"present": uv_lock.is_file(),
|
|
370
|
+
"version": None,
|
|
371
|
+
"revision": None,
|
|
372
|
+
"requiresPython": None,
|
|
373
|
+
"packages": [],
|
|
374
|
+
"warnings": [],
|
|
375
|
+
}
|
|
376
|
+
if not uv_lock.is_file():
|
|
377
|
+
poetry_lock = root / "poetry.lock"
|
|
378
|
+
if poetry_lock.is_file():
|
|
379
|
+
result["warnings"].append(
|
|
380
|
+
"poetry.lock was found; this package targets uv, so only uv.lock is analysed."
|
|
381
|
+
)
|
|
382
|
+
return result
|
|
383
|
+
data, error = load_toml(uv_lock)
|
|
384
|
+
if error:
|
|
385
|
+
result["warnings"].append(error)
|
|
386
|
+
return result
|
|
387
|
+
if not isinstance(data, dict):
|
|
388
|
+
result["warnings"].append("uv.lock did not contain a table.")
|
|
389
|
+
return result
|
|
390
|
+
result["version"] = data.get("version")
|
|
391
|
+
result["revision"] = data.get("revision")
|
|
392
|
+
result["requiresPython"] = data.get("requires-python")
|
|
393
|
+
packages = data.get("package")
|
|
394
|
+
if isinstance(packages, list):
|
|
395
|
+
for entry in packages:
|
|
396
|
+
if not isinstance(entry, dict) or not isinstance(entry.get("name"), str):
|
|
397
|
+
continue
|
|
398
|
+
source = entry.get("source")
|
|
399
|
+
source_kind = None
|
|
400
|
+
if isinstance(source, dict):
|
|
401
|
+
source_kind = next(iter(source.keys()), None)
|
|
402
|
+
result["packages"].append(
|
|
403
|
+
{
|
|
404
|
+
"name": entry.get("name"),
|
|
405
|
+
"normalized": normalize(entry["name"]),
|
|
406
|
+
"version": entry.get("version"),
|
|
407
|
+
"source": source_kind,
|
|
408
|
+
"dependencies": parse_lock_dependencies(entry),
|
|
409
|
+
}
|
|
410
|
+
)
|
|
411
|
+
return result
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def compare_lock(lock: dict, manifest: dict) -> dict:
|
|
415
|
+
"""Report lockfile drift for the direct dependencies only.
|
|
416
|
+
|
|
417
|
+
Transitive packages legitimately exist in the lock without being declared,
|
|
418
|
+
so only the declared set is checked. `packaging` is optional: without it the
|
|
419
|
+
specifier comparison is skipped and reported as unavailable.
|
|
420
|
+
"""
|
|
421
|
+
report = {
|
|
422
|
+
"specifierCheckAvailable": False,
|
|
423
|
+
"missingFromLock": [],
|
|
424
|
+
"unsatisfiedInLock": [],
|
|
425
|
+
"requiresPythonMismatch": None,
|
|
426
|
+
"checkedCount": 0,
|
|
427
|
+
}
|
|
428
|
+
if not lock.get("present"):
|
|
429
|
+
return report
|
|
430
|
+
declared = list(manifest.get("dependencies") or [])
|
|
431
|
+
for group in (manifest.get("optionalDependencies") or {}).values():
|
|
432
|
+
declared.extend(group)
|
|
433
|
+
declared = [item for item in declared if isinstance(item, dict)]
|
|
434
|
+
index = {}
|
|
435
|
+
for package in lock.get("packages") or []:
|
|
436
|
+
index.setdefault(package.get("normalized"), package)
|
|
437
|
+
|
|
438
|
+
manifest_python = manifest.get("requiresPython")
|
|
439
|
+
lock_python = lock.get("requiresPython")
|
|
440
|
+
if manifest_python and lock_python and manifest_python != lock_python:
|
|
441
|
+
report["requiresPythonMismatch"] = {
|
|
442
|
+
"manifest": manifest_python,
|
|
443
|
+
"lock": lock_python,
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
try:
|
|
447
|
+
from packaging.requirements import Requirement
|
|
448
|
+
from packaging.version import InvalidVersion, Version
|
|
449
|
+
except Exception:
|
|
450
|
+
# `packaging` is an optional analyser dependency. It may be missing, or
|
|
451
|
+
# present but broken, and either way the scan must continue with the
|
|
452
|
+
# weaker name-only comparison instead of failing.
|
|
453
|
+
for item in declared:
|
|
454
|
+
if item["normalized"] not in index:
|
|
455
|
+
report["missingFromLock"].append(item["name"])
|
|
456
|
+
report["checkedCount"] = len(declared)
|
|
457
|
+
return report
|
|
458
|
+
|
|
459
|
+
report["specifierCheckAvailable"] = True
|
|
460
|
+
for item in declared:
|
|
461
|
+
entry = index.get(item["normalized"])
|
|
462
|
+
if entry is None:
|
|
463
|
+
report["missingFromLock"].append(item["name"])
|
|
464
|
+
continue
|
|
465
|
+
version = entry.get("version")
|
|
466
|
+
if not version or not item["specifier"]:
|
|
467
|
+
continue
|
|
468
|
+
try:
|
|
469
|
+
requirement = Requirement(item["raw"])
|
|
470
|
+
if not requirement.specifier:
|
|
471
|
+
continue
|
|
472
|
+
satisfied = requirement.specifier.contains(Version(version), prereleases=True)
|
|
473
|
+
except Exception:
|
|
474
|
+
continue
|
|
475
|
+
if not satisfied:
|
|
476
|
+
report["unsatisfiedInLock"].append(
|
|
477
|
+
{
|
|
478
|
+
"name": item["name"],
|
|
479
|
+
"specifier": item["specifier"],
|
|
480
|
+
"locked": version,
|
|
481
|
+
}
|
|
482
|
+
)
|
|
483
|
+
report["checkedCount"] = len(declared)
|
|
484
|
+
return report
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def installed_providers() -> dict:
|
|
488
|
+
"""Map top-level import name -> distributions providing it (host env only)."""
|
|
489
|
+
try:
|
|
490
|
+
from importlib.metadata import packages_distributions
|
|
491
|
+
|
|
492
|
+
return {
|
|
493
|
+
module: sorted(set(distributions))
|
|
494
|
+
for module, distributions in packages_distributions().items()
|
|
495
|
+
}
|
|
496
|
+
except Exception:
|
|
497
|
+
return {}
|
|
498
|
+
|
|
499
|
+
def _is_type_checking_test(test) -> bool:
|
|
500
|
+
"""Recognize `if TYPE_CHECKING:` and `if typing.TYPE_CHECKING:` guards.
|
|
501
|
+
|
|
502
|
+
Imports behind that guard only need to exist for type checkers, so they must
|
|
503
|
+
never be reported as a missing runtime dependency.
|
|
504
|
+
"""
|
|
505
|
+
if isinstance(test, ast.Name):
|
|
506
|
+
return test.id == "TYPE_CHECKING"
|
|
507
|
+
if isinstance(test, ast.Attribute):
|
|
508
|
+
return test.attr == "TYPE_CHECKING"
|
|
509
|
+
return False
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
class ImportCollector(ast.NodeVisitor):
|
|
513
|
+
def __init__(self) -> None:
|
|
514
|
+
self.all: set[str] = set()
|
|
515
|
+
self.type_checking: set[str] = set()
|
|
516
|
+
self._guard_depth = 0
|
|
517
|
+
|
|
518
|
+
def _record(self, name: str) -> None:
|
|
519
|
+
if not name:
|
|
520
|
+
return
|
|
521
|
+
self.all.add(name)
|
|
522
|
+
if self._guard_depth > 0:
|
|
523
|
+
self.type_checking.add(name)
|
|
524
|
+
|
|
525
|
+
def visit_If(self, node: ast.If) -> None:
|
|
526
|
+
guarded = _is_type_checking_test(node.test)
|
|
527
|
+
if guarded:
|
|
528
|
+
self._guard_depth += 1
|
|
529
|
+
for child in node.body:
|
|
530
|
+
self.visit(child)
|
|
531
|
+
if guarded:
|
|
532
|
+
self._guard_depth -= 1
|
|
533
|
+
for child in node.orelse:
|
|
534
|
+
self.visit(child)
|
|
535
|
+
|
|
536
|
+
def visit_Import(self, node: ast.Import) -> None:
|
|
537
|
+
for alias in node.names:
|
|
538
|
+
self._record(alias.name.split(".")[0])
|
|
539
|
+
|
|
540
|
+
def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
|
|
541
|
+
if node.level: # relative import -> always local
|
|
542
|
+
return
|
|
543
|
+
if node.module:
|
|
544
|
+
self._record(node.module.split(".")[0])
|
|
545
|
+
|
|
546
|
+
|
|
547
|
+
def scan_imports(root: Path, max_files: int) -> dict:
|
|
548
|
+
stdlib_available = hasattr(sys, "stdlib_module_names")
|
|
549
|
+
stdlib = set(getattr(sys, "stdlib_module_names", FALLBACK_STDLIB)) | {"__future__"}
|
|
550
|
+
layout, local_modules = detect_layout(root)
|
|
551
|
+
local = set(local_modules) | {"conftest", "setup", "__main__"}
|
|
552
|
+
|
|
553
|
+
providers = installed_providers()
|
|
554
|
+
files = []
|
|
555
|
+
unparsable = []
|
|
556
|
+
truncated = False
|
|
557
|
+
by_import: dict[str, list[str]] = {}
|
|
558
|
+
guarded_by_import: dict[str, list[str]] = {}
|
|
559
|
+
|
|
560
|
+
for dirpath, dirnames, filenames in os.walk(root):
|
|
561
|
+
dirnames[:] = sorted(
|
|
562
|
+
name
|
|
563
|
+
for name in dirnames
|
|
564
|
+
if name not in EXCLUDED_DIRS and not name.endswith(".egg-info")
|
|
565
|
+
)
|
|
566
|
+
for filename in sorted(filenames):
|
|
567
|
+
if not filename.endswith(".py"):
|
|
568
|
+
continue
|
|
569
|
+
if len(files) >= max_files:
|
|
570
|
+
truncated = True
|
|
571
|
+
dirnames[:] = []
|
|
572
|
+
break
|
|
573
|
+
path = Path(dirpath) / filename
|
|
574
|
+
relative = os.path.relpath(path, root)
|
|
575
|
+
try:
|
|
576
|
+
source = path.read_text(encoding="utf-8", errors="replace")
|
|
577
|
+
tree = ast.parse(source, filename=relative)
|
|
578
|
+
except SyntaxError as exc:
|
|
579
|
+
unparsable.append({"path": relative, "error": f"SyntaxError: {exc.msg}"})
|
|
580
|
+
continue
|
|
581
|
+
except (OSError, ValueError) as exc:
|
|
582
|
+
unparsable.append({"path": relative, "error": str(exc)})
|
|
583
|
+
continue
|
|
584
|
+
collector = ImportCollector()
|
|
585
|
+
collector.visit(tree)
|
|
586
|
+
names = collector.all
|
|
587
|
+
names.discard("")
|
|
588
|
+
files.append(
|
|
589
|
+
{
|
|
590
|
+
"path": relative,
|
|
591
|
+
"imports": sorted(names),
|
|
592
|
+
"typeCheckingImports": sorted(collector.type_checking),
|
|
593
|
+
}
|
|
594
|
+
)
|
|
595
|
+
for name in names:
|
|
596
|
+
by_import.setdefault(name, []).append(relative)
|
|
597
|
+
if name in collector.type_checking:
|
|
598
|
+
guarded_by_import.setdefault(name, []).append(relative)
|
|
599
|
+
|
|
600
|
+
third_party = []
|
|
601
|
+
for name in sorted(by_import):
|
|
602
|
+
if name in stdlib or name in local:
|
|
603
|
+
continue
|
|
604
|
+
importers = sorted(by_import[name])
|
|
605
|
+
guarded = guarded_by_import.get(name, [])
|
|
606
|
+
third_party.append(
|
|
607
|
+
{
|
|
608
|
+
"import": name,
|
|
609
|
+
"files": importers[:20],
|
|
610
|
+
"fileCount": len(importers),
|
|
611
|
+
"providers": providers.get(name, []),
|
|
612
|
+
"typeCheckingOnly": bool(guarded) and len(guarded) == len(importers),
|
|
613
|
+
"typeCheckingFiles": guarded[:20],
|
|
614
|
+
}
|
|
615
|
+
)
|
|
616
|
+
|
|
617
|
+
return {
|
|
618
|
+
"pythonVersion": sys.version.split()[0],
|
|
619
|
+
"stdlibAvailable": stdlib_available,
|
|
620
|
+
"layout": layout,
|
|
621
|
+
"localModules": sorted(local),
|
|
622
|
+
"files": files,
|
|
623
|
+
"thirdParty": third_party,
|
|
624
|
+
"providersUnavailable": not bool(providers),
|
|
625
|
+
"unparsable": unparsable,
|
|
626
|
+
"scannedFiles": len(files),
|
|
627
|
+
"truncated": truncated,
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
|
|
631
|
+
def describe_environment(root: Path) -> dict:
|
|
632
|
+
"""Describe the interpreter that is running this script.
|
|
633
|
+
|
|
634
|
+
The extension resolves an interpreter first and then asks it about itself,
|
|
635
|
+
so `sys.executable` here is the interpreter the project commands will use.
|
|
636
|
+
"""
|
|
637
|
+
base_prefix = getattr(sys, "base_prefix", sys.prefix)
|
|
638
|
+
venv_dir = os.environ.get("VIRTUAL_ENV")
|
|
639
|
+
if not venv_dir:
|
|
640
|
+
local_venv = root / ".venv"
|
|
641
|
+
if local_venv.is_dir():
|
|
642
|
+
venv_dir = str(local_venv)
|
|
643
|
+
return {
|
|
644
|
+
"version": sys.version.split()[0],
|
|
645
|
+
"versionInfo": list(sys.version_info[:3]),
|
|
646
|
+
"executable": sys.executable,
|
|
647
|
+
"prefix": sys.prefix,
|
|
648
|
+
"basePrefix": base_prefix,
|
|
649
|
+
"inVirtualEnvironment": sys.prefix != base_prefix,
|
|
650
|
+
"virtualEnv": os.environ.get("VIRTUAL_ENV"),
|
|
651
|
+
"condaPrefix": os.environ.get("CONDA_PREFIX"),
|
|
652
|
+
"candidateVenvDir": venv_dir,
|
|
653
|
+
"implementation": sys.implementation.name,
|
|
654
|
+
"platform": sys.platform,
|
|
655
|
+
"stdlibModuleNames": hasattr(sys, "stdlib_module_names"),
|
|
656
|
+
"tomlAvailable": toml_module()[0] is not None,
|
|
657
|
+
}
|
|
658
|
+
|
|
659
|
+
|
|
660
|
+
def parse_arguments(argv: list) -> tuple[argparse.Namespace | None, int]:
|
|
661
|
+
"""Parse CLI flags. Every value also arrives through the stdin request."""
|
|
662
|
+
parser = argparse.ArgumentParser(
|
|
663
|
+
prog="scan_project.py",
|
|
664
|
+
description=(
|
|
665
|
+
"Read-only Python project scanner. A JSON request is read from stdin and "
|
|
666
|
+
"one JSON document is written to stdout; diagnostics go to stderr."
|
|
667
|
+
),
|
|
668
|
+
epilog=(
|
|
669
|
+
"exit codes: 0 success, 1 unexpected failure, 2 invalid input. "
|
|
670
|
+
"Sections: environment, manifest, imports, all."
|
|
671
|
+
),
|
|
672
|
+
)
|
|
673
|
+
parser.add_argument(
|
|
674
|
+
"--mode",
|
|
675
|
+
default=None,
|
|
676
|
+
help="Comma-separated sections to scan (environment,manifest,imports) or all.",
|
|
677
|
+
)
|
|
678
|
+
parser.add_argument("--root", default=None, help="Project root; defaults to the cwd.")
|
|
679
|
+
parser.add_argument("--max-files", type=int, default=None, help="Cap on scanned Python files.")
|
|
680
|
+
parser.add_argument(
|
|
681
|
+
"--version",
|
|
682
|
+
action="version",
|
|
683
|
+
version=f"scan_project.py (scanner protocol {SCANNER_VERSION})",
|
|
684
|
+
)
|
|
685
|
+
try:
|
|
686
|
+
return parser.parse_args(argv), EXIT_OK
|
|
687
|
+
except SystemExit as exc:
|
|
688
|
+
code = exc.code if isinstance(exc.code, int) else EXIT_USAGE
|
|
689
|
+
return None, (EXIT_OK if code == 0 else EXIT_USAGE)
|
|
690
|
+
|
|
691
|
+
|
|
692
|
+
def resolve_sections(raw) -> list:
|
|
693
|
+
"""Expand a mode value into known sections, rejecting unknown ones."""
|
|
694
|
+
if raw is None or raw == "" or raw == "all":
|
|
695
|
+
return list(KNOWN_SECTIONS)
|
|
696
|
+
if not isinstance(raw, str):
|
|
697
|
+
raise ValueError(f"mode must be a string, got {type(raw).__name__}")
|
|
698
|
+
requested = [part.strip() for part in raw.split(",") if part.strip()]
|
|
699
|
+
if not requested:
|
|
700
|
+
raise ValueError("mode must not be empty")
|
|
701
|
+
if "all" in requested:
|
|
702
|
+
return list(KNOWN_SECTIONS)
|
|
703
|
+
unknown = [part for part in requested if part not in KNOWN_SECTIONS]
|
|
704
|
+
if unknown:
|
|
705
|
+
raise ValueError(
|
|
706
|
+
f"unknown mode section(s): {', '.join(unknown)}; expected {', '.join(KNOWN_SECTIONS)} or all"
|
|
707
|
+
)
|
|
708
|
+
# Preserve canonical order so the document layout does not depend on input order.
|
|
709
|
+
return [section for section in KNOWN_SECTIONS if section in requested]
|
|
710
|
+
|
|
711
|
+
|
|
712
|
+
def fail(message: str, code: int) -> int:
|
|
713
|
+
"""Report a problem on stdout and stderr: stdout stays machine-readable."""
|
|
714
|
+
print(json.dumps({"error": message}))
|
|
715
|
+
print(f"scan_project.py: {message}", file=sys.stderr)
|
|
716
|
+
return code
|
|
717
|
+
|
|
718
|
+
|
|
719
|
+
def main(argv: list | None = None) -> int:
|
|
720
|
+
args, argument_code = parse_arguments(list(sys.argv[1:] if argv is None else argv))
|
|
721
|
+
if args is None:
|
|
722
|
+
return argument_code
|
|
723
|
+
|
|
724
|
+
raw = sys.stdin.read()
|
|
725
|
+
try:
|
|
726
|
+
request = json.loads(raw) if raw.strip() else {}
|
|
727
|
+
except json.JSONDecodeError as exc:
|
|
728
|
+
return fail(f"invalid request JSON: {exc}", EXIT_USAGE)
|
|
729
|
+
if not isinstance(request, dict):
|
|
730
|
+
return fail("the request must be a JSON object", EXIT_USAGE)
|
|
731
|
+
|
|
732
|
+
# Explicit flags take precedence over the request, then the defaults.
|
|
733
|
+
raw_mode = args.mode if args.mode is not None else request.get("mode")
|
|
734
|
+
raw_root = args.root if args.root is not None else request.get("root")
|
|
735
|
+
raw_max = args.max_files if args.max_files is not None else request.get("maxFiles")
|
|
736
|
+
try:
|
|
737
|
+
sections = resolve_sections(raw_mode)
|
|
738
|
+
except ValueError as exc:
|
|
739
|
+
return fail(str(exc), EXIT_USAGE)
|
|
740
|
+
try:
|
|
741
|
+
max_files = int(raw_max if raw_max is not None else 2000)
|
|
742
|
+
except (TypeError, ValueError):
|
|
743
|
+
return fail(f"maxFiles must be an integer, got {raw_max!r}", EXIT_USAGE)
|
|
744
|
+
if max_files < 1:
|
|
745
|
+
return fail("maxFiles must be at least 1", EXIT_USAGE)
|
|
746
|
+
|
|
747
|
+
root = Path(raw_root or os.getcwd()).resolve()
|
|
748
|
+
if not root.is_dir():
|
|
749
|
+
return fail(f"not a directory: {root}", EXIT_USAGE)
|
|
750
|
+
|
|
751
|
+
payload: dict = {
|
|
752
|
+
"scannerVersion": SCANNER_VERSION,
|
|
753
|
+
"root": str(root),
|
|
754
|
+
"mode": ",".join(sections),
|
|
755
|
+
"pythonVersion": sys.version.split()[0],
|
|
756
|
+
"tomlAvailable": toml_module()[0] is not None,
|
|
757
|
+
}
|
|
758
|
+
if "environment" in sections:
|
|
759
|
+
payload["environment"] = describe_environment(root)
|
|
760
|
+
if "manifest" in sections:
|
|
761
|
+
payload["manifest"] = scan_manifests(root)
|
|
762
|
+
payload["lock"] = scan_lock(root)
|
|
763
|
+
payload["lockComparison"] = compare_lock(payload["lock"], payload["manifest"])
|
|
764
|
+
if "imports" in sections:
|
|
765
|
+
payload["imports"] = scan_imports(root, max_files)
|
|
766
|
+
|
|
767
|
+
print(json.dumps(payload))
|
|
768
|
+
return EXIT_OK
|
|
769
|
+
|
|
770
|
+
|
|
771
|
+
if __name__ == "__main__":
|
|
772
|
+
try:
|
|
773
|
+
sys.exit(main())
|
|
774
|
+
except KeyboardInterrupt:
|
|
775
|
+
sys.exit(EXIT_FAILURE)
|
|
776
|
+
except Exception as exc: # a crash must still reach the caller as structured JSON
|
|
777
|
+
sys.exit(fail(f"scanner failed: {type(exc).__name__}: {exc}", EXIT_FAILURE))
|