python-auto-req 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- auto_req/__init__.py +16 -0
- auto_req/__main__.py +5 -0
- auto_req/cli.py +162 -0
- auto_req/resolver.py +258 -0
- auto_req/scanner.py +154 -0
- python_auto_req-0.1.0.dist-info/METADATA +177 -0
- python_auto_req-0.1.0.dist-info/RECORD +10 -0
- python_auto_req-0.1.0.dist-info/WHEEL +4 -0
- python_auto_req-0.1.0.dist-info/entry_points.txt +2 -0
- python_auto_req-0.1.0.dist-info/licenses/LICENSE +21 -0
auto_req/__init__.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""python-auto-req – scan project files and populate requirements.txt."""
|
|
2
|
+
|
|
3
|
+
from .scanner import collect_imports, imports_from_source, imports_from_notebook, stdlib_module_names
|
|
4
|
+
from .resolver import resolve_all, resolve_from_env, search_pypi, ngram_score
|
|
5
|
+
|
|
6
|
+
__version__ = "0.1.0"
|
|
7
|
+
__all__ = [
|
|
8
|
+
"collect_imports",
|
|
9
|
+
"imports_from_source",
|
|
10
|
+
"imports_from_notebook",
|
|
11
|
+
"stdlib_module_names",
|
|
12
|
+
"resolve_all",
|
|
13
|
+
"resolve_from_env",
|
|
14
|
+
"search_pypi",
|
|
15
|
+
"ngram_score",
|
|
16
|
+
]
|
auto_req/__main__.py
ADDED
auto_req/cli.py
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"""
|
|
2
|
+
cli.py – command-line interface for python-auto-req.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import argparse
|
|
8
|
+
import re
|
|
9
|
+
import subprocess
|
|
10
|
+
import sys
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from .scanner import collect_imports, stdlib_module_names
|
|
14
|
+
from .resolver import resolve_all
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
# ---------------------------------------------------------------------------
|
|
18
|
+
# requirements.txt helpers
|
|
19
|
+
# ---------------------------------------------------------------------------
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def read_existing_requirements(path: Path) -> set[str]:
|
|
23
|
+
"""Return lower-cased dist names already present in *path*."""
|
|
24
|
+
if not path.exists():
|
|
25
|
+
return set()
|
|
26
|
+
existing: set[str] = set()
|
|
27
|
+
for line in path.read_text(encoding="utf-8").splitlines():
|
|
28
|
+
line = line.strip()
|
|
29
|
+
if not line or line.startswith("#"):
|
|
30
|
+
continue
|
|
31
|
+
name = re.split(r"[=<>!~\s\[;]", line)[0].lower()
|
|
32
|
+
existing.add(name)
|
|
33
|
+
return existing
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
# ---------------------------------------------------------------------------
|
|
37
|
+
# Entry point
|
|
38
|
+
# ---------------------------------------------------------------------------
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def main(argv: list[str] | None = None) -> int:
|
|
42
|
+
"""
|
|
43
|
+
Run the scanner and optionally write requirements.txt.
|
|
44
|
+
|
|
45
|
+
Returns 0 on success, 1 on error.
|
|
46
|
+
"""
|
|
47
|
+
parser = argparse.ArgumentParser(
|
|
48
|
+
prog="auto-req",
|
|
49
|
+
description=(
|
|
50
|
+
"Scan .py / .ipynb project files and add discovered third-party "
|
|
51
|
+
"packages to requirements.txt.\n\n"
|
|
52
|
+
"Only packages explicitly imported in your source files are considered.\n"
|
|
53
|
+
"Packages installed in .venv but never imported are ignored."
|
|
54
|
+
),
|
|
55
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
56
|
+
)
|
|
57
|
+
parser.add_argument(
|
|
58
|
+
"--dry-run",
|
|
59
|
+
type=int,
|
|
60
|
+
default=1,
|
|
61
|
+
metavar="0|1",
|
|
62
|
+
help="1 = preview only (default); 0 = write requirements.txt",
|
|
63
|
+
)
|
|
64
|
+
parser.add_argument(
|
|
65
|
+
"--auto",
|
|
66
|
+
action="store_true",
|
|
67
|
+
help="Always pick the top PyPI match for unknown modules (no prompts)",
|
|
68
|
+
)
|
|
69
|
+
parser.add_argument(
|
|
70
|
+
"--dir",
|
|
71
|
+
type=Path,
|
|
72
|
+
default=Path("."),
|
|
73
|
+
metavar="DIR",
|
|
74
|
+
help="Project root to scan (default: current directory)",
|
|
75
|
+
)
|
|
76
|
+
parser.add_argument(
|
|
77
|
+
"--requirements",
|
|
78
|
+
type=Path,
|
|
79
|
+
default=Path("requirements.txt"),
|
|
80
|
+
metavar="FILE",
|
|
81
|
+
help="Path to requirements.txt (default: ./requirements.txt)",
|
|
82
|
+
)
|
|
83
|
+
args = parser.parse_args(argv)
|
|
84
|
+
|
|
85
|
+
dry_run: bool = bool(args.dry_run)
|
|
86
|
+
root: Path = args.dir.resolve()
|
|
87
|
+
req_path: Path = args.requirements
|
|
88
|
+
|
|
89
|
+
print(f"Scanning : {root}")
|
|
90
|
+
print(f"Dry-run : {dry_run} (pass --dry-run 0 to write)")
|
|
91
|
+
print(f"Mode : {'auto' if args.auto else 'interactive'}\n")
|
|
92
|
+
|
|
93
|
+
# ---- scan source files (excludes .venv and other non-project dirs) ----
|
|
94
|
+
stdlib = stdlib_module_names()
|
|
95
|
+
all_imports = collect_imports(root)
|
|
96
|
+
|
|
97
|
+
if not all_imports:
|
|
98
|
+
print("No imports detected.")
|
|
99
|
+
return 0
|
|
100
|
+
|
|
101
|
+
print(f"Imports found in project files: {len(all_imports)}")
|
|
102
|
+
|
|
103
|
+
# ---- resolve module names → dist names --------------------------------
|
|
104
|
+
resolved, skipped = resolve_all(all_imports, stdlib, args.auto)
|
|
105
|
+
|
|
106
|
+
# ---- compare against existing requirements.txt ------------------------
|
|
107
|
+
existing = read_existing_requirements(req_path)
|
|
108
|
+
new_pkgs = {
|
|
109
|
+
dist: ver
|
|
110
|
+
for dist, ver in resolved.items()
|
|
111
|
+
if dist.lower() not in existing
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
print(f"\nPackages resolved : {len(resolved)}")
|
|
115
|
+
print(f"Already in req : {len(resolved) - len(new_pkgs)}")
|
|
116
|
+
print(f"New to add : {len(new_pkgs)}")
|
|
117
|
+
if skipped:
|
|
118
|
+
print(f"Skipped (manual) : {len(skipped)} -> {', '.join(skipped)}")
|
|
119
|
+
|
|
120
|
+
if not new_pkgs:
|
|
121
|
+
print("\nrequirements.txt is already up-to-date.")
|
|
122
|
+
return 0
|
|
123
|
+
|
|
124
|
+
lines_to_add = [
|
|
125
|
+
f"{dist}=={ver}" if ver else dist
|
|
126
|
+
for dist, ver in sorted(new_pkgs.items())
|
|
127
|
+
]
|
|
128
|
+
|
|
129
|
+
print("\nPackages to add:")
|
|
130
|
+
for line in lines_to_add:
|
|
131
|
+
print(f" + {line}")
|
|
132
|
+
|
|
133
|
+
if dry_run:
|
|
134
|
+
print("\n[dry-run] No files were modified. Pass --dry-run 0 to apply.")
|
|
135
|
+
return 0
|
|
136
|
+
|
|
137
|
+
# ---- write ------------------------------------------------------------
|
|
138
|
+
with req_path.open("a", encoding="utf-8") as fh:
|
|
139
|
+
if req_path.exists() and req_path.stat().st_size > 0:
|
|
140
|
+
fh.write("\n")
|
|
141
|
+
fh.write("\n".join(lines_to_add) + "\n")
|
|
142
|
+
|
|
143
|
+
print(f"\nWritten to {req_path} ({len(lines_to_add)} package(s) added).")
|
|
144
|
+
|
|
145
|
+
# ---- offer pip install ------------------------------------------------
|
|
146
|
+
try:
|
|
147
|
+
answer = input(f"\nRun 'pip install -r {req_path}' now? [y/N] ").strip().lower()
|
|
148
|
+
except (EOFError, KeyboardInterrupt):
|
|
149
|
+
print()
|
|
150
|
+
return 0
|
|
151
|
+
|
|
152
|
+
if answer == "y":
|
|
153
|
+
print()
|
|
154
|
+
result = subprocess.run(
|
|
155
|
+
[sys.executable, "-m", "pip", "install", "-r", str(req_path)],
|
|
156
|
+
check=False,
|
|
157
|
+
)
|
|
158
|
+
return result.returncode
|
|
159
|
+
else:
|
|
160
|
+
print("Skipped. Run it manually when ready.")
|
|
161
|
+
|
|
162
|
+
return 0
|
auto_req/resolver.py
ADDED
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
"""
|
|
2
|
+
resolver.py – map imported module names to PyPI distribution names.
|
|
3
|
+
|
|
4
|
+
Resolution order for each module:
|
|
5
|
+
1. Installed packages in the active Python environment (no I/O)
|
|
6
|
+
2. PyPI search + interactive prompt (or --auto for unattended use)
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import sys
|
|
12
|
+
import urllib.parse
|
|
13
|
+
import urllib.request
|
|
14
|
+
import re
|
|
15
|
+
from importlib.metadata import PackageNotFoundError, packages_distributions
|
|
16
|
+
from importlib.metadata import version as pkg_version
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
# ---------------------------------------------------------------------------
|
|
20
|
+
# Confident resolution: installed env only
|
|
21
|
+
# ---------------------------------------------------------------------------
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def resolve_from_env(module: str) -> str | None:
|
|
25
|
+
"""
|
|
26
|
+
Return the distribution name for *module* if it is installed in the
|
|
27
|
+
current environment, else None.
|
|
28
|
+
|
|
29
|
+
This uses only packages that are actually importable – it never surfaces
|
|
30
|
+
packages that happen to be in .venv but aren't referenced in project files.
|
|
31
|
+
The caller is responsible for only passing modules that were found by
|
|
32
|
+
scanning source files.
|
|
33
|
+
"""
|
|
34
|
+
dists = packages_distributions().get(module)
|
|
35
|
+
return dists[0] if dists else None
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def installed_version(dist: str) -> str:
|
|
39
|
+
"""Return the installed version of *dist*, or '' if not installed."""
|
|
40
|
+
try:
|
|
41
|
+
return pkg_version(dist)
|
|
42
|
+
except PackageNotFoundError:
|
|
43
|
+
return ""
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
# ---------------------------------------------------------------------------
|
|
47
|
+
# PyPI search + n-gram ranking
|
|
48
|
+
# ---------------------------------------------------------------------------
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def search_pypi(query: str, n: int = 5) -> list[str]:
|
|
52
|
+
"""
|
|
53
|
+
Return up to *n* package names from a PyPI simple search for *query*.
|
|
54
|
+
Returns an empty list on any network/parse failure.
|
|
55
|
+
"""
|
|
56
|
+
url = "https://pypi.org/search/?" + urllib.parse.urlencode({"q": query})
|
|
57
|
+
try:
|
|
58
|
+
req = urllib.request.Request(
|
|
59
|
+
url, headers={"User-Agent": "python-auto-req/1.0 (github.com/Amit-Roy/python-auto-req)"}
|
|
60
|
+
)
|
|
61
|
+
with urllib.request.urlopen(req, timeout=8) as resp:
|
|
62
|
+
html = resp.read().decode("utf-8", errors="ignore")
|
|
63
|
+
except Exception as exc:
|
|
64
|
+
print(f" [warn] PyPI search failed: {exc}")
|
|
65
|
+
return []
|
|
66
|
+
|
|
67
|
+
names = re.findall(
|
|
68
|
+
r'class="package-snippet__name"[^>]*>\s*([^\s<]+)\s*<', html
|
|
69
|
+
)
|
|
70
|
+
return names[:n]
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def ngram_score(a: str, b: str, n: int = 2) -> float:
|
|
74
|
+
"""
|
|
75
|
+
Character n-gram Jaccard similarity in [0, 1].
|
|
76
|
+
Higher = more similar.
|
|
77
|
+
"""
|
|
78
|
+
a, b = a.lower(), b.lower()
|
|
79
|
+
if not a or not b:
|
|
80
|
+
return 0.0
|
|
81
|
+
|
|
82
|
+
def ngrams(s: str) -> set[str]:
|
|
83
|
+
return {s[i : i + n] for i in range(len(s) - n + 1)} if len(s) >= n else {s}
|
|
84
|
+
|
|
85
|
+
sa, sb = ngrams(a), ngrams(b)
|
|
86
|
+
union = sa | sb
|
|
87
|
+
return len(sa & sb) / len(union) if union else 0.0
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def rank_candidates(module: str, candidates: list[str]) -> list[str]:
|
|
91
|
+
"""Re-rank *candidates* by n-gram similarity to *module*, best first."""
|
|
92
|
+
return sorted(candidates, key=lambda c: -ngram_score(module, c))
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
# ---------------------------------------------------------------------------
|
|
96
|
+
# Interactive / auto prompt
|
|
97
|
+
# ---------------------------------------------------------------------------
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
class StopResolution(Exception):
|
|
101
|
+
"""Raised when the user chooses to abort the resolution loop."""
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _prompt(
|
|
105
|
+
module: str,
|
|
106
|
+
candidates: list[str],
|
|
107
|
+
auto_session: bool,
|
|
108
|
+
) -> tuple[str | None, bool]:
|
|
109
|
+
"""
|
|
110
|
+
Present the resolution menu for one unknown *module*.
|
|
111
|
+
|
|
112
|
+
Returns
|
|
113
|
+
-------
|
|
114
|
+
(dist_name, new_auto_session)
|
|
115
|
+
dist_name is None → skip this module
|
|
116
|
+
Raises StopResolution when user presses [q].
|
|
117
|
+
"""
|
|
118
|
+
if auto_session:
|
|
119
|
+
pick = candidates[0] if candidates else module
|
|
120
|
+
print(f" [auto] '{module}' -> '{pick}'")
|
|
121
|
+
return pick, True
|
|
122
|
+
|
|
123
|
+
print(f"\n Unrecognised module: '{module}'")
|
|
124
|
+
if candidates:
|
|
125
|
+
print(" Top PyPI matches:")
|
|
126
|
+
for i, name in enumerate(candidates, 1):
|
|
127
|
+
print(f" [{i}] {name} (similarity {ngram_score(module, name):.2f})")
|
|
128
|
+
else:
|
|
129
|
+
print(" No PyPI results found.")
|
|
130
|
+
|
|
131
|
+
print()
|
|
132
|
+
print(" [1-5] select a match above")
|
|
133
|
+
print(" [a] auto-select top match for ALL remaining unknowns")
|
|
134
|
+
print(" [m] enter the correct dist name manually")
|
|
135
|
+
print(" [s] skip (omit from requirements, continue)")
|
|
136
|
+
print(" [q] quit (stop resolution now)")
|
|
137
|
+
print()
|
|
138
|
+
|
|
139
|
+
while True:
|
|
140
|
+
try:
|
|
141
|
+
choice = input(" Your choice: ").strip().lower()
|
|
142
|
+
except (EOFError, KeyboardInterrupt):
|
|
143
|
+
print()
|
|
144
|
+
raise StopResolution
|
|
145
|
+
|
|
146
|
+
if choice == "q":
|
|
147
|
+
raise StopResolution
|
|
148
|
+
|
|
149
|
+
if choice == "s":
|
|
150
|
+
return None, False
|
|
151
|
+
|
|
152
|
+
if choice == "m":
|
|
153
|
+
try:
|
|
154
|
+
name = input(" Dist name: ").strip()
|
|
155
|
+
except (EOFError, KeyboardInterrupt):
|
|
156
|
+
print()
|
|
157
|
+
raise StopResolution
|
|
158
|
+
return (name or None), False
|
|
159
|
+
|
|
160
|
+
if choice == "a":
|
|
161
|
+
pick = candidates[0] if candidates else module
|
|
162
|
+
print(f" [auto] '{module}' -> '{pick}'")
|
|
163
|
+
return pick, True
|
|
164
|
+
|
|
165
|
+
if choice.isdigit():
|
|
166
|
+
idx = int(choice) - 1
|
|
167
|
+
if 0 <= idx < len(candidates):
|
|
168
|
+
return candidates[idx], False
|
|
169
|
+
print(f" Invalid number. Pick 1–{len(candidates)}.")
|
|
170
|
+
continue
|
|
171
|
+
|
|
172
|
+
print(" Unknown choice. Try again.")
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def resolve_via_pypi(
|
|
176
|
+
module: str,
|
|
177
|
+
auto_mode: bool,
|
|
178
|
+
) -> tuple[str | None, bool]:
|
|
179
|
+
"""
|
|
180
|
+
Search PyPI for *module* and resolve via prompt (or auto).
|
|
181
|
+
|
|
182
|
+
Returns
|
|
183
|
+
-------
|
|
184
|
+
(dist_name, updated_auto_mode)
|
|
185
|
+
dist_name is None → user chose to skip
|
|
186
|
+
Raises StopResolution on quit.
|
|
187
|
+
"""
|
|
188
|
+
if not auto_mode:
|
|
189
|
+
print(f"\n Searching PyPI for '{module}'...", end=" ", flush=True)
|
|
190
|
+
|
|
191
|
+
raw = search_pypi(module)
|
|
192
|
+
candidates = rank_candidates(module, raw) if raw else []
|
|
193
|
+
|
|
194
|
+
if not auto_mode:
|
|
195
|
+
print(f"{len(candidates)} result(s)." if candidates else "no results.")
|
|
196
|
+
|
|
197
|
+
return _prompt(module, candidates, auto_mode)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
# ---------------------------------------------------------------------------
|
|
201
|
+
# Full resolution pipeline
|
|
202
|
+
# ---------------------------------------------------------------------------
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def resolve_all(
|
|
206
|
+
module_names: set[str],
|
|
207
|
+
stdlib: set[str],
|
|
208
|
+
auto_mode: bool,
|
|
209
|
+
) -> tuple[dict[str, str], list[str]]:
|
|
210
|
+
"""
|
|
211
|
+
Resolve *module_names* (found by scanning project files) to a dict of
|
|
212
|
+
``{dist_name: version}``.
|
|
213
|
+
|
|
214
|
+
Modules that are part of the stdlib or Python builtins are silently
|
|
215
|
+
ignored. Unknown modules go through PyPI search + user interaction.
|
|
216
|
+
|
|
217
|
+
Parameters
|
|
218
|
+
----------
|
|
219
|
+
module_names:
|
|
220
|
+
Top-level module names collected from project source files only.
|
|
221
|
+
stdlib:
|
|
222
|
+
Set of stdlib module names (from ``scanner.stdlib_module_names()``).
|
|
223
|
+
auto_mode:
|
|
224
|
+
When True, always pick the top PyPI result without prompting.
|
|
225
|
+
|
|
226
|
+
Returns
|
|
227
|
+
-------
|
|
228
|
+
packages:
|
|
229
|
+
``{dist_name: installed_version_or_empty}``
|
|
230
|
+
skipped:
|
|
231
|
+
Module names the user chose to skip or that couldn't be resolved.
|
|
232
|
+
"""
|
|
233
|
+
packages: dict[str, str] = {}
|
|
234
|
+
skipped: list[str] = []
|
|
235
|
+
auto_session = auto_mode
|
|
236
|
+
|
|
237
|
+
for module in sorted(module_names):
|
|
238
|
+
if module in stdlib or module in sys.builtin_module_names:
|
|
239
|
+
continue
|
|
240
|
+
|
|
241
|
+
# Fast path: installed in current env → confident mapping
|
|
242
|
+
dist = resolve_from_env(module)
|
|
243
|
+
|
|
244
|
+
if dist is None:
|
|
245
|
+
# Slow path: PyPI search + user input
|
|
246
|
+
try:
|
|
247
|
+
dist, auto_session = resolve_via_pypi(module, auto_session)
|
|
248
|
+
except StopResolution:
|
|
249
|
+
print("\n Resolution stopped. Remaining modules skipped.")
|
|
250
|
+
break
|
|
251
|
+
|
|
252
|
+
if dist is None:
|
|
253
|
+
skipped.append(module)
|
|
254
|
+
continue
|
|
255
|
+
|
|
256
|
+
packages[dist] = installed_version(dist)
|
|
257
|
+
|
|
258
|
+
return packages, skipped
|
auto_req/scanner.py
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"""
|
|
2
|
+
scanner.py – stdlib detection and import extraction from .py / .ipynb files.
|
|
3
|
+
|
|
4
|
+
Only imports that are explicitly referenced in project source files are
|
|
5
|
+
collected. The virtual-environment directory (and other non-project paths)
|
|
6
|
+
are excluded so that installed-but-unreferenced packages are never surfaced.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import ast
|
|
12
|
+
import json
|
|
13
|
+
import re
|
|
14
|
+
import sys
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
# Directories that are never part of project source code.
|
|
18
|
+
_EXCLUDE_DIRS: frozenset[str] = frozenset(
|
|
19
|
+
{
|
|
20
|
+
".venv",
|
|
21
|
+
"venv",
|
|
22
|
+
".env",
|
|
23
|
+
"env",
|
|
24
|
+
"__pycache__",
|
|
25
|
+
".git",
|
|
26
|
+
".hg",
|
|
27
|
+
".tox",
|
|
28
|
+
".nox",
|
|
29
|
+
"node_modules",
|
|
30
|
+
"dist",
|
|
31
|
+
"build",
|
|
32
|
+
"site-packages",
|
|
33
|
+
".mypy_cache",
|
|
34
|
+
".pytest_cache",
|
|
35
|
+
".ruff_cache",
|
|
36
|
+
}
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
# ---------------------------------------------------------------------------
|
|
41
|
+
# stdlib detection
|
|
42
|
+
# ---------------------------------------------------------------------------
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def stdlib_module_names() -> set[str]:
|
|
46
|
+
"""Return the set of top-level stdlib module names for the running Python."""
|
|
47
|
+
if hasattr(sys, "stdlib_module_names"): # Python 3.10+
|
|
48
|
+
return set(sys.stdlib_module_names) # type: ignore[attr-defined]
|
|
49
|
+
# Fallback for older Python
|
|
50
|
+
import sysconfig
|
|
51
|
+
|
|
52
|
+
names: set[str] = set()
|
|
53
|
+
for key in ("stdlib", "platstdlib"):
|
|
54
|
+
p = sysconfig.get_paths().get(key, "")
|
|
55
|
+
if not p:
|
|
56
|
+
continue
|
|
57
|
+
for item in Path(p).iterdir():
|
|
58
|
+
names.add(item.stem if item.is_file() else item.name)
|
|
59
|
+
return names
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
# ---------------------------------------------------------------------------
|
|
63
|
+
# Import extraction
|
|
64
|
+
# ---------------------------------------------------------------------------
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def imports_from_source(source: str) -> set[str]:
|
|
68
|
+
"""
|
|
69
|
+
Parse *source* as Python and return every top-level imported module name.
|
|
70
|
+
|
|
71
|
+
Only absolute imports are considered (relative imports are project-internal).
|
|
72
|
+
Returns an empty set on SyntaxError rather than raising.
|
|
73
|
+
"""
|
|
74
|
+
names: set[str] = set()
|
|
75
|
+
try:
|
|
76
|
+
tree = ast.parse(source)
|
|
77
|
+
except SyntaxError:
|
|
78
|
+
return names
|
|
79
|
+
|
|
80
|
+
for node in ast.walk(tree):
|
|
81
|
+
if isinstance(node, ast.Import):
|
|
82
|
+
for alias in node.names:
|
|
83
|
+
names.add(alias.name.split(".")[0])
|
|
84
|
+
elif isinstance(node, ast.ImportFrom):
|
|
85
|
+
if node.module and node.level == 0:
|
|
86
|
+
names.add(node.module.split(".")[0])
|
|
87
|
+
|
|
88
|
+
return names
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def imports_from_notebook(path: Path) -> set[str]:
|
|
92
|
+
"""
|
|
93
|
+
Parse a Jupyter notebook and return every top-level imported module name
|
|
94
|
+
across all code cells.
|
|
95
|
+
"""
|
|
96
|
+
names: set[str] = set()
|
|
97
|
+
try:
|
|
98
|
+
nb = json.loads(path.read_text(encoding="utf-8"))
|
|
99
|
+
except (json.JSONDecodeError, OSError):
|
|
100
|
+
return names
|
|
101
|
+
|
|
102
|
+
for cell in nb.get("cells", []):
|
|
103
|
+
if cell.get("cell_type") != "code":
|
|
104
|
+
continue
|
|
105
|
+
source = "".join(cell.get("source", []))
|
|
106
|
+
# Strip IPython line/cell magics and shell commands before parsing
|
|
107
|
+
source = re.sub(r"^\s*%%?\w+.*$", "", source, flags=re.MULTILINE)
|
|
108
|
+
source = re.sub(r"^\s*!.*$", "", source, flags=re.MULTILINE)
|
|
109
|
+
names |= imports_from_source(source)
|
|
110
|
+
|
|
111
|
+
return names
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
# ---------------------------------------------------------------------------
|
|
115
|
+
# Directory walker
|
|
116
|
+
# ---------------------------------------------------------------------------
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _is_excluded(path: Path) -> bool:
|
|
120
|
+
"""Return True if *path* lives inside an excluded directory."""
|
|
121
|
+
return any(part in _EXCLUDE_DIRS for part in path.parts)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def collect_imports(root: Path, self_path: Path | None = None) -> set[str]:
|
|
125
|
+
"""
|
|
126
|
+
Recursively walk *root* and collect every top-level imported module name
|
|
127
|
+
from .py and .ipynb files, skipping excluded directories.
|
|
128
|
+
|
|
129
|
+
Parameters
|
|
130
|
+
----------
|
|
131
|
+
root:
|
|
132
|
+
Directory to scan.
|
|
133
|
+
self_path:
|
|
134
|
+
Absolute path of this script/package – excluded from scanning so the
|
|
135
|
+
tool doesn't try to resolve its own imports.
|
|
136
|
+
"""
|
|
137
|
+
names: set[str] = set()
|
|
138
|
+
|
|
139
|
+
for py in root.rglob("*.py"):
|
|
140
|
+
if _is_excluded(py.relative_to(root)):
|
|
141
|
+
continue
|
|
142
|
+
if self_path and py.resolve() == self_path:
|
|
143
|
+
continue
|
|
144
|
+
try:
|
|
145
|
+
names |= imports_from_source(py.read_text(encoding="utf-8", errors="ignore"))
|
|
146
|
+
except OSError:
|
|
147
|
+
pass
|
|
148
|
+
|
|
149
|
+
for nb in root.rglob("*.ipynb"):
|
|
150
|
+
if _is_excluded(nb.relative_to(root)):
|
|
151
|
+
continue
|
|
152
|
+
names |= imports_from_notebook(nb)
|
|
153
|
+
|
|
154
|
+
return names
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: python-auto-req
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Scan project source files and populate requirements.txt – with PyPI search and interactive resolution.
|
|
5
|
+
Project-URL: Homepage, https://github.com/Amit-Roy/python-auto-req
|
|
6
|
+
Project-URL: Repository, https://github.com/Amit-Roy/python-auto-req
|
|
7
|
+
Project-URL: Issues, https://github.com/Amit-Roy/python-auto-req/issues
|
|
8
|
+
Author-email: Amit Roy <a.roy.0593@gmail.com>
|
|
9
|
+
License: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: dependencies,pypi,requirements,scanner
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
22
|
+
Classifier: Topic :: Utilities
|
|
23
|
+
Requires-Python: >=3.10
|
|
24
|
+
Provides-Extra: dev
|
|
25
|
+
Requires-Dist: build; extra == 'dev'
|
|
26
|
+
Requires-Dist: hatchling; extra == 'dev'
|
|
27
|
+
Requires-Dist: pytest; extra == 'dev'
|
|
28
|
+
Requires-Dist: twine; extra == 'dev'
|
|
29
|
+
Description-Content-Type: text/markdown
|
|
30
|
+
|
|
31
|
+
# python-auto-req
|
|
32
|
+
|
|
33
|
+
Scan your Python project's `.py` and `.ipynb` files, discover every third-party
|
|
34
|
+
package that is actually **imported**, and add them to `requirements.txt`.
|
|
35
|
+
|
|
36
|
+
> **Only packages explicitly referenced in your source files are considered.**
|
|
37
|
+
> Packages installed in `.venv` but never imported are ignored.
|
|
38
|
+
|
|
39
|
+
---
|
|
40
|
+
|
|
41
|
+
## Features
|
|
42
|
+
|
|
43
|
+
- Recursively scans `.py` and `.ipynb` files (skips `.venv`, `.git`, `__pycache__`, etc.)
|
|
44
|
+
- Uses the active Python environment to resolve module → distribution name + version
|
|
45
|
+
- Falls back to a **live PyPI search** for unknown modules with an interactive menu:
|
|
46
|
+
- Pick from top-5 results ranked by n-gram similarity
|
|
47
|
+
- Auto-select top match (`[a]` or `--auto`)
|
|
48
|
+
- Enter a name manually (`[m]`)
|
|
49
|
+
- Skip (`[s]`) or quit (`[q]`)
|
|
50
|
+
- Offers to run `pip install -r requirements.txt` after writing
|
|
51
|
+
|
|
52
|
+
---
|
|
53
|
+
|
|
54
|
+
## Installation
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
pip install python-auto-req
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
Or from source:
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
git clone https://github.com/Amit-Roy/python-auto-req
|
|
64
|
+
cd python-auto-req
|
|
65
|
+
pip install -e .
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
---
|
|
69
|
+
|
|
70
|
+
## Usage
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
# Preview what would be added (dry-run, interactive)
|
|
74
|
+
auto-req
|
|
75
|
+
|
|
76
|
+
# Write requirements.txt
|
|
77
|
+
auto-req --dry-run 0
|
|
78
|
+
|
|
79
|
+
# Write requirements.txt, auto-pick top PyPI match for unknowns
|
|
80
|
+
auto-req --dry-run 0 --auto
|
|
81
|
+
|
|
82
|
+
# Scan a specific directory
|
|
83
|
+
auto-req --dir /path/to/project --dry-run 0
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Or via Python:
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
python -m auto_req --dry-run 0
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
### CLI options
|
|
93
|
+
|
|
94
|
+
| Option | Default | Description |
|
|
95
|
+
|---|---|---|
|
|
96
|
+
| `--dry-run 0\|1` | `1` | `1` = preview only; `0` = write to file |
|
|
97
|
+
| `--auto` | off | Always pick the top PyPI match (no prompts) |
|
|
98
|
+
| `--dir DIR` | `.` | Project root to scan |
|
|
99
|
+
| `--requirements FILE` | `./requirements.txt` | Path to requirements.txt |
|
|
100
|
+
|
|
101
|
+
---
|
|
102
|
+
|
|
103
|
+
## Interactive resolution example
|
|
104
|
+
|
|
105
|
+
```
|
|
106
|
+
Unrecognised module: 'yfinance'
|
|
107
|
+
Searching PyPI for 'yfinance'... 5 result(s).
|
|
108
|
+
Top PyPI matches:
|
|
109
|
+
[1] yfinance (similarity 1.00)
|
|
110
|
+
[2] yfinance-cache (similarity 0.78)
|
|
111
|
+
[3] yfinance-ez (similarity 0.72)
|
|
112
|
+
...
|
|
113
|
+
|
|
114
|
+
[1-5] select a match above
|
|
115
|
+
[a] auto-select top match for ALL remaining unknowns
|
|
116
|
+
[m] enter the correct dist name manually
|
|
117
|
+
[s] skip (omit from requirements, continue)
|
|
118
|
+
[q] quit (stop resolution now)
|
|
119
|
+
|
|
120
|
+
Your choice: 1
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
---
|
|
124
|
+
|
|
125
|
+
## Development
|
|
126
|
+
|
|
127
|
+
```bash
|
|
128
|
+
pip install -e ".[dev]"
|
|
129
|
+
pytest
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
---
|
|
133
|
+
|
|
134
|
+
## Publishing to PyPI
|
|
135
|
+
|
|
136
|
+
Publishing is handled automatically by GitHub Actions ([.github/workflows/publish.yml](.github/workflows/publish.yml)) whenever you push a version tag.
|
|
137
|
+
|
|
138
|
+
### One-time PyPI Trusted Publisher setup
|
|
139
|
+
|
|
140
|
+
1. Go to <https://pypi.org/manage/account/publishing/>
|
|
141
|
+
2. Add a **pending publisher** with:
|
|
142
|
+
- **PyPI project name**: `python-auto-req`
|
|
143
|
+
- **GitHub owner**: `Amit-Roy`
|
|
144
|
+
- **Repository name**: `python-auto-req`
|
|
145
|
+
- **Workflow filename**: `publish.yml`
|
|
146
|
+
- **Environment name**: `pypi`
|
|
147
|
+
3. On GitHub, create a repository **Environment** named `pypi`
|
|
148
|
+
*(Settings → Environments → New environment)*
|
|
149
|
+
|
|
150
|
+
### Releasing a new version
|
|
151
|
+
|
|
152
|
+
```bash
|
|
153
|
+
# 1. Bump the version in pyproject.toml (and src/auto_req/__init__.py)
|
|
154
|
+
# 2. Commit and push
|
|
155
|
+
git add pyproject.toml src/auto_req/__init__.py
|
|
156
|
+
git commit -m "chore: release v0.2.0"
|
|
157
|
+
git push
|
|
158
|
+
|
|
159
|
+
# 3. Tag and push – this triggers the publish workflow
|
|
160
|
+
git tag v0.2.0
|
|
161
|
+
git push --tags
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
The workflow will:
|
|
165
|
+
1. Run the full test matrix
|
|
166
|
+
2. Build wheel + sdist
|
|
167
|
+
3. Verify the tag matches the package version
|
|
168
|
+
4. Publish to PyPI via OIDC (no API token required)
|
|
169
|
+
|
|
170
|
+
### Local build (optional)
|
|
171
|
+
|
|
172
|
+
```bash
|
|
173
|
+
pip install -e ".[dev]"
|
|
174
|
+
python -m build
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
MIT
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
auto_req/__init__.py,sha256=9uUGtjOTIrWr1BWTEGSwlADe_OAUW96peL19_fokTTE,481
|
|
2
|
+
auto_req/__main__.py,sha256=QXj_L91dBemDWwJDm0tGwZdcYvMm8mCVjbHYTM2lhcI,94
|
|
3
|
+
auto_req/cli.py,sha256=Yfdp-ocpyu56ttdmdr87zXrmhHf3dnuYH4GT7qXhRNU,5050
|
|
4
|
+
auto_req/resolver.py,sha256=D_0cUrv1wabo1ApNjb1AyFvb3EaGdn0ZpAS8HQ5zIf4,7977
|
|
5
|
+
auto_req/scanner.py,sha256=jbvoDlXTN1bh8Aels58FWKLPNSVk4aeIZBk35h66wvI,4590
|
|
6
|
+
python_auto_req-0.1.0.dist-info/METADATA,sha256=p-cGOcGT0e2_XwkA1-wM9HhpgtG8KA7oi5IoAwIs_Z0,4769
|
|
7
|
+
python_auto_req-0.1.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
|
|
8
|
+
python_auto_req-0.1.0.dist-info/entry_points.txt,sha256=i1vr2iQDlMqhTVYuV8z40Uc1x4qLJHZ-ZTYIxkJSI8w,47
|
|
9
|
+
python_auto_req-0.1.0.dist-info/licenses/LICENSE,sha256=ESYyLizI0WWtxMeS7rGVcX3ivMezm-HOd5WdeOh-9oU,1056
|
|
10
|
+
python_auto_req-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|