pip-auto 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- autopip/__init__.py +730 -0
- autopip/__main__.py +268 -0
- autopip/build.py +215 -0
- autopip/fast.py +1709 -0
- autopip/gui.py +220 -0
- autopip/safety.py +130 -0
- autopip/scripts.py +91 -0
- autopip/smart.py +121 -0
- autopip/usage.py +188 -0
- autopip.pth +1 -0
- pip_auto-0.3.0.dist-info/METADATA +102 -0
- pip_auto-0.3.0.dist-info/RECORD +15 -0
- pip_auto-0.3.0.dist-info/WHEEL +4 -0
- pip_auto-0.3.0.dist-info/entry_points.txt +2 -0
- pip_auto-0.3.0.dist-info/licenses/LICENSE +21 -0
autopip/__init__.py
ADDED
|
@@ -0,0 +1,730 @@
|
|
|
1
|
+
"""autopip: import した瞬間に無いモジュールを PyPI から自動で入れる。
|
|
2
|
+
|
|
3
|
+
有効化すると sys.meta_path の最後にフックが入る。
|
|
4
|
+
普通の import が全部失敗したときだけ呼ばれるので、既にあるモジュールには一切触らない。
|
|
5
|
+
"""
|
|
6
|
+
import _thread
|
|
7
|
+
import importlib
|
|
8
|
+
import importlib.machinery
|
|
9
|
+
import os
|
|
10
|
+
import sys
|
|
11
|
+
import time
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class _Lazy:
|
|
15
|
+
"""使うまで import しない(.pth で毎回起動時に読まれるので、起動を重くしない)"""
|
|
16
|
+
|
|
17
|
+
def __init__(self, name):
|
|
18
|
+
self._name, self._mod = name, None
|
|
19
|
+
|
|
20
|
+
def __getattr__(self, a):
|
|
21
|
+
if self._mod is None:
|
|
22
|
+
self._mod = importlib.import_module(self._name)
|
|
23
|
+
return getattr(self._mod, a)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
ast = _Lazy("ast")
|
|
27
|
+
json = _Lazy("json")
|
|
28
|
+
subprocess = _Lazy("subprocess")
|
|
29
|
+
|
|
30
|
+
_HERE = os.path.dirname(os.path.abspath(__file__))
|
|
31
|
+
HOME = os.environ.get("AUTOPIP_HOME") or os.path.join(os.path.expanduser("~"), ".autopip")
|
|
32
|
+
|
|
33
|
+
# import名 と pip名 が違う有名どころ(これに無いものは下の対応表 → 名前の変種の順で探す)
|
|
34
|
+
NAME_MAP = {
|
|
35
|
+
"cv2": "opencv-python",
|
|
36
|
+
"PIL": "pillow",
|
|
37
|
+
"sklearn": "scikit-learn",
|
|
38
|
+
"skimage": "scikit-image",
|
|
39
|
+
"yaml": "pyyaml",
|
|
40
|
+
"bs4": "beautifulsoup4",
|
|
41
|
+
"dateutil": "python-dateutil",
|
|
42
|
+
"dotenv": "python-dotenv",
|
|
43
|
+
"Crypto": "pycryptodome",
|
|
44
|
+
"serial": "pyserial",
|
|
45
|
+
"usb": "pyusb",
|
|
46
|
+
"win32api": "pywin32",
|
|
47
|
+
"win32con": "pywin32",
|
|
48
|
+
"win32gui": "pywin32",
|
|
49
|
+
"pythoncom": "pywin32",
|
|
50
|
+
"wx": "wxPython",
|
|
51
|
+
"gi": "PyGObject",
|
|
52
|
+
"attr": "attrs",
|
|
53
|
+
"jwt": "PyJWT",
|
|
54
|
+
"magic": "python-magic",
|
|
55
|
+
"docx": "python-docx",
|
|
56
|
+
"pptx": "python-pptx",
|
|
57
|
+
"fitz": "PyMuPDF",
|
|
58
|
+
"telegram": "python-telegram-bot",
|
|
59
|
+
"discord": "discord.py",
|
|
60
|
+
"google.generativeai": "google-generativeai",
|
|
61
|
+
"google.genai": "google-genai",
|
|
62
|
+
"googleapiclient": "google-api-python-client",
|
|
63
|
+
"OpenGL": "PyOpenGL",
|
|
64
|
+
"Levenshtein": "python-Levenshtein",
|
|
65
|
+
"zmq": "pyzmq",
|
|
66
|
+
"MySQLdb": "mysqlclient",
|
|
67
|
+
"psycopg2": "psycopg2-binary",
|
|
68
|
+
"websocket": "websocket-client",
|
|
69
|
+
"pkg_resources": "setuptools",
|
|
70
|
+
}
|
|
71
|
+
_MAP_URL = "https://raw.githubusercontent.com/bndr/pipreqs/master/pipreqs/mapping"
|
|
72
|
+
|
|
73
|
+
_STDLIB = set(getattr(sys, "stdlib_module_names", ())) | set(sys.builtin_module_names)
|
|
74
|
+
# Python 自身が起動時などに「あれば読む」名前。PyPI に探しに行かない
|
|
75
|
+
_STDLIB |= {"sitecustomize", "usercustomize", "__main__", "__mp_main__", "_virtualenv", "_distutils_hack"}
|
|
76
|
+
_tried = set()
|
|
77
|
+
_enabled = False
|
|
78
|
+
_busy = _thread.allocate_lock()
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _log(msg):
|
|
82
|
+
if not os.environ.get("AUTOPIP_QUIET"):
|
|
83
|
+
print(f"[autopip] {msg}", file=sys.stderr, flush=True)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _cfg():
|
|
87
|
+
from .fast import load_config
|
|
88
|
+
return load_config()
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
# ---------------- 失敗の記録(次の実行でも覚えておく) ----------------
|
|
92
|
+
_FAILED = os.path.join(HOME, "failed.json")
|
|
93
|
+
RETRY_AFTER = 24 * 3600
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _failed_load():
|
|
97
|
+
try:
|
|
98
|
+
with open(_FAILED, encoding="utf-8") as f:
|
|
99
|
+
return json.load(f)
|
|
100
|
+
except Exception:
|
|
101
|
+
return {}
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _failed_save(d):
|
|
105
|
+
try:
|
|
106
|
+
os.makedirs(HOME, exist_ok=True)
|
|
107
|
+
with open(_FAILED, "w", encoding="utf-8") as f:
|
|
108
|
+
json.dump(d, f, ensure_ascii=False, indent=1)
|
|
109
|
+
except OSError:
|
|
110
|
+
pass
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def failed_recent(mod):
|
|
114
|
+
if os.environ.get("AUTOPIP_RETRY"):
|
|
115
|
+
return False
|
|
116
|
+
ts = _failed_load().get(mod)
|
|
117
|
+
return ts is not None and time.time() - ts < RETRY_AFTER
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def failed_add(mod):
|
|
121
|
+
d = _failed_load()
|
|
122
|
+
d[mod] = time.time()
|
|
123
|
+
_failed_save(d)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def failed_clear(mod=None):
|
|
127
|
+
d = {} if mod is None else {k: v for k, v in _failed_load().items() if k != mod}
|
|
128
|
+
_failed_save(d)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
# ---------------- import名 → パッケージ名 ----------------
|
|
132
|
+
_mapping_cache = None
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _mapping():
|
|
136
|
+
"""pipreqs の import名:パッケージ名 対応表(約 1000 件)。30 日キャッシュ。"""
|
|
137
|
+
global _mapping_cache
|
|
138
|
+
if _mapping_cache is not None:
|
|
139
|
+
return _mapping_cache
|
|
140
|
+
p = os.path.join(HOME, "mapping.txt")
|
|
141
|
+
if not os.path.exists(p) or time.time() - os.path.getmtime(p) > 30 * 86400:
|
|
142
|
+
try:
|
|
143
|
+
import urllib.request
|
|
144
|
+
data = urllib.request.urlopen(_MAP_URL, timeout=5).read()
|
|
145
|
+
os.makedirs(HOME, exist_ok=True)
|
|
146
|
+
with open(p, "wb") as f:
|
|
147
|
+
f.write(data)
|
|
148
|
+
except Exception:
|
|
149
|
+
pass
|
|
150
|
+
out = {}
|
|
151
|
+
try:
|
|
152
|
+
with open(p, encoding="utf-8") as f:
|
|
153
|
+
for line in f:
|
|
154
|
+
a, _, b = line.strip().partition(":")
|
|
155
|
+
if a and b:
|
|
156
|
+
out.setdefault(a, b)
|
|
157
|
+
except OSError:
|
|
158
|
+
pass
|
|
159
|
+
_mapping_cache = out
|
|
160
|
+
return out
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _prefixes(n):
|
|
164
|
+
parts = n.split(".")
|
|
165
|
+
return [".".join(parts[:i]) for i in range(len(parts), 0, -1)]
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def candidates(fullname, stmt=()):
|
|
169
|
+
"""[(パッケージ名, そのパッケージが含むべきモジュール)] を優先順で。
|
|
170
|
+
stmt: import 文から拾った [(名前, from由来か)]"""
|
|
171
|
+
from .fast import norm, modindex
|
|
172
|
+
rel = lambda n: n == fullname or n.startswith(fullname + ".")
|
|
173
|
+
imp = sorted({n for n, frm in stmt if not frm and rel(n)} | {fullname}, key=len, reverse=True)
|
|
174
|
+
frm = sorted({n for n, f in stmt if f and rel(n)} - set(imp), key=len, reverse=True)
|
|
175
|
+
mp, learned = _mapping(), modindex()
|
|
176
|
+
|
|
177
|
+
def known(p):
|
|
178
|
+
return [m for m in (NAME_MAP.get(p), *learned.get(p, ()), mp.get(p)) if m]
|
|
179
|
+
|
|
180
|
+
out = []
|
|
181
|
+
for n in imp: # 1. import 文の名前: 長い方から「対応表・学習済み → a.b→a-b」
|
|
182
|
+
for p in _prefixes(n):
|
|
183
|
+
out += [(m, p) for m in known(p)]
|
|
184
|
+
if "." in p:
|
|
185
|
+
out.append((p.replace(".", "-"), p))
|
|
186
|
+
top = fullname.split(".")[0] # 2. 名前そのもの + よくある変種
|
|
187
|
+
for v in (top, top.replace("_", "-"), f"python-{top}", f"py{top}", f"{top}-python", f"py-{top}", f"{top}-py"):
|
|
188
|
+
out.append((v, top))
|
|
189
|
+
for n in frm: # 3. from a import b 由来(b は関数かもしれないので後回し)
|
|
190
|
+
for p in _prefixes(n):
|
|
191
|
+
out += [(m, p) for m in known(p)]
|
|
192
|
+
if "." in p:
|
|
193
|
+
out.append((p.replace(".", "-"), p))
|
|
194
|
+
seen, res = set(), []
|
|
195
|
+
for pkg, mod in out:
|
|
196
|
+
if (mod == fullname or mod.startswith(fullname + ".")) and norm(pkg) not in seen:
|
|
197
|
+
seen.add(norm(pkg))
|
|
198
|
+
res.append((pkg, mod))
|
|
199
|
+
return res
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def find_package(fullname, stmt=(), log=_log):
|
|
203
|
+
"""中身まで確かめて、fullname を本当に提供するパッケージを返す。(pkg, mod) / None
|
|
204
|
+
候補の存在確認も中身の確認も全部並列で投げ、優先順で最初に当たったものを採る"""
|
|
205
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
206
|
+
from . import fast
|
|
207
|
+
cands = candidates(fullname, stmt)
|
|
208
|
+
quiet = lambda m: None
|
|
209
|
+
with ThreadPoolExecutor(16) as ex:
|
|
210
|
+
exists = list(ex.map(lambda c: bool(fast._quiet(fast.project, c[0])), cands))
|
|
211
|
+
live = [c for c, e in zip(cands, exists) if e]
|
|
212
|
+
futs = [ex.submit(fast.check_provides, pkg, mod, quiet) for pkg, mod in live]
|
|
213
|
+
for (pkg, mod), f in zip(live, futs):
|
|
214
|
+
r = f.result()
|
|
215
|
+
if r:
|
|
216
|
+
for g in futs:
|
|
217
|
+
g.cancel()
|
|
218
|
+
return pkg, mod
|
|
219
|
+
if r is False:
|
|
220
|
+
log(f"PyPI の {pkg} は '{mod}' を含まない(別物)→ 次を探す")
|
|
221
|
+
return None
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
# ---------------- 誰が import したか ----------------
|
|
225
|
+
_trees = {}
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def _tree(fn):
|
|
229
|
+
if fn not in _trees:
|
|
230
|
+
try:
|
|
231
|
+
with open(fn, encoding="utf-8", errors="replace") as f:
|
|
232
|
+
_trees[fn] = ast.parse(f.read(), fn)
|
|
233
|
+
except Exception:
|
|
234
|
+
_trees[fn] = None
|
|
235
|
+
return _trees[fn]
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
_IMPORT_ERRORS = {"ImportError", "ModuleNotFoundError", "Exception", "BaseException"}
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _catches_import_error(h):
|
|
242
|
+
if h.type is None:
|
|
243
|
+
return True
|
|
244
|
+
ts = h.type.elts if isinstance(h.type, ast.Tuple) else [h.type]
|
|
245
|
+
for t in ts:
|
|
246
|
+
name = t.id if isinstance(t, ast.Name) else t.attr if isinstance(t, ast.Attribute) else None
|
|
247
|
+
if name in _IMPORT_ERRORS:
|
|
248
|
+
return True
|
|
249
|
+
return False
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def stmt_info(fn, line):
|
|
253
|
+
"""(try: で ImportError を拾っているか, [(import名, from由来か)])"""
|
|
254
|
+
tree = _tree(fn)
|
|
255
|
+
if tree is None:
|
|
256
|
+
return False, []
|
|
257
|
+
guarded, names = False, []
|
|
258
|
+
tries = tuple(t for t in (getattr(ast, "Try", None), getattr(ast, "TryStar", None)) if t)
|
|
259
|
+
for node in ast.walk(tree):
|
|
260
|
+
if isinstance(node, (ast.Import, ast.ImportFrom)) and node.lineno <= line <= (node.end_lineno or node.lineno):
|
|
261
|
+
if isinstance(node, ast.Import):
|
|
262
|
+
names += [(a.name, False) for a in node.names]
|
|
263
|
+
elif node.module and not node.level:
|
|
264
|
+
names.append((node.module, False))
|
|
265
|
+
names += [(f"{node.module}.{a.name}", True) for a in node.names if a.name != "*"]
|
|
266
|
+
elif isinstance(node, tries) and node.body:
|
|
267
|
+
if node.body[0].lineno <= line <= (node.body[-1].end_lineno or node.body[-1].lineno):
|
|
268
|
+
if any(_catches_import_error(h) for h in node.handlers):
|
|
269
|
+
guarded = True
|
|
270
|
+
return guarded, names
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
_PROBES = {"find_spec", "find_loader", "get_loader", "iter_modules", "_find_spec"}
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _origin():
|
|
277
|
+
"""import を書いた場所のフレームと、find_spec などでの「有無チェック」かどうか"""
|
|
278
|
+
f = sys._getframe(2)
|
|
279
|
+
probe = False
|
|
280
|
+
while f:
|
|
281
|
+
fn = f.f_code.co_filename
|
|
282
|
+
low = fn.replace("\\", "/")
|
|
283
|
+
if low.startswith("<frozen"):
|
|
284
|
+
f = f.f_back
|
|
285
|
+
continue
|
|
286
|
+
if not fn.startswith("<") and os.path.dirname(os.path.abspath(fn)) == _HERE:
|
|
287
|
+
return None, True # autopip 自身の import(packaging があれば使う等)には反応しない
|
|
288
|
+
if "/importlib/" in low or low.endswith("/pkgutil.py"):
|
|
289
|
+
if f.f_code.co_name in _PROBES and low.endswith(("/importlib/util.py", "/pkgutil.py")):
|
|
290
|
+
probe = True
|
|
291
|
+
f = f.f_back
|
|
292
|
+
continue
|
|
293
|
+
return f, probe
|
|
294
|
+
return None, probe
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def _kind(fn):
|
|
298
|
+
if fn.startswith("<"):
|
|
299
|
+
return "user"
|
|
300
|
+
low = os.path.abspath(fn).replace("\\", "/").lower()
|
|
301
|
+
if "/site-packages/" in low or "/dist-packages/" in low:
|
|
302
|
+
return "lib"
|
|
303
|
+
for base in {sys.base_prefix, sys.prefix}:
|
|
304
|
+
b = os.path.abspath(base).replace("\\", "/").lower()
|
|
305
|
+
if low.startswith(b + "/lib/"):
|
|
306
|
+
return "stdlib"
|
|
307
|
+
# そのファイルを含む一番深い sys.path の場所で決める:
|
|
308
|
+
# スクリプトのフォルダ(sys.path[0])やカレントなら自分のコード、それ以外(site-packages・PYTHONPATH)はライブラリ
|
|
309
|
+
best, best_is_user = "", True
|
|
310
|
+
user_dirs = {os.path.abspath(sys.path[0] or os.getcwd()).replace("\\", "/").lower(),
|
|
311
|
+
os.getcwd().replace("\\", "/").lower()} if sys.path else set()
|
|
312
|
+
for d in sys.path:
|
|
313
|
+
if not d:
|
|
314
|
+
continue
|
|
315
|
+
dd = os.path.abspath(d).replace("\\", "/").lower().rstrip("/")
|
|
316
|
+
if low.startswith(dd + "/") and len(dd) > len(best):
|
|
317
|
+
best, best_is_user = dd, dd in user_dirs
|
|
318
|
+
return "user" if best_is_user else "lib"
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
# ---------------- import ロックを外して作業する ----------------
|
|
322
|
+
def _without_import_lock(fn):
|
|
323
|
+
"""import 中はグローバル import ロックを握っているので、一時的に外してスレッドを使えるようにする"""
|
|
324
|
+
import _imp
|
|
325
|
+
held = False
|
|
326
|
+
try:
|
|
327
|
+
_imp.release_lock()
|
|
328
|
+
held = True
|
|
329
|
+
except RuntimeError:
|
|
330
|
+
pass
|
|
331
|
+
try:
|
|
332
|
+
return fn()
|
|
333
|
+
finally:
|
|
334
|
+
if held:
|
|
335
|
+
_imp.acquire_lock()
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
# ---------------- 確認 ----------------
|
|
339
|
+
def _ask(fullname, pkg):
|
|
340
|
+
ask = os.environ.get("AUTOPIP_ASK") or _cfg().get("ask")
|
|
341
|
+
if not ask or str(ask).lower() in ("0", "false", "no"):
|
|
342
|
+
return True
|
|
343
|
+
line = console_input(f"[autopip] '{fullname}' のために {pkg} を入れる? [Y/n] ")
|
|
344
|
+
if line is None:
|
|
345
|
+
_log(f"確認モードだけど聞ける画面が無い → {pkg} は入れない")
|
|
346
|
+
return False
|
|
347
|
+
if not line: # 何も入力されずに終わった → いいえ
|
|
348
|
+
return False
|
|
349
|
+
return line.strip().lower() in ("", "y", "yes", "はい")
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def console_input(prompt):
|
|
353
|
+
"""標準入力ではなく端末(Windows: CONIN$ / それ以外: /dev/tty)から直接 1 行読む。
|
|
354
|
+
標準入力がパイプやファイルでも聞ける。端末が無ければ None"""
|
|
355
|
+
timeout = float(os.environ.get("AUTOPIP_ASK_TIMEOUT", "60"))
|
|
356
|
+
fin = fout = None
|
|
357
|
+
try:
|
|
358
|
+
if os.name == "nt":
|
|
359
|
+
import ctypes
|
|
360
|
+
import msvcrt
|
|
361
|
+
k32, u32 = ctypes.windll.kernel32, ctypes.windll.user32
|
|
362
|
+
hwnd = k32.GetConsoleWindow()
|
|
363
|
+
if not hwnd or not u32.IsWindowVisible(hwnd):
|
|
364
|
+
return None # コンソールが無い / 見えない(誰も答えられない)
|
|
365
|
+
fout = open("CONOUT$", "w", errors="replace")
|
|
366
|
+
fout.write(prompt)
|
|
367
|
+
fout.flush()
|
|
368
|
+
buf, end = [], time.time() + timeout
|
|
369
|
+
while time.time() < end:
|
|
370
|
+
if not msvcrt.kbhit():
|
|
371
|
+
time.sleep(0.05)
|
|
372
|
+
continue
|
|
373
|
+
ch = msvcrt.getwch()
|
|
374
|
+
if ch in ("\r", "\n"):
|
|
375
|
+
fout.write("\n")
|
|
376
|
+
return "".join(buf) + "\n"
|
|
377
|
+
if ch == "\x03":
|
|
378
|
+
raise KeyboardInterrupt
|
|
379
|
+
if ch == "\b":
|
|
380
|
+
if buf:
|
|
381
|
+
buf.pop()
|
|
382
|
+
fout.write("\b \b")
|
|
383
|
+
else:
|
|
384
|
+
buf.append(ch)
|
|
385
|
+
fout.write(ch)
|
|
386
|
+
fout.flush()
|
|
387
|
+
fout.write("\n(時間切れ)\n")
|
|
388
|
+
return ""
|
|
389
|
+
import select
|
|
390
|
+
fin = open("/dev/tty", "r")
|
|
391
|
+
fout = open("/dev/tty", "w")
|
|
392
|
+
fout.write(prompt)
|
|
393
|
+
fout.flush()
|
|
394
|
+
if not select.select([fin], [], [], timeout)[0]:
|
|
395
|
+
fout.write("\n(時間切れ)\n")
|
|
396
|
+
return ""
|
|
397
|
+
line = fin.readline()
|
|
398
|
+
if not line:
|
|
399
|
+
fout.write("\n")
|
|
400
|
+
return line
|
|
401
|
+
except OSError:
|
|
402
|
+
return None
|
|
403
|
+
finally:
|
|
404
|
+
for f in (fin, fout):
|
|
405
|
+
if f:
|
|
406
|
+
try:
|
|
407
|
+
f.close()
|
|
408
|
+
except OSError:
|
|
409
|
+
pass
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
# ---------------- 本体 ----------------
|
|
413
|
+
def _safe_to_install(pkg, fullname, log):
|
|
414
|
+
"""安全チェック。strict で怪しければ入れない"""
|
|
415
|
+
from . import safety
|
|
416
|
+
ok, reasons = safety.check(pkg, fullname)
|
|
417
|
+
for r in reasons:
|
|
418
|
+
log(f"注意: {pkg}: {r}")
|
|
419
|
+
if not ok:
|
|
420
|
+
log(f"安全のため {pkg} は自動では入れない。本当に要るなら: python -m autopip install {pkg}"
|
|
421
|
+
f"(または python -m autopip config safety warn)")
|
|
422
|
+
return ok
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def _install_reqs(reqs, why, log):
|
|
426
|
+
"""要求を入れる。自前が失敗したら pip。入ったもののリスト / None"""
|
|
427
|
+
from . import fast, usage
|
|
428
|
+
try:
|
|
429
|
+
got = fast.install(reqs, log=log)
|
|
430
|
+
except Exception as e:
|
|
431
|
+
log(f"高速インストール失敗 ({e}) → pip で試す")
|
|
432
|
+
got = None
|
|
433
|
+
for r in reqs:
|
|
434
|
+
g = _pip_install(r, log)
|
|
435
|
+
if g is None:
|
|
436
|
+
return None
|
|
437
|
+
got = (got or []) + g
|
|
438
|
+
if got:
|
|
439
|
+
usage.note_installed(got, why)
|
|
440
|
+
return got
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
_pep723_done = set()
|
|
444
|
+
|
|
445
|
+
|
|
446
|
+
def _ensure_pep723(log):
|
|
447
|
+
"""__main__ のスクリプトに PEP 723 の依存が書いてあれば、最初の 1 回でまとめて入れる"""
|
|
448
|
+
main = getattr(sys.modules.get("__main__"), "__file__", None)
|
|
449
|
+
if not main or main in _pep723_done:
|
|
450
|
+
return False
|
|
451
|
+
_pep723_done.add(main)
|
|
452
|
+
from . import smart
|
|
453
|
+
try:
|
|
454
|
+
meta = smart.pep723(main)
|
|
455
|
+
except Exception as e:
|
|
456
|
+
log(f"PEP 723 ブロックが読めない: {e}")
|
|
457
|
+
return False
|
|
458
|
+
if not meta or not meta["dependencies"]:
|
|
459
|
+
return False
|
|
460
|
+
log(f"スクリプトに書かれた依存をまとめて入れる: {', '.join(meta['dependencies'])}")
|
|
461
|
+
return _install_reqs(meta["dependencies"], f"PEP 723 ({os.path.basename(main)})", log) is not None
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
def ensure(fullname, stmt=(), log=_log, src=None):
|
|
465
|
+
"""fullname を import できるようにする。できたら True
|
|
466
|
+
src: (import を書いたファイル, 行) ─ 行末の `# autopip: 要求` を読むのに使う"""
|
|
467
|
+
from . import fast, smart
|
|
468
|
+
t0 = time.perf_counter()
|
|
469
|
+
top = fullname.split(".")[0]
|
|
470
|
+
|
|
471
|
+
# 1. スクリプトの PEP 723 ブロック
|
|
472
|
+
if _ensure_pep723(log):
|
|
473
|
+
importlib.invalidate_caches()
|
|
474
|
+
if importlib.machinery.PathFinder.find_spec(top) is not None:
|
|
475
|
+
return True
|
|
476
|
+
|
|
477
|
+
# 2. 行末コメント `# autopip: numpy<2` があればその通りに
|
|
478
|
+
req = smart.comment_req_for(*src) if src else None
|
|
479
|
+
if req:
|
|
480
|
+
if not _ask(fullname, req):
|
|
481
|
+
return False
|
|
482
|
+
log(f"'{fullname}' が無い → コメントの指定どおり {req} を入れる")
|
|
483
|
+
got = _install_reqs([req], f"コメント指定 {req}", log)
|
|
484
|
+
importlib.invalidate_caches()
|
|
485
|
+
if got is not None and importlib.machinery.PathFinder.find_spec(top) is not None:
|
|
486
|
+
log(f"{', '.join(got) or req} 入った ({time.perf_counter() - t0:.2f}s)")
|
|
487
|
+
return True
|
|
488
|
+
log(f"{req} を入れても '{fullname}' が使えなかった")
|
|
489
|
+
if got:
|
|
490
|
+
fast.uninstall(got)
|
|
491
|
+
return False
|
|
492
|
+
|
|
493
|
+
# 3. 名前から探す
|
|
494
|
+
if failed_recent(fullname):
|
|
495
|
+
log(f"'{fullname}' は最近失敗したので飛ばす(AUTOPIP_RETRY=1 か python -m autopip forget で再挑戦)")
|
|
496
|
+
return False
|
|
497
|
+
try:
|
|
498
|
+
found = find_package(fullname, stmt, log)
|
|
499
|
+
except Exception as e:
|
|
500
|
+
log(f"探せなかった: {e}")
|
|
501
|
+
return False
|
|
502
|
+
if not found:
|
|
503
|
+
log(f"'{fullname}' を提供するパッケージが PyPI に見つからない")
|
|
504
|
+
failed_add(fullname)
|
|
505
|
+
return False
|
|
506
|
+
pkg, mod = found
|
|
507
|
+
if not _safe_to_install(pkg, fullname, log):
|
|
508
|
+
failed_add(fullname)
|
|
509
|
+
return False
|
|
510
|
+
if not _ask(fullname, pkg):
|
|
511
|
+
return False
|
|
512
|
+
log(f"'{fullname}' が無い → {pkg} を入れる")
|
|
513
|
+
got = _install_reqs([pkg], f"import {fullname}", log)
|
|
514
|
+
if got is None:
|
|
515
|
+
failed_add(fullname)
|
|
516
|
+
return False
|
|
517
|
+
importlib.invalidate_caches()
|
|
518
|
+
if importlib.machinery.PathFinder.find_spec(mod.split(".")[0]) is None:
|
|
519
|
+
log(f"'{mod}' が使えなかった → 今回入ったものを全部消す: {', '.join(got)}")
|
|
520
|
+
fast.uninstall(got)
|
|
521
|
+
failed_add(fullname)
|
|
522
|
+
return False
|
|
523
|
+
failed_clear(fullname)
|
|
524
|
+
log(f"{', '.join(got) or pkg} 入った ({time.perf_counter() - t0:.2f}s)")
|
|
525
|
+
return True
|
|
526
|
+
|
|
527
|
+
|
|
528
|
+
def _pip_install(pkg, log):
|
|
529
|
+
"""最後の手段。前後の差分で入ったものを返す(失敗したら差分を全部消して None)"""
|
|
530
|
+
from . import fast
|
|
531
|
+
t = fast.Target.current()
|
|
532
|
+
before = set(t.dists())
|
|
533
|
+
env = dict(os.environ, AUTOPIP_DISABLE="1")
|
|
534
|
+
r = subprocess.run([sys.executable, "-m", "pip", "install", "-q", "--disable-pip-version-check", pkg], env=env)
|
|
535
|
+
importlib.invalidate_caches()
|
|
536
|
+
added = sorted(set(t.dists()) - before)
|
|
537
|
+
if r.returncode != 0:
|
|
538
|
+
if added:
|
|
539
|
+
subprocess.run([sys.executable, "-m", "pip", "uninstall", "-y", "-q", *added], env=env)
|
|
540
|
+
return None
|
|
541
|
+
return added
|
|
542
|
+
|
|
543
|
+
|
|
544
|
+
class AutoPipFinder:
|
|
545
|
+
def find_spec(self, fullname, path=None, target=None):
|
|
546
|
+
if os.environ.get("AUTOPIP_DISABLE") or _busy.locked():
|
|
547
|
+
return None
|
|
548
|
+
top = fullname.split(".")[0]
|
|
549
|
+
if top in _STDLIB or top.startswith("_") or fullname in _tried:
|
|
550
|
+
return None
|
|
551
|
+
# 自分より後ろに登録された finder (six.moves など) に先に聞く
|
|
552
|
+
mp = sys.meta_path
|
|
553
|
+
if self in mp:
|
|
554
|
+
for f in mp[mp.index(self) + 1:]:
|
|
555
|
+
fs = getattr(f, "find_spec", None)
|
|
556
|
+
spec = fs(fullname, path, target) if fs else None
|
|
557
|
+
if spec is not None:
|
|
558
|
+
return spec
|
|
559
|
+
if "." in fullname and sys.modules.get(fullname.rpartition(".")[0]) is None:
|
|
560
|
+
return None
|
|
561
|
+
frame, probe = _origin()
|
|
562
|
+
if probe:
|
|
563
|
+
return None # importlib.util.find_spec("x") みたいな「あるかチェック」
|
|
564
|
+
kind = _kind(frame.f_code.co_filename) if frame else "user"
|
|
565
|
+
if kind == "stdlib":
|
|
566
|
+
return None
|
|
567
|
+
fn, line = (frame.f_code.co_filename, frame.f_lineno) if frame else (None, 0)
|
|
568
|
+
guarded, stmt = stmt_info(fn, line) if frame else (False, [])
|
|
569
|
+
if kind == "lib" and guarded:
|
|
570
|
+
return None # ライブラリの try: import optional は入れない
|
|
571
|
+
_tried.add(fullname)
|
|
572
|
+
with _busy:
|
|
573
|
+
ok = _without_import_lock(lambda: ensure(fullname, stmt, src=(fn, line) if kind == "user" else None))
|
|
574
|
+
if not ok:
|
|
575
|
+
return None
|
|
576
|
+
importlib.invalidate_caches()
|
|
577
|
+
return importlib.machinery.PathFinder.find_spec(fullname, path)
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
def enable():
|
|
581
|
+
global _enabled
|
|
582
|
+
if _enabled or os.environ.get("AUTOPIP_DISABLE"):
|
|
583
|
+
return
|
|
584
|
+
sys.meta_path.append(AutoPipFinder())
|
|
585
|
+
_enabled = True
|
|
586
|
+
if not os.environ.get("AUTOPIP_NO_USAGE"):
|
|
587
|
+
import atexit
|
|
588
|
+
global _startup_modules
|
|
589
|
+
_startup_modules = set(sys.modules) # 起動時に .pth などで読まれただけのものは「使った」に数えない
|
|
590
|
+
atexit.register(_record_usage)
|
|
591
|
+
|
|
592
|
+
|
|
593
|
+
_startup_modules = set()
|
|
594
|
+
|
|
595
|
+
|
|
596
|
+
def usage_file(prefix=None):
|
|
597
|
+
key = "".join(c if c.isalnum() else "_" for c in os.path.normcase(os.path.abspath(prefix or sys.prefix)))
|
|
598
|
+
return os.path.join(HOME, "usage", key[-120:] + ".txt")
|
|
599
|
+
|
|
600
|
+
|
|
601
|
+
def read_usage(prefix=None):
|
|
602
|
+
"""{トップ名: 最後に使った日 (epoch 日数)}"""
|
|
603
|
+
out = {}
|
|
604
|
+
try:
|
|
605
|
+
with open(usage_file(prefix), encoding="utf-8") as f:
|
|
606
|
+
for line in f:
|
|
607
|
+
name, _, day = line.strip().partition(" ")
|
|
608
|
+
if day.isdigit():
|
|
609
|
+
out[name] = int(day)
|
|
610
|
+
except OSError:
|
|
611
|
+
pass
|
|
612
|
+
return out
|
|
613
|
+
|
|
614
|
+
|
|
615
|
+
def write_usage(tops):
|
|
616
|
+
"""tops を今日使ったことにする。もう今日の記録があれば何もしない(json も hashlib も使わない)"""
|
|
617
|
+
today = int(time.time() // 86400)
|
|
618
|
+
d = read_usage()
|
|
619
|
+
if all(d.get(t) == today for t in tops):
|
|
620
|
+
return
|
|
621
|
+
for t in tops:
|
|
622
|
+
d[t] = today
|
|
623
|
+
p = usage_file()
|
|
624
|
+
try:
|
|
625
|
+
os.makedirs(os.path.dirname(p), exist_ok=True)
|
|
626
|
+
tmp = f"{p}.{os.getpid()}"
|
|
627
|
+
with open(tmp, "w", encoding="utf-8") as f:
|
|
628
|
+
f.write("".join(f"{k} {v}\n" for k, v in sorted(d.items())))
|
|
629
|
+
os.replace(tmp, p)
|
|
630
|
+
except OSError:
|
|
631
|
+
pass
|
|
632
|
+
|
|
633
|
+
|
|
634
|
+
def site_tops(only=None):
|
|
635
|
+
"""site-packages から読み込まれたトップレベルのモジュール名"""
|
|
636
|
+
st = sys.modules.get("site") # 起動時に必ず読まれている(sysconfig は読まれていないことがある)
|
|
637
|
+
if st is None:
|
|
638
|
+
return []
|
|
639
|
+
dirs = list(getattr(st, "getsitepackages", lambda: [])())
|
|
640
|
+
try:
|
|
641
|
+
dirs.append(st.getusersitepackages())
|
|
642
|
+
except Exception:
|
|
643
|
+
pass
|
|
644
|
+
roots = tuple({os.path.normcase(os.path.abspath(d)) + os.sep for d in dirs})
|
|
645
|
+
out = []
|
|
646
|
+
for name, m in list(sys.modules.items()):
|
|
647
|
+
if "." in name or m is None or name in _STDLIB or (only is not None and name not in only):
|
|
648
|
+
continue
|
|
649
|
+
f = getattr(m, "__file__", None)
|
|
650
|
+
if f and os.path.normcase(os.path.abspath(f)).startswith(roots):
|
|
651
|
+
out.append(name)
|
|
652
|
+
return out
|
|
653
|
+
|
|
654
|
+
|
|
655
|
+
def _record_usage():
|
|
656
|
+
if "pip._internal" in sys.modules or os.environ.get("AUTOPIP_NO_USAGE"):
|
|
657
|
+
return # pip の実行中は触らない
|
|
658
|
+
new = {n for n in sys.modules if "." not in n and n not in _startup_modules}
|
|
659
|
+
if not new:
|
|
660
|
+
return # 何も使ってなければ即終了
|
|
661
|
+
tops = site_tops(new)
|
|
662
|
+
if tops:
|
|
663
|
+
write_usage(tops)
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
def scan(path):
|
|
667
|
+
"""ファイルを実行せずに AST から import を全部拾う。無いものだけ返す [(名前, stmt, 行)]"""
|
|
668
|
+
with open(path, encoding="utf-8") as fp:
|
|
669
|
+
tree = ast.parse(fp.read(), path)
|
|
670
|
+
found, lines = {}, {}
|
|
671
|
+
for node in ast.walk(tree):
|
|
672
|
+
if isinstance(node, ast.Import):
|
|
673
|
+
for a in node.names:
|
|
674
|
+
t = a.name.split(".")[0]
|
|
675
|
+
found.setdefault(t, []).append((a.name, False))
|
|
676
|
+
lines.setdefault(t, node.lineno)
|
|
677
|
+
elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module:
|
|
678
|
+
t = node.module.split(".")[0]
|
|
679
|
+
found.setdefault(t, []).extend(
|
|
680
|
+
[(node.module, False)] + [(f"{node.module}.{a.name}", True) for a in node.names if a.name != "*"])
|
|
681
|
+
lines.setdefault(t, node.lineno)
|
|
682
|
+
here = os.path.dirname(os.path.abspath(path))
|
|
683
|
+
missing = []
|
|
684
|
+
for top in sorted(found):
|
|
685
|
+
if top in _STDLIB:
|
|
686
|
+
continue
|
|
687
|
+
if os.path.exists(os.path.join(here, top + ".py")) or os.path.isdir(os.path.join(here, top)):
|
|
688
|
+
continue
|
|
689
|
+
try:
|
|
690
|
+
if __import__("importlib.util").util.find_spec(top) is not None:
|
|
691
|
+
continue
|
|
692
|
+
except (ImportError, ValueError):
|
|
693
|
+
pass
|
|
694
|
+
missing.append((top, found[top], lines[top]))
|
|
695
|
+
return missing
|
|
696
|
+
|
|
697
|
+
|
|
698
|
+
def plan_file(path, log=_log):
|
|
699
|
+
"""ファイルが必要とする要求の一覧: PEP 723 > 行末コメント > 名前から探す"""
|
|
700
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
701
|
+
from . import smart
|
|
702
|
+
reqs = []
|
|
703
|
+
meta = smart.pep723(path)
|
|
704
|
+
if meta and meta["dependencies"]:
|
|
705
|
+
reqs += meta["dependencies"]
|
|
706
|
+
comments = smart.comment_reqs(path)
|
|
707
|
+
miss = scan(path)
|
|
708
|
+
todo = []
|
|
709
|
+
for name, stmt, line in miss:
|
|
710
|
+
if line in comments:
|
|
711
|
+
reqs.append(comments[line])
|
|
712
|
+
elif not meta or not meta["dependencies"]:
|
|
713
|
+
todo.append((name, stmt))
|
|
714
|
+
with ThreadPoolExecutor(8) as ex:
|
|
715
|
+
found = list(ex.map(lambda m: find_package(m[0], m[1], log), todo))
|
|
716
|
+
for (name, _), f in zip(todo, found):
|
|
717
|
+
if f is None:
|
|
718
|
+
log(f"'{name}' を提供するパッケージが見つからない")
|
|
719
|
+
elif _safe_to_install(f[0], name, log) and _ask(name, f[0]):
|
|
720
|
+
reqs.append(f[0])
|
|
721
|
+
return list(dict.fromkeys(reqs))
|
|
722
|
+
|
|
723
|
+
|
|
724
|
+
def ensure_file(path, log=_log):
|
|
725
|
+
"""ファイルが使うのに無いものを、全部まとめて 1 回で入れる"""
|
|
726
|
+
reqs = plan_file(path, log)
|
|
727
|
+
if not reqs:
|
|
728
|
+
return []
|
|
729
|
+
log(f"まとめて入れる: {', '.join(reqs)}")
|
|
730
|
+
return _install_reqs(reqs, f"run {os.path.basename(path)}", log) or []
|