pykitool 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pykitool/__init__.py +0 -0
- pykitool/base/__init__.py +0 -0
- pykitool/base/cache.py +352 -0
- pykitool/base/enums.py +102 -0
- pykitool/base/exception.py +6 -0
- pykitool/base/response.py +82 -0
- pykitool/base/tlog.py +230 -0
- pykitool/sqliter/__init__.py +0 -0
- pykitool/sqliter/exception.py +30 -0
- pykitool/sqliter/middleware.py +84 -0
- pykitool/sqliter/plus.py +125 -0
- pykitool/sqliter/repo.py +188 -0
- pykitool/utils/__init__.py +0 -0
- pykitool/utils/cbfile.py +697 -0
- pykitool/utils/cbrequest.py +473 -0
- pykitool/utils/cbruntime.py +870 -0
- pykitool/utils/cbutils.py +518 -0
- pykitool-0.0.1.dist-info/METADATA +36 -0
- pykitool-0.0.1.dist-info/RECORD +21 -0
- pykitool-0.0.1.dist-info/WHEEL +5 -0
- pykitool-0.0.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,870 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import concurrent.futures
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import platform
|
|
6
|
+
import random
|
|
7
|
+
import re
|
|
8
|
+
import shutil
|
|
9
|
+
import signal
|
|
10
|
+
import socket
|
|
11
|
+
import subprocess
|
|
12
|
+
import sys
|
|
13
|
+
import tempfile
|
|
14
|
+
import threading
|
|
15
|
+
import time
|
|
16
|
+
from collections import deque
|
|
17
|
+
from importlib import metadata
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Any, Callable, Coroutine, Dict, List, Optional, TypeVar, Union
|
|
20
|
+
|
|
21
|
+
from loguru import logger
|
|
22
|
+
from tqdm import tqdm
|
|
23
|
+
|
|
24
|
+
# 延迟初始化标志,避免模块导入时执行 subprocess
|
|
25
|
+
_ensurepip_initialized = False
|
|
26
|
+
|
|
27
|
+
# 泛型类型变量
|
|
28
|
+
T = TypeVar("T")
|
|
29
|
+
|
|
30
|
+
# 全局事件循环和执行器(延迟初始化)
|
|
31
|
+
_loop = None
|
|
32
|
+
_executor = None
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _get_loop():
|
|
36
|
+
global _loop
|
|
37
|
+
try:
|
|
38
|
+
_loop = asyncio.get_running_loop()
|
|
39
|
+
except RuntimeError:
|
|
40
|
+
if _loop is None:
|
|
41
|
+
_loop = asyncio.new_event_loop()
|
|
42
|
+
asyncio.set_event_loop(_loop)
|
|
43
|
+
return _loop
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _get_executor():
|
|
47
|
+
global _executor
|
|
48
|
+
if _executor is None or _executor._shutdown:
|
|
49
|
+
_executor = concurrent.futures.ThreadPoolExecutor(max_workers=10)
|
|
50
|
+
return _executor
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
# ====================================================== package ======================================================
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
# 将值格式化为指定长度的字符串,按指定字符填充
|
|
57
|
+
def pad_string(value: Any, length: int = 10, align: str = "right", char: str = " ") -> str:
|
|
58
|
+
str_value = str(value)
|
|
59
|
+
pad_len = length - len(str_value)
|
|
60
|
+
if pad_len <= 0:
|
|
61
|
+
return str_value
|
|
62
|
+
padding = char * pad_len
|
|
63
|
+
return str_value + padding if align == "left" else padding + str_value
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
# 读取包名称(去除版本号)
|
|
67
|
+
def read_requirements_names(file_path: str) -> List[str]:
|
|
68
|
+
package_names = []
|
|
69
|
+
with open(file_path, "r", encoding="utf-8") as f:
|
|
70
|
+
for line in f:
|
|
71
|
+
line = line.strip()
|
|
72
|
+
if line and not line.startswith("#"): # 跳过空行和注释
|
|
73
|
+
name = line.split("==")[0]
|
|
74
|
+
name = name.split("[", 1)[0].strip()
|
|
75
|
+
package_names.append(name)
|
|
76
|
+
return package_names
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
# 打印并返回平台、Python 版本和 requirements 中包的实际安装版本
|
|
80
|
+
def get_environment_package(requirements_path: str = "requirements.txt") -> Dict[str, Any]:
|
|
81
|
+
_package_versions = {}
|
|
82
|
+
|
|
83
|
+
# 系统通用信息
|
|
84
|
+
info: Dict[str, Any] = {
|
|
85
|
+
"Platform": platform.system(),
|
|
86
|
+
"Python version": platform.python_version(),
|
|
87
|
+
"==============": "",
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
# 获取每个包的安装版本
|
|
91
|
+
for name in read_requirements_names(requirements_path):
|
|
92
|
+
try:
|
|
93
|
+
_package_versions[name] = metadata.version(name)
|
|
94
|
+
except metadata.PackageNotFoundError:
|
|
95
|
+
_package_versions[name] = "N/A"
|
|
96
|
+
|
|
97
|
+
# 合并信息
|
|
98
|
+
info.update(_package_versions)
|
|
99
|
+
return info
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
# 检查指定的所有包是否都已安装
|
|
103
|
+
def is_installed(packages: List[str]) -> bool:
|
|
104
|
+
for package_name in packages:
|
|
105
|
+
try:
|
|
106
|
+
metadata.version(package_name)
|
|
107
|
+
except metadata.PackageNotFoundError:
|
|
108
|
+
return False
|
|
109
|
+
return True
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
# 包管理(安装或卸载)
|
|
113
|
+
def package_manage(packages: List[str], uninstall: bool = False, default_index: str = "https://pypi.org/simple") -> None:
|
|
114
|
+
# 解析
|
|
115
|
+
def is_installed_with_version(package_spec: str) -> bool:
|
|
116
|
+
if "==" in package_spec:
|
|
117
|
+
pkg_name, expected_ver = package_spec.split("==")
|
|
118
|
+
else:
|
|
119
|
+
pkg_name, expected_ver = package_spec, None
|
|
120
|
+
try:
|
|
121
|
+
installed_ver = metadata.version(pkg_name)
|
|
122
|
+
return installed_ver == expected_ver if expected_ver else True
|
|
123
|
+
except metadata.PackageNotFoundError:
|
|
124
|
+
return False
|
|
125
|
+
|
|
126
|
+
# 卸载
|
|
127
|
+
if uninstall:
|
|
128
|
+
installed = [pkg for pkg in packages if is_installed_with_version(pkg)]
|
|
129
|
+
if not installed:
|
|
130
|
+
print("ℹ️ None of the specified packages are installed.")
|
|
131
|
+
return
|
|
132
|
+
print(f"🗑️ Uninstalling packages: {installed}")
|
|
133
|
+
cmd = ["uv", "pip", "uninstall", "-y", *installed]
|
|
134
|
+
subprocess.check_call(cmd)
|
|
135
|
+
|
|
136
|
+
# 安装
|
|
137
|
+
else:
|
|
138
|
+
missing = [pkg for pkg in packages if not is_installed_with_version(pkg)]
|
|
139
|
+
if not missing:
|
|
140
|
+
print("✅ All packages are already installed.")
|
|
141
|
+
return
|
|
142
|
+
print(f"📦 Installing missing packages: {missing}")
|
|
143
|
+
cmd = ["uv", "pip", "install", *missing, f"--default-index={default_index}"]
|
|
144
|
+
subprocess.check_call(cmd)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
# 初始化 ensurepip
|
|
148
|
+
def get_ensurepip():
|
|
149
|
+
global _ensurepip_initialized
|
|
150
|
+
if not _ensurepip_initialized:
|
|
151
|
+
subprocess.run([sys.executable, "-m", "ensurepip", "--upgrade"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
152
|
+
_ensurepip_initialized = True
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
# ====================================================== environment ======================================================
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
# 获取环境变量
|
|
159
|
+
def get_env(key: str, val: Any = None, print: bool = False) -> Any:
|
|
160
|
+
val = os.getenv(key, val)
|
|
161
|
+
if val == None and print:
|
|
162
|
+
logger.warning("environment variable not found '{}'", key)
|
|
163
|
+
return val
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
# 获取命令行参数
|
|
167
|
+
def get_arg(keys: List[str], default: Any = None) -> Any:
|
|
168
|
+
# 根据default推断转换函数
|
|
169
|
+
cast_func = None
|
|
170
|
+
if default is not None:
|
|
171
|
+
if isinstance(default, bool):
|
|
172
|
+
return any(arg in sys.argv for arg in keys)
|
|
173
|
+
if isinstance(default, int):
|
|
174
|
+
cast_func = int
|
|
175
|
+
elif isinstance(default, float):
|
|
176
|
+
cast_func = float
|
|
177
|
+
elif isinstance(default, str):
|
|
178
|
+
cast_func = str
|
|
179
|
+
else:
|
|
180
|
+
cast_func = lambda x: x
|
|
181
|
+
else:
|
|
182
|
+
# default是None,返回字符串
|
|
183
|
+
cast_func = lambda x: x
|
|
184
|
+
|
|
185
|
+
for i, arg in enumerate(sys.argv):
|
|
186
|
+
if arg in keys:
|
|
187
|
+
try:
|
|
188
|
+
value = cast_func(sys.argv[i + 1])
|
|
189
|
+
return value
|
|
190
|
+
except (IndexError, ValueError):
|
|
191
|
+
logger.error(f"Invalid value for {arg}, using default: {default}")
|
|
192
|
+
return default
|
|
193
|
+
return default
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
# ====================================================== subprocess ======================================================
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
# 拆分命令列表
|
|
200
|
+
def split_cmd(cmd_list: List[str]) -> List[str]:
|
|
201
|
+
"""
|
|
202
|
+
把 cmd 列表里的每个元素拆分空格,但只拆第一个空格,将参数和值分开。例如 "--host 0.0.0.0" -> ["--host", "0.0.0.0"]
|
|
203
|
+
路径或其他包含空格的值保持完整。
|
|
204
|
+
保留单独的选项(没有空格)不变
|
|
205
|
+
"""
|
|
206
|
+
result = []
|
|
207
|
+
for item in cmd_list:
|
|
208
|
+
if " " in item:
|
|
209
|
+
key, value = item.split(" ", 1) # 只拆第一个空格
|
|
210
|
+
result.append(key)
|
|
211
|
+
result.append(value)
|
|
212
|
+
else:
|
|
213
|
+
result.append(item)
|
|
214
|
+
|
|
215
|
+
print(" ".join(f'"{c}"' if " " in c else c for c in result))
|
|
216
|
+
return result
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
# 执行子进程并等待完成
|
|
220
|
+
def subprocess_run(cmd, cwd: str = None, isclean: bool = False, check: bool = True) -> str:
|
|
221
|
+
# 复制环境
|
|
222
|
+
env = os.environ.copy()
|
|
223
|
+
# 是否清除
|
|
224
|
+
if isclean:
|
|
225
|
+
# 关键:清除当前虚拟环境信息,让 uv 自己判断目标项目环境
|
|
226
|
+
env.pop("VIRTUAL_ENV", None)
|
|
227
|
+
env.pop("PYTHONPATH", None)
|
|
228
|
+
|
|
229
|
+
# 执行命令
|
|
230
|
+
if check:
|
|
231
|
+
result = subprocess.run(
|
|
232
|
+
args=cmd,
|
|
233
|
+
cwd=cwd,
|
|
234
|
+
capture_output=True, # 捕获输出
|
|
235
|
+
text=True, # 自动解码输出为 str
|
|
236
|
+
encoding="utf-8", # 编码
|
|
237
|
+
env=env, # 环境
|
|
238
|
+
check=check, # 出错时抛出异常
|
|
239
|
+
)
|
|
240
|
+
return result.stdout.strip() + "\n" + result.stderr.strip()
|
|
241
|
+
else:
|
|
242
|
+
subprocess.run(
|
|
243
|
+
args=cmd,
|
|
244
|
+
cwd=cwd,
|
|
245
|
+
stdout=None,
|
|
246
|
+
stderr=None,
|
|
247
|
+
env=env, # 环境
|
|
248
|
+
)
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
# 启动独立子进程
|
|
252
|
+
def subprocess_popen(cmd, cwd: str = None, log: str = None, isclean: bool = False) -> subprocess.Popen:
|
|
253
|
+
# 复制环境
|
|
254
|
+
env = os.environ.copy()
|
|
255
|
+
# 是否清除
|
|
256
|
+
if isclean:
|
|
257
|
+
# 关键:清除当前虚拟环境信息,让 uv 自己判断目标项目环境
|
|
258
|
+
env.pop("VIRTUAL_ENV", None)
|
|
259
|
+
env.pop("PYTHONPATH", None)
|
|
260
|
+
|
|
261
|
+
# 输出重定向(写入文件、输出控制台)
|
|
262
|
+
if log:
|
|
263
|
+
# 打开日志文件
|
|
264
|
+
f = open(log, "a", encoding="utf-8")
|
|
265
|
+
stdout = f
|
|
266
|
+
stderr = subprocess.STDOUT
|
|
267
|
+
if os.name == "nt":
|
|
268
|
+
# Console 打印
|
|
269
|
+
stdout = None
|
|
270
|
+
stderr = None
|
|
271
|
+
|
|
272
|
+
# 执行命令
|
|
273
|
+
proc = subprocess.Popen(
|
|
274
|
+
cmd,
|
|
275
|
+
cwd=cwd,
|
|
276
|
+
stdout=stdout,
|
|
277
|
+
stderr=stderr,
|
|
278
|
+
stdin=subprocess.DEVNULL,
|
|
279
|
+
text=True, # 自动解码输出为 str
|
|
280
|
+
encoding="utf-8", # 编码
|
|
281
|
+
close_fds=True, # 关闭不必要的文件描述符
|
|
282
|
+
env=env, # 环境
|
|
283
|
+
bufsize=1, # 行缓冲
|
|
284
|
+
)
|
|
285
|
+
# Colab 实时打印
|
|
286
|
+
if not log and os.name != "nt":
|
|
287
|
+
for line in proc.stdout:
|
|
288
|
+
print(line, end="")
|
|
289
|
+
|
|
290
|
+
return proc
|
|
291
|
+
else:
|
|
292
|
+
# 直接继承控制台
|
|
293
|
+
return subprocess.Popen(
|
|
294
|
+
cmd,
|
|
295
|
+
cwd=cwd,
|
|
296
|
+
stdout=None,
|
|
297
|
+
stderr=None,
|
|
298
|
+
env=env, # 环境
|
|
299
|
+
)
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
# 通过nvidia-smi检测NVIDIA GPU是否可用
|
|
303
|
+
def is_nvidia_available() -> bool:
|
|
304
|
+
try:
|
|
305
|
+
result = subprocess.run(["nvidia-smi"], capture_output=True, text=True, timeout=10)
|
|
306
|
+
return result.returncode == 0
|
|
307
|
+
except (subprocess.TimeoutExpired, FileNotFoundError, subprocess.SubprocessError):
|
|
308
|
+
return False
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
# 重启
|
|
312
|
+
def reboot(py_path: str = sys.executable, delay: int = 3) -> None:
|
|
313
|
+
from pyngrok import ngrok
|
|
314
|
+
|
|
315
|
+
# 延迟重启函数
|
|
316
|
+
def delayed_restart():
|
|
317
|
+
# 停止代理
|
|
318
|
+
is_ngrok = get_arg(["--ngrok"], False)
|
|
319
|
+
if is_ngrok:
|
|
320
|
+
try:
|
|
321
|
+
ngrok.kill()
|
|
322
|
+
except Exception as e:
|
|
323
|
+
logger.warning(f"ngrok disconnect failed: {str(e)}")
|
|
324
|
+
|
|
325
|
+
# 倒计时
|
|
326
|
+
for i in range(delay, 0, -1):
|
|
327
|
+
logger.info(f"{i}s")
|
|
328
|
+
time.sleep(1)
|
|
329
|
+
logger.info("reboot...")
|
|
330
|
+
|
|
331
|
+
# 从环境变量中获取启动参数
|
|
332
|
+
try:
|
|
333
|
+
argv = json.loads(os.environ["REBOOT_ARGS"])
|
|
334
|
+
except Exception as e:
|
|
335
|
+
logger.error("Failed to load REBOOT_ARGS:", str(e))
|
|
336
|
+
argv = sys.argv
|
|
337
|
+
|
|
338
|
+
# 重新启动
|
|
339
|
+
py_path_abs = os.path.abspath(py_path)
|
|
340
|
+
# 打印调试命令
|
|
341
|
+
logger.info(" ".join(f'"{a}"' if " " in a else a for a in [py_path_abs, *argv]))
|
|
342
|
+
# os.execl(py_path_abs, py_path_abs, *argv)
|
|
343
|
+
subprocess.run([py_path_abs, *argv])
|
|
344
|
+
|
|
345
|
+
# 启动线程来延迟重启
|
|
346
|
+
threading.Thread(target=delayed_restart).start()
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
# 打开浏览器
|
|
350
|
+
def open_browser(uri: str) -> None:
|
|
351
|
+
import webbrowser
|
|
352
|
+
|
|
353
|
+
try:
|
|
354
|
+
if uri.startswith("http://") or uri.startswith("https://"):
|
|
355
|
+
webbrowser.open(uri)
|
|
356
|
+
else:
|
|
357
|
+
webbrowser.open(Path(uri).as_uri())
|
|
358
|
+
except Exception as e:
|
|
359
|
+
logger.error(f"{uri} -> open error: {str(e)}")
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
# ====================================================== process ======================================================
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
# 在指定范围(如 8010-8080)内随机选择一个未被占用的端口
|
|
366
|
+
def find_free_port(start: int = 8080, end: int = 8099, max_attempts: int = 3) -> int:
|
|
367
|
+
for _ in range(max_attempts):
|
|
368
|
+
port = random.randint(start, end)
|
|
369
|
+
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
|
370
|
+
try:
|
|
371
|
+
s.bind(("", port))
|
|
372
|
+
return port
|
|
373
|
+
except OSError:
|
|
374
|
+
continue
|
|
375
|
+
raise RuntimeError(f"Unable to find a free port in the range {start}-{end}")
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
# 等待端口可访问
|
|
379
|
+
def wait_port(host: str, port: int, timeout: int = 5) -> bool:
|
|
380
|
+
host = "127.0.0.1" if host == "0.0.0.0" else host
|
|
381
|
+
start = time.time()
|
|
382
|
+
while time.time() - start < timeout:
|
|
383
|
+
try:
|
|
384
|
+
with socket.create_connection((host, port), timeout=1):
|
|
385
|
+
return True
|
|
386
|
+
except Exception:
|
|
387
|
+
time.sleep(0.1)
|
|
388
|
+
return False
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
# 删除指定端口进程
|
|
392
|
+
def kill_process(pid: int = None, port: int = None, force: bool = True) -> bool:
|
|
393
|
+
"""
|
|
394
|
+
Kill a process by PID or by port (cross-platform).
|
|
395
|
+
|
|
396
|
+
Args:
|
|
397
|
+
pid (int, optional): Process ID to kill.
|
|
398
|
+
port (int, optional): Port number to kill processes using it.
|
|
399
|
+
force (bool): Force kill (Windows: /F, Unix: SIGKILL). Default True.
|
|
400
|
+
"""
|
|
401
|
+
system = platform.system()
|
|
402
|
+
|
|
403
|
+
if pid is None and port is None:
|
|
404
|
+
raise ValueError("Either pid or port must be provided.")
|
|
405
|
+
|
|
406
|
+
# --- Case 1: Kill by PID ---
|
|
407
|
+
if pid is not None:
|
|
408
|
+
try:
|
|
409
|
+
if system == "Windows":
|
|
410
|
+
cmd = ["taskkill", "/PID", str(pid)]
|
|
411
|
+
if force:
|
|
412
|
+
cmd.append("/F")
|
|
413
|
+
subprocess.run(cmd, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
414
|
+
else:
|
|
415
|
+
os.kill(pid, signal.SIGKILL if force else signal.SIGTERM)
|
|
416
|
+
logger.info(f"Process {pid} terminated successfully.")
|
|
417
|
+
return True
|
|
418
|
+
except Exception as e:
|
|
419
|
+
logger.error(f"Failed to terminate process {pid}: {str(e)}")
|
|
420
|
+
return False
|
|
421
|
+
|
|
422
|
+
# --- Case 2: Kill by port ---
|
|
423
|
+
pids = []
|
|
424
|
+
try:
|
|
425
|
+
if system == "Windows":
|
|
426
|
+
result = subprocess.run(["netstat", "-ano"], capture_output=True, text=True)
|
|
427
|
+
for line in result.stdout.splitlines():
|
|
428
|
+
if f":{port} " in line and "LISTENING" in line:
|
|
429
|
+
pid = line.strip().split()[-1]
|
|
430
|
+
pids.append(pid)
|
|
431
|
+
else:
|
|
432
|
+
result = subprocess.run(["lsof", "-t", f"-i:{port}"], capture_output=True, text=True)
|
|
433
|
+
pids = [p.strip() for p in result.stdout.splitlines() if p.strip()]
|
|
434
|
+
except Exception as e:
|
|
435
|
+
logger.error(f"Failed to find processes on port {port}: {str(e)}")
|
|
436
|
+
return False
|
|
437
|
+
|
|
438
|
+
if not pids:
|
|
439
|
+
logger.warning(f"No process found listening on port {port}.")
|
|
440
|
+
return False
|
|
441
|
+
|
|
442
|
+
success = False
|
|
443
|
+
for pid in pids:
|
|
444
|
+
logger.info(f"Killing process {pid} listening on port {port}...")
|
|
445
|
+
success = kill_process(pid=int(pid), force=force) or success
|
|
446
|
+
|
|
447
|
+
return success
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
# 杀掉残余的 localtunnel 进程,防止 subdomain 被占用导致名称随机
|
|
451
|
+
def kill_processes_tunnel(port: int) -> None:
|
|
452
|
+
import psutil
|
|
453
|
+
|
|
454
|
+
for proc in psutil.process_iter(["pid", "name", "cmdline"]):
|
|
455
|
+
try:
|
|
456
|
+
cmdline = " ".join(proc.info.get("cmdline") or [])
|
|
457
|
+
if ("localtunnel" in cmdline or " lt " in cmdline or cmdline.endswith(" lt")) and str(port) in cmdline:
|
|
458
|
+
proc.terminate()
|
|
459
|
+
logger.info(f"Killed localtunnel process pid={proc.pid} cmdline={cmdline!r}")
|
|
460
|
+
except (psutil.NoSuchProcess, psutil.AccessDenied):
|
|
461
|
+
pass
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
# ====================================================== executor ======================================================
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
# 在主线程中直接运行异步函数(协程)
|
|
468
|
+
def run(func: Coroutine[Any, Any, T]) -> T:
|
|
469
|
+
return asyncio.run(func)
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
# 在已运行的事件循环中从另一个线程安全地提交协程执行
|
|
473
|
+
def run_coroutine_threadsafe(coro: Coroutine[Any, Any, T]) -> concurrent.futures.Future[T]:
|
|
474
|
+
return asyncio.run_coroutine_threadsafe(coro, _get_loop())
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
# 在后台线程中执行函数
|
|
478
|
+
def run_background(func: Callable[..., Any], *args, **kwargs) -> None:
|
|
479
|
+
|
|
480
|
+
def _safe() -> None:
|
|
481
|
+
try:
|
|
482
|
+
func(*args, **kwargs)
|
|
483
|
+
except Exception as e:
|
|
484
|
+
logger.error(f"Background task error: {str(e)}")
|
|
485
|
+
|
|
486
|
+
threading.Thread(target=_safe, daemon=True).start()
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
# 在异步代码中安全地调用同步函数(推荐)
|
|
490
|
+
async def to_thread(func: Callable[..., T], *args) -> T:
|
|
491
|
+
return await asyncio.to_thread(func, *args)
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
# 在异步代码中安全地调用同步函数(兼容旧版本)
|
|
495
|
+
def run_in_executor(func: Callable[..., T], *args) -> asyncio.Future[T]:
|
|
496
|
+
return _get_loop().run_in_executor(_get_executor(), func, *args)
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
# ====================================================== tool ======================================================
|
|
500
|
+
|
|
501
|
+
# 工具配置字典
|
|
502
|
+
TOOL_CONFIG = {
|
|
503
|
+
"uv": {"display_name": "uv", "version_pattern": r"uv\s+([\d.]+\d)"},
|
|
504
|
+
"python": {"display_name": "Python", "version_pattern": r"Python\s+([\d.]+\d)"},
|
|
505
|
+
"pip": {"display_name": "pip", "version_pattern": r"pip\s+([\d.]+\d)"},
|
|
506
|
+
"ffmpeg": {"display_name": "ffmpeg", "version_pattern": r"ffmpeg\s+version\s+([\d.]+\d)"},
|
|
507
|
+
"ffprobe": {"display_name": "ffprobe", "version_pattern": r"ffprobe\s+version\s+([\d.]+\d)"},
|
|
508
|
+
"git": {"display_name": "git", "version_pattern": r"git\s+version\s+([\d.]+\d)"},
|
|
509
|
+
"aria2c": {"display_name": "aria2", "version_pattern": r"aria2\s+version\s+([\d.]+\d)"},
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
# 检查指定的软件是否已安装
|
|
514
|
+
class ToolEnvChecker:
|
|
515
|
+
|
|
516
|
+
def __init__(self, name: str, fallback_dirs: Union[str, List[str]] = None):
|
|
517
|
+
self.name = name.lower()
|
|
518
|
+
if isinstance(fallback_dirs, str):
|
|
519
|
+
self.fallback_dirs = [os.path.abspath(fallback_dirs)]
|
|
520
|
+
elif isinstance(fallback_dirs, list):
|
|
521
|
+
self.fallback_dirs = [os.path.abspath(d) for d in fallback_dirs]
|
|
522
|
+
else:
|
|
523
|
+
self.fallback_dirs = []
|
|
524
|
+
self.order = []
|
|
525
|
+
self._tool_path_cache: Optional[str] = None
|
|
526
|
+
self._version_cache: Optional[str] = None
|
|
527
|
+
|
|
528
|
+
# 查找工具路径
|
|
529
|
+
def find_tool(self, auto_add_to_path: bool = True) -> Optional[str]:
|
|
530
|
+
if self._tool_path_cache:
|
|
531
|
+
return self._tool_path_cache
|
|
532
|
+
|
|
533
|
+
# 1. 优先系统 PATH
|
|
534
|
+
tool_path = shutil.which(self.name)
|
|
535
|
+
if tool_path:
|
|
536
|
+
self._tool_path_cache = Path(tool_path).as_posix()
|
|
537
|
+
return self._tool_path_cache
|
|
538
|
+
|
|
539
|
+
# 2. 再查备用目录
|
|
540
|
+
for directory in self.fallback_dirs:
|
|
541
|
+
if os.path.isdir(directory):
|
|
542
|
+
for fname in os.listdir(directory):
|
|
543
|
+
fpath = os.path.join(directory, fname)
|
|
544
|
+
if fname.lower().startswith(self.name) and os.access(fpath, os.X_OK):
|
|
545
|
+
if auto_add_to_path:
|
|
546
|
+
self._add_to_path(directory)
|
|
547
|
+
self._tool_path_cache = Path(fpath).as_posix()
|
|
548
|
+
return self._tool_path_cache
|
|
549
|
+
return None
|
|
550
|
+
|
|
551
|
+
# 添加环境变量
|
|
552
|
+
def _add_to_path(self, directory: str) -> None:
|
|
553
|
+
directory = os.path.abspath(directory)
|
|
554
|
+
if directory not in self.order:
|
|
555
|
+
self.order.append(directory)
|
|
556
|
+
current_path = os.environ.get("PATH", "")
|
|
557
|
+
paths = current_path.split(os.pathsep)
|
|
558
|
+
paths = [p for p in paths if p not in self.order]
|
|
559
|
+
new_path = os.pathsep.join(self.order + paths)
|
|
560
|
+
os.environ["PATH"] = new_path
|
|
561
|
+
logger.info(f"Added {directory} to PATH")
|
|
562
|
+
|
|
563
|
+
# 获取工具版本
|
|
564
|
+
def get_version(self, tool_path: Optional[str] = None, version_arg: str = "--version") -> Optional[str]:
|
|
565
|
+
if self._version_cache:
|
|
566
|
+
return self._version_cache
|
|
567
|
+
|
|
568
|
+
try:
|
|
569
|
+
if tool_path is None:
|
|
570
|
+
tool_path = self.find_tool()
|
|
571
|
+
if tool_path is None:
|
|
572
|
+
return None
|
|
573
|
+
|
|
574
|
+
result = subprocess.run([tool_path, version_arg], capture_output=True, text=True, timeout=5)
|
|
575
|
+
version = result.stdout.strip() or result.stderr.strip()
|
|
576
|
+
self._version_cache = version if version else None
|
|
577
|
+
return self._version_cache
|
|
578
|
+
except (FileNotFoundError, subprocess.TimeoutExpired, subprocess.SubprocessError) as e:
|
|
579
|
+
logger.debug(f"Failed to get version for {self.name}: {str(e)}")
|
|
580
|
+
return None
|
|
581
|
+
|
|
582
|
+
# 检查工具是否可用
|
|
583
|
+
def is_available(self) -> bool:
|
|
584
|
+
return self.find_tool() is not None
|
|
585
|
+
|
|
586
|
+
|
|
587
|
+
# 统一的工具检查函数
|
|
588
|
+
def check_tool(tool_name: str, show_print: bool = False, length: int = 15) -> ToolEnvChecker:
|
|
589
|
+
checker = ToolEnvChecker(tool_name)
|
|
590
|
+
tool_path = checker.find_tool()
|
|
591
|
+
config = TOOL_CONFIG.get(tool_name, {})
|
|
592
|
+
display_name = config.get("display_name", tool_name)
|
|
593
|
+
|
|
594
|
+
if not tool_path:
|
|
595
|
+
logger.warning(f"{tool_name} is not available")
|
|
596
|
+
if show_print:
|
|
597
|
+
print(f"{pad_string(display_name + ' Not Found', align='left', length=length)} not available")
|
|
598
|
+
else:
|
|
599
|
+
print(f"{display_name} Not Found")
|
|
600
|
+
return checker
|
|
601
|
+
|
|
602
|
+
raw_version = checker.get_version()
|
|
603
|
+
version = "Unknown"
|
|
604
|
+
if raw_version:
|
|
605
|
+
pattern = config.get("version_pattern")
|
|
606
|
+
if pattern:
|
|
607
|
+
match = re.search(pattern, raw_version, re.IGNORECASE)
|
|
608
|
+
if match:
|
|
609
|
+
version = match.group(1)
|
|
610
|
+
else:
|
|
611
|
+
version = raw_version.split()[0] if raw_version else "Unknown"
|
|
612
|
+
|
|
613
|
+
if show_print:
|
|
614
|
+
label = f"{display_name} {version}"
|
|
615
|
+
print(f"{pad_string(label, align='left', length=length)} Using {tool_path}")
|
|
616
|
+
else:
|
|
617
|
+
print(f"{display_name} {version}")
|
|
618
|
+
print(f"Using {tool_path}")
|
|
619
|
+
|
|
620
|
+
return checker
|
|
621
|
+
|
|
622
|
+
|
|
623
|
+
# 检查 uv 工具
|
|
624
|
+
def check_uv(show_print: bool = False, length: int = 15) -> ToolEnvChecker:
|
|
625
|
+
return check_tool(tool_name="uv", show_print=show_print, length=length)
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
# 检查 Python
|
|
629
|
+
def check_python(show_print: bool = False, length: int = 15) -> ToolEnvChecker:
|
|
630
|
+
return check_tool(tool_name="python", show_print=show_print, length=length)
|
|
631
|
+
|
|
632
|
+
|
|
633
|
+
# 检查 ffplay
|
|
634
|
+
def check_ffplay(show_print: bool = False, length: int = 15) -> ToolEnvChecker:
|
|
635
|
+
return check_tool(tool_name="ffplay", show_print=show_print, length=length)
|
|
636
|
+
|
|
637
|
+
|
|
638
|
+
# 检查 ffmpeg
|
|
639
|
+
def check_ffmpeg(show_print: bool = False, length: int = 15) -> ToolEnvChecker:
|
|
640
|
+
return check_tool(tool_name="ffmpeg", show_print=show_print, length=length)
|
|
641
|
+
|
|
642
|
+
|
|
643
|
+
# 检查 ffprobe
|
|
644
|
+
def check_ffprobe(show_print: bool = False, length: int = 15) -> ToolEnvChecker:
|
|
645
|
+
return check_tool(tool_name="ffprobe", show_print=show_print, length=length)
|
|
646
|
+
|
|
647
|
+
|
|
648
|
+
# 检查 git
|
|
649
|
+
def check_git(show_print: bool = False, length: int = 15) -> ToolEnvChecker:
|
|
650
|
+
return check_tool(tool_name="git", show_print=show_print, length=length)
|
|
651
|
+
|
|
652
|
+
|
|
653
|
+
# 检查 aria2c
|
|
654
|
+
def check_aria2c(show_print: bool = False, length: int = 15) -> ToolEnvChecker:
|
|
655
|
+
return check_tool(tool_name="aria2c", show_print=show_print, length=length)
|
|
656
|
+
|
|
657
|
+
|
|
658
|
+
# ====================================================== ffmpeg ======================================================
|
|
659
|
+
|
|
660
|
+
|
|
661
|
+
# 是否是指定类型
|
|
662
|
+
def is_codec_type(path: str) -> str:
|
|
663
|
+
try:
|
|
664
|
+
streams = process_probe_metadata(path, "streams", [])
|
|
665
|
+
has_video = any(s["codec_type"] == "video" for s in streams)
|
|
666
|
+
has_audio = any(s["codec_type"] == "audio" for s in streams)
|
|
667
|
+
if has_video and not has_audio:
|
|
668
|
+
return "video" # 纯视频(无音轨)
|
|
669
|
+
elif has_audio and not has_video:
|
|
670
|
+
return "audio" # 纯音频
|
|
671
|
+
elif has_audio and has_video:
|
|
672
|
+
return "video+audio" # 常见的带声音的视频文件
|
|
673
|
+
else:
|
|
674
|
+
return "unknown" # 没有检测到音频/视频流
|
|
675
|
+
except Exception as e:
|
|
676
|
+
return "unknown"
|
|
677
|
+
|
|
678
|
+
|
|
679
|
+
# 获取元数据
|
|
680
|
+
def process_probe_metadata(path: str, metadata: str = "format", default: Union[Dict, List] = None) -> Union[Dict, List[Dict]]:
|
|
681
|
+
import ffmpeg
|
|
682
|
+
|
|
683
|
+
if default is None:
|
|
684
|
+
default = {}
|
|
685
|
+
try:
|
|
686
|
+
probe = ffmpeg.probe(path)
|
|
687
|
+
return probe.get(metadata, default)
|
|
688
|
+
except Exception as e:
|
|
689
|
+
logger.error(f"Error: {path} {str(e)}")
|
|
690
|
+
return default
|
|
691
|
+
|
|
692
|
+
|
|
693
|
+
# 运行 ffmpeg 命令并显示进度
|
|
694
|
+
def process_ffmpeg(duration: float, cmd: List[str], debug: bool = False) -> None:
|
|
695
|
+
logger.debug("command: {}", " ".join(cmd))
|
|
696
|
+
checker = check_ffmpeg()
|
|
697
|
+
if not checker.is_available():
|
|
698
|
+
logger.error("Ffmpeg is not available, please install ffmpeg first")
|
|
699
|
+
return
|
|
700
|
+
# debug 模式:捕获所有输出,便于调试
|
|
701
|
+
if debug:
|
|
702
|
+
res = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
|
703
|
+
if res.returncode != 0:
|
|
704
|
+
raise RuntimeError(f"FFmpeg failed (debug mode). returncode={res.returncode}\n\nSTDERR:\n{res.stderr}")
|
|
705
|
+
return
|
|
706
|
+
# 启动子进程
|
|
707
|
+
proc = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True, bufsize=1, encoding="utf-8") # drop stdout to avoid filling buffer
|
|
708
|
+
# 进度条
|
|
709
|
+
pbar = tqdm(total=round(duration, 2), unit="s", ncols=100, desc="Processing", dynamic_ncols=True)
|
|
710
|
+
# 解析 stderr 中的 time=HH:MM:SS.xx
|
|
711
|
+
time_pattern = re.compile(r"time=(\d+):(\d+):(\d+(?:\.\d+)?)")
|
|
712
|
+
# 保留stderr的尾部用于调试
|
|
713
|
+
last_lines = deque(maxlen=500)
|
|
714
|
+
prev_sec = 0.0
|
|
715
|
+
# 实时读取 stderr
|
|
716
|
+
try:
|
|
717
|
+
for line in iter(proc.stderr.readline, ""):
|
|
718
|
+
if not line:
|
|
719
|
+
break
|
|
720
|
+
last_lines.append(line)
|
|
721
|
+
line = line.strip()
|
|
722
|
+
m = time_pattern.search(line)
|
|
723
|
+
if m:
|
|
724
|
+
h, mm, ss = m.groups()
|
|
725
|
+
sec = int(h) * 3600 + int(mm) * 60 + float(ss)
|
|
726
|
+
delta = max(0.0, sec - prev_sec)
|
|
727
|
+
if delta > 0:
|
|
728
|
+
pbar.update(delta)
|
|
729
|
+
prev_sec = sec
|
|
730
|
+
finally:
|
|
731
|
+
try:
|
|
732
|
+
remaining = proc.stderr.read()
|
|
733
|
+
if remaining:
|
|
734
|
+
last_lines.append(remaining)
|
|
735
|
+
except Exception:
|
|
736
|
+
pass
|
|
737
|
+
pbar.close()
|
|
738
|
+
proc.wait()
|
|
739
|
+
# 检查返回码
|
|
740
|
+
if proc.returncode != 0:
|
|
741
|
+
tail = "".join(last_lines)
|
|
742
|
+
raise RuntimeError(f"FFmpeg process failed (returncode={proc.returncode}). Last stderr lines:\n{tail}")
|
|
743
|
+
|
|
744
|
+
|
|
745
|
+
# 删除 ffmpeg 进程
|
|
746
|
+
def terminate_ffmpeg_process():
|
|
747
|
+
import psutil
|
|
748
|
+
|
|
749
|
+
# current_user = getpass.getuser()
|
|
750
|
+
for proc in psutil.process_iter(attrs=["pid", "name", "username"]):
|
|
751
|
+
# if proc.info["username"] != current_user:
|
|
752
|
+
# continue
|
|
753
|
+
name = proc.info.get("name", "")
|
|
754
|
+
if name and "ffmpeg" in name.lower():
|
|
755
|
+
try:
|
|
756
|
+
proc.kill()
|
|
757
|
+
except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.TimeoutExpired):
|
|
758
|
+
try:
|
|
759
|
+
proc.terminate()
|
|
760
|
+
proc.wait(timeout=3)
|
|
761
|
+
except Exception as e:
|
|
762
|
+
print(f"Failed to terminate ffmpeg process {proc.pid}: {str(e)}")
|
|
763
|
+
|
|
764
|
+
|
|
765
|
+
# ====================================================== aria2 ======================================================
|
|
766
|
+
|
|
767
|
+
|
|
768
|
+
# 多线程下载
|
|
769
|
+
def process_aria2(url: str, save_folder: str = None, filename: str = None) -> str:
|
|
770
|
+
# 默认下载地址
|
|
771
|
+
if not save_folder:
|
|
772
|
+
save_folder = tempfile.gettempdir()
|
|
773
|
+
|
|
774
|
+
# 默认文件名
|
|
775
|
+
if not filename:
|
|
776
|
+
filename = url.split("/")[-1]
|
|
777
|
+
|
|
778
|
+
# 创建文件夹(如果文件夹不存在)
|
|
779
|
+
Path(save_folder).mkdir(parents=True, exist_ok=True)
|
|
780
|
+
|
|
781
|
+
# 拼接完整的文件路径
|
|
782
|
+
file_path = Path(os.path.join(save_folder, filename)).as_posix()
|
|
783
|
+
|
|
784
|
+
# 文件是否存在
|
|
785
|
+
if os.path.exists(file_path):
|
|
786
|
+
return file_path
|
|
787
|
+
|
|
788
|
+
# 检查工具
|
|
789
|
+
checker = check_aria2c()
|
|
790
|
+
if not checker.is_available():
|
|
791
|
+
logger.error("Aria2c is not available, please install aria2c first")
|
|
792
|
+
return None
|
|
793
|
+
|
|
794
|
+
# 默认连接数
|
|
795
|
+
connections = 16
|
|
796
|
+
|
|
797
|
+
# 构建命令
|
|
798
|
+
cmd = [
|
|
799
|
+
"aria2c",
|
|
800
|
+
url,
|
|
801
|
+
f"--dir {save_folder}",
|
|
802
|
+
"--continue=true", # 断点续传
|
|
803
|
+
f"--max-connection-per-server {str(connections)}", # 每服务器连接数
|
|
804
|
+
f"--split {str(connections)}", # 下载分片数量
|
|
805
|
+
"--min-split-size=1M", # 每个分片最小1MB
|
|
806
|
+
"--auto-file-renaming=false", # 禁止重复重命名
|
|
807
|
+
"--retry-wait=3", # 失败重试等待时间
|
|
808
|
+
"--max-tries=0", # 无限重试
|
|
809
|
+
"--console-log-level=warn", # 控制台日志级别(warn以上)
|
|
810
|
+
]
|
|
811
|
+
|
|
812
|
+
# 执行命令
|
|
813
|
+
proc = subprocess.Popen(
|
|
814
|
+
split_cmd(cmd),
|
|
815
|
+
stdout=subprocess.PIPE,
|
|
816
|
+
stderr=subprocess.STDOUT,
|
|
817
|
+
text=True,
|
|
818
|
+
encoding="utf-8",
|
|
819
|
+
errors="replace",
|
|
820
|
+
)
|
|
821
|
+
|
|
822
|
+
# 进度条 (匹配类似 “[#1 12.3MiB/3.6GiB(0%) CN:16 DL:1.2MiB]” 的行)
|
|
823
|
+
pattern = re.compile(r"\((\d+)%\)")
|
|
824
|
+
pbar = tqdm(total=100, unit="%", desc=filename, dynamic_ncols=True)
|
|
825
|
+
last_percent = 0
|
|
826
|
+
for line in proc.stdout:
|
|
827
|
+
line = line.strip()
|
|
828
|
+
match = pattern.search(line)
|
|
829
|
+
if match:
|
|
830
|
+
percent = int(match.group(1))
|
|
831
|
+
delta = percent - last_percent
|
|
832
|
+
if delta > 0:
|
|
833
|
+
pbar.update(delta)
|
|
834
|
+
last_percent = percent
|
|
835
|
+
proc.wait()
|
|
836
|
+
pbar.close()
|
|
837
|
+
# 判断
|
|
838
|
+
if proc.returncode == 0:
|
|
839
|
+
logger.info(f"File has been saved to: {file_path}")
|
|
840
|
+
else:
|
|
841
|
+
logger.error("Failed to download error_code: {}", proc.returncode)
|
|
842
|
+
return file_path
|
|
843
|
+
|
|
844
|
+
|
|
845
|
+
if __name__ == "__main__":
|
|
846
|
+
|
|
847
|
+
# print(is_installed("edge_tts"))
|
|
848
|
+
# print(get_env("HUGGINGFACEHUB_API_TOKEN"))
|
|
849
|
+
|
|
850
|
+
# # 检查 Python
|
|
851
|
+
# print("\nTools")
|
|
852
|
+
# check_uv(show_print=True)
|
|
853
|
+
# check_python(show_print=True)
|
|
854
|
+
# check_pip(show_print=True)
|
|
855
|
+
# check_ffmpeg(show_print=True)
|
|
856
|
+
# check_ffprobe(show_print=True)
|
|
857
|
+
# check_git(show_print=True)
|
|
858
|
+
# check_aria2c(show_print=True)
|
|
859
|
+
|
|
860
|
+
# print(is_codec_type("C:/Users/xiesx/Desktop/test/test.mp4"))
|
|
861
|
+
# print(process_probe_metadata("C:/Users/xiesx/Desktop/test/test.mp4"))
|
|
862
|
+
|
|
863
|
+
# print(process_aria2("https://github.com/xiesx123/CreatorBox/releases/download/1.0.20/deepspeed-0.15.1+d5315d0-cp310-cp310-win_amd64.whl"))
|
|
864
|
+
|
|
865
|
+
# checker = check_git()
|
|
866
|
+
# if checker.get_version() == "None":
|
|
867
|
+
# logger.error("git is not available. please install git first.")
|
|
868
|
+
# subprocess_popen(["git", "clone", "https://github.com/Sanster/IOPaint.git", "extensions/stable_diffusion_webui2"]).wait()
|
|
869
|
+
|
|
870
|
+
print(read_requirements_names("requirements.txt"))
|