thf 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- thf/__init__.py +3 -0
- thf/__main__.py +3 -0
- thf/app.py +1707 -0
- thf-0.1.0.dist-info/METADATA +135 -0
- thf-0.1.0.dist-info/RECORD +9 -0
- thf-0.1.0.dist-info/WHEEL +5 -0
- thf-0.1.0.dist-info/entry_points.txt +2 -0
- thf-0.1.0.dist-info/licenses/LICENSE +21 -0
- thf-0.1.0.dist-info/top_level.txt +1 -0
thf/app.py
ADDED
|
@@ -0,0 +1,1707 @@
|
|
|
1
|
+
"""Hugging Face 관리 TUI.
|
|
2
|
+
|
|
3
|
+
캐시/허브/내 저장소/정리 탭. 공유 서버에서 여러 사람이 같은 캐시(HF_HUB_CACHE)를 쓰는 경우를
|
|
4
|
+
기본으로 생각해, 다른 사람이 받은 항목은 소유자를 보여주고 지울 때 이름을 직접 입력하게 한다.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import io
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
import shutil
|
|
13
|
+
import signal
|
|
14
|
+
import subprocess
|
|
15
|
+
import sys
|
|
16
|
+
import threading
|
|
17
|
+
import time
|
|
18
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from typing import Optional
|
|
21
|
+
|
|
22
|
+
from huggingface_hub import HfApi, constants, get_token, scan_cache_dir, snapshot_download
|
|
23
|
+
from huggingface_hub.errors import CacheNotFound, HfHubHTTPError
|
|
24
|
+
from huggingface_hub.utils import disable_progress_bars, filter_repo_objects
|
|
25
|
+
from huggingface_hub.utils import logging as hf_logging
|
|
26
|
+
from rich.text import Text
|
|
27
|
+
from tqdm.std import tqdm
|
|
28
|
+
from textual import on, work
|
|
29
|
+
from textual.app import App, ComposeResult
|
|
30
|
+
from textual.binding import Binding
|
|
31
|
+
from textual.containers import Grid, Horizontal, Vertical, VerticalScroll
|
|
32
|
+
from textual.screen import ModalScreen
|
|
33
|
+
from textual.widgets import (
|
|
34
|
+
Button,
|
|
35
|
+
Checkbox,
|
|
36
|
+
DataTable,
|
|
37
|
+
Footer,
|
|
38
|
+
Header,
|
|
39
|
+
Input,
|
|
40
|
+
Label,
|
|
41
|
+
RichLog,
|
|
42
|
+
Select,
|
|
43
|
+
Static,
|
|
44
|
+
TabbedContent,
|
|
45
|
+
TabPane,
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
try: # 캐시 전체에서 공유하는 blob 저장소 (huggingface_hub 2.x)
|
|
49
|
+
from huggingface_hub.utils import _shared_blobs
|
|
50
|
+
except ImportError: # pragma: no cover
|
|
51
|
+
_shared_blobs = None
|
|
52
|
+
|
|
53
|
+
HUB = constants.ENDPOINT.rstrip("/")
|
|
54
|
+
|
|
55
|
+
# 허브 검색 필터. 허깅페이스 웹사이트 Models/Datasets 페이지의 필터와 같은 값을 쓴다
|
|
56
|
+
MODEL_TASKS = [
|
|
57
|
+
("Text Generation", "text-generation"), ("Image-Text-to-Text", "image-text-to-text"),
|
|
58
|
+
("Any-to-Any", "any-to-any"), ("Text-to-Image", "text-to-image"), ("Text-to-Video", "text-to-video"),
|
|
59
|
+
("Image-to-Image", "image-to-image"), ("Feature Extraction", "feature-extraction"),
|
|
60
|
+
("Sentence Similarity", "sentence-similarity"), ("Text Classification", "text-classification"),
|
|
61
|
+
("Token Classification", "token-classification"), ("Zero-Shot Classification", "zero-shot-classification"),
|
|
62
|
+
("Fill-Mask", "fill-mask"), ("Question Answering", "question-answering"), ("Translation", "translation"),
|
|
63
|
+
("Summarization", "summarization"), ("Automatic Speech Recognition", "automatic-speech-recognition"),
|
|
64
|
+
("Text-to-Speech", "text-to-speech"), ("Image Classification", "image-classification"),
|
|
65
|
+
("Object Detection", "object-detection"), ("Image Segmentation", "image-segmentation"),
|
|
66
|
+
("Depth Estimation", "depth-estimation"),
|
|
67
|
+
]
|
|
68
|
+
DATASET_TASKS = [
|
|
69
|
+
("Text Generation", "text-generation"), ("Question Answering", "question-answering"),
|
|
70
|
+
("Text Classification", "text-classification"), ("Token Classification", "token-classification"),
|
|
71
|
+
("Translation", "translation"), ("Summarization", "summarization"),
|
|
72
|
+
("Image Classification", "image-classification"), ("Image-to-Text", "image-to-text"),
|
|
73
|
+
("Visual Question Answering", "visual-question-answering"),
|
|
74
|
+
("Automatic Speech Recognition", "automatic-speech-recognition"),
|
|
75
|
+
]
|
|
76
|
+
PARAMS = [("< 1B", "max:1B"), ("1B – 6B", "min:1B,max:6B"), ("6B – 12B", "min:6B,max:12B"),
|
|
77
|
+
("12B – 32B", "min:12B,max:32B"), ("32B – 128B", "min:32B,max:128B"), ("> 128B", "min:128B")]
|
|
78
|
+
LIBRARIES = [("Transformers", "transformers"), ("Diffusers", "diffusers"),
|
|
79
|
+
("sentence-transformers", "sentence-transformers"), ("GGUF", "gguf"), ("MLX", "mlx"),
|
|
80
|
+
("ONNX", "onnx"), ("PEFT", "peft"), ("timm", "timm"), ("Safetensors", "safetensors")]
|
|
81
|
+
APPS = [("Ollama", "ollama"), ("vLLM", "vllm"), ("llama.cpp", "llama.cpp"), ("LM Studio", "lmstudio"),
|
|
82
|
+
("MLX LM", "mlx-lm"), ("SGLang", "sglang"), ("Jan", "jan")]
|
|
83
|
+
LANGUAGES = [("English", "en"), ("Korean", "ko"), ("Chinese", "zh"), ("Japanese", "ja"), ("French", "fr"),
|
|
84
|
+
("German", "de"), ("Spanish", "es")]
|
|
85
|
+
LICENSES = [("apache-2.0", "apache-2.0"), ("mit", "mit"), ("cc-by-4.0", "cc-by-4.0"),
|
|
86
|
+
("cc-by-nc-4.0", "cc-by-nc-4.0"), ("openrail", "openrail"), ("gpl-3.0", "gpl-3.0")]
|
|
87
|
+
SIZES = [("< 1K rows", "n<1K"), ("1K – 10K", "1K<n<10K"), ("10K – 100K", "10K<n<100K"),
|
|
88
|
+
("100K – 1M", "100K<n<1M"), ("1M – 10M", "1M<n<10M"), ("10M – 100M", "10M<n<100M"),
|
|
89
|
+
("100M – 1B", "100M<n<1B"), ("> 1T", "n>1T")]
|
|
90
|
+
ACCESS = [("Open (no access request)", "open"), ("Gated", "gated")]
|
|
91
|
+
SORTS = [("Trending", "trending_score"), ("Most downloads", "downloads"), ("Most likes", "likes"),
|
|
92
|
+
("Recently updated", "last_modified"), ("Recently created", "created_at")]
|
|
93
|
+
# 타입마다 보여줄 필터 (이름, 선택지). 선택지가 None 이면 자유 입력
|
|
94
|
+
HUB_FILTERS = {
|
|
95
|
+
"model": [("task", "Task", MODEL_TASKS), ("params", "Parameters", PARAMS), ("library", "Library", LIBRARIES),
|
|
96
|
+
("app", "Runs with", APPS), ("language", "Language", LANGUAGES), ("license", "License", LICENSES),
|
|
97
|
+
("access", "Access", ACCESS), ("tags", "Other tags", None), ("sort", "Sort", SORTS)],
|
|
98
|
+
"dataset": [("task", "Task", DATASET_TASKS), ("size", "Size", SIZES), ("language", "Language", LANGUAGES),
|
|
99
|
+
("license", "License", LICENSES), ("access", "Access", ACCESS), ("tags", "Other tags", None),
|
|
100
|
+
("sort", "Sort", SORTS)],
|
|
101
|
+
"space": [("tags", "Other tags", None), ("sort", "Sort", [x for x in SORTS if x[1] != "downloads"])],
|
|
102
|
+
}
|
|
103
|
+
# 받아 둔 목록의 필터: 허브와 같은 항목 (단, "Runs with" 는 허브만 계산할 수 있어 뺀다)
|
|
104
|
+
LOCAL_SORTS = [("Largest", "size"), ("Recently used", "used"), ("Least recently used", "unused"), ("Name", "name")]
|
|
105
|
+
LOCAL_FILTERS = {rt: [f for f in fields if f[0] not in ("app", "sort")] + [("sort", "Sort", LOCAL_SORTS)]
|
|
106
|
+
for rt, fields in HUB_FILTERS.items()}
|
|
107
|
+
|
|
108
|
+
# 받아 둔 리포의 허브 정보 (용도, 라이브러리, 태그, 파라미터 수). 리포마다 한 번 물어보고 저장해 둔다
|
|
109
|
+
META_FILE = os.path.expanduser("~/.cache/thf/hub_info.json")
|
|
110
|
+
META_TTL = 7 * 86400 # 이보다 오래된 정보는 다시 묻는다
|
|
111
|
+
META_RETRY = 86400 # 못 가져온 리포(삭제, 비공개, 오프라인)는 하루 뒤에 다시 묻는다
|
|
112
|
+
REPO_TYPES = [("Models", "model"), ("Datasets", "dataset"), ("Spaces", "space")]
|
|
113
|
+
CACHE_TYPES = [t for _, t in REPO_TYPES]
|
|
114
|
+
URL_PREFIX = {"model": "", "dataset": "datasets/", "space": "spaces/"}
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
CONFIG_FILE = os.path.expanduser("~/.config/thf/config.json")
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def load_config() -> dict:
|
|
121
|
+
try:
|
|
122
|
+
with open(CONFIG_FILE) as f:
|
|
123
|
+
return json.load(f)
|
|
124
|
+
except (OSError, ValueError):
|
|
125
|
+
return {}
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def save_config(**kw) -> None:
|
|
129
|
+
cfg = load_config() | kw
|
|
130
|
+
os.makedirs(os.path.dirname(CONFIG_FILE), exist_ok=True)
|
|
131
|
+
with open(CONFIG_FILE, "w") as f:
|
|
132
|
+
json.dump(cfg, f, indent=1, ensure_ascii=False)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def fmt_bytes(n: float) -> str:
|
|
136
|
+
for unit in ("B", "KB", "MB", "GB", "TB"):
|
|
137
|
+
if n < 1000 or unit == "TB":
|
|
138
|
+
return f"{n:.1f}{unit}" if unit != "B" else f"{int(n)}B"
|
|
139
|
+
n /= 1000
|
|
140
|
+
return str(n)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def fmt_count(n: Optional[int]) -> str:
|
|
144
|
+
if n is None:
|
|
145
|
+
return ""
|
|
146
|
+
for unit, k in (("M", 1_000_000), ("k", 1_000)):
|
|
147
|
+
if n >= k:
|
|
148
|
+
return f"{n / k:.1f}{unit}"
|
|
149
|
+
return str(n)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def ago(t: Optional[float]) -> str:
|
|
153
|
+
if not t:
|
|
154
|
+
return ""
|
|
155
|
+
d = (time.time() - t) / 86400
|
|
156
|
+
if d < 1:
|
|
157
|
+
return "today"
|
|
158
|
+
if d < 30:
|
|
159
|
+
return f"{int(d)}d ago"
|
|
160
|
+
if d < 365:
|
|
161
|
+
return f"{int(d / 30)}mo ago"
|
|
162
|
+
return f"{int(d / 365)}y ago"
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def ago_dt(dt) -> str:
|
|
166
|
+
return ago(dt.timestamp()) if dt else ""
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def repo_url(repo_id: str, repo_type: str) -> str:
|
|
170
|
+
return f"{HUB}/{URL_PREFIX[repo_type]}{repo_id}"
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def matches(text: str, query: str) -> bool:
|
|
174
|
+
"""공백으로 나눈 단어가 모두 들어 있으면 맞는다 (대소문자 무시)."""
|
|
175
|
+
text = text.lower()
|
|
176
|
+
return all(w in text for w in query.lower().split())
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def hub_query_args(repo_type: str, f: dict) -> dict:
|
|
180
|
+
"""필터 값을 list_models / list_datasets / list_spaces 인자로 바꾼다."""
|
|
181
|
+
kw: dict = {"sort": f.get("sort") or "trending_score"}
|
|
182
|
+
tags = list(f.get("tags", "").split())
|
|
183
|
+
if f.get("library"):
|
|
184
|
+
tags.append(f["library"])
|
|
185
|
+
if f.get("license"):
|
|
186
|
+
tags.append(f"license:{f['license']}")
|
|
187
|
+
if f.get("access"):
|
|
188
|
+
kw["gated"] = f["access"] == "gated"
|
|
189
|
+
if repo_type == "model":
|
|
190
|
+
if f.get("language"):
|
|
191
|
+
tags.append(f["language"])
|
|
192
|
+
kw.update({k: f[name] for k, name in (("pipeline_tag", "task"), ("num_parameters", "params"),
|
|
193
|
+
("apps", "app")) if f.get(name)})
|
|
194
|
+
elif repo_type == "dataset":
|
|
195
|
+
kw.update({k: f[name] for k, name in (("task_categories", "task"), ("size_categories", "size"),
|
|
196
|
+
("language", "language")) if f.get(name)})
|
|
197
|
+
if tags:
|
|
198
|
+
kw["filter"] = tags
|
|
199
|
+
return kw
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def filter_summary(fields: list, f: dict) -> str:
|
|
203
|
+
labels, sort = [], ""
|
|
204
|
+
for key, _, choices in fields:
|
|
205
|
+
if key == "sort":
|
|
206
|
+
sort = dict((v, k) for k, v in choices).get(f.get("sort") or choices[0][1], "")
|
|
207
|
+
elif f.get(key):
|
|
208
|
+
labels.append(f[key] if choices is None else dict((v, k) for k, v in choices).get(f[key], f[key]))
|
|
209
|
+
return " · ".join(labels + [f"sorted by {sort.lower()}"])
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def parse_count(s: str) -> float:
|
|
213
|
+
units = {"K": 1e3, "M": 1e6, "B": 1e9, "T": 1e12}
|
|
214
|
+
return float(s[:-1]) * units[s[-1]] if s[-1:] in units else float(s)
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def meta_matches(repo_type: str, meta: Optional[dict], f: dict) -> bool:
|
|
218
|
+
"""받아 둔 리포의 허브 정보가 필터에 맞는지. 허브의 필터와 같은 기준으로 판단한다."""
|
|
219
|
+
if not any(v for k, v in f.items() if k != "sort"):
|
|
220
|
+
return True
|
|
221
|
+
if not meta or not meta.get("ok"):
|
|
222
|
+
return False # 정보가 없으면 판단할 수 없으므로 뺀다 (화면에 개수를 따로 알린다)
|
|
223
|
+
tags = set(meta.get("tags") or [])
|
|
224
|
+
prefix = "" if repo_type == "model" else "language:"
|
|
225
|
+
checks = [
|
|
226
|
+
("license", lambda v: f"license:{v}" in tags),
|
|
227
|
+
("access", lambda v: bool(meta.get("gated")) == (v == "gated")),
|
|
228
|
+
("tags", lambda v: all(t in tags for t in v.split())),
|
|
229
|
+
("language", lambda v: f"{prefix}{v}" in tags),
|
|
230
|
+
]
|
|
231
|
+
if repo_type == "model":
|
|
232
|
+
checks += [
|
|
233
|
+
("task", lambda v: meta.get("task") == v),
|
|
234
|
+
("library", lambda v: v in tags or meta.get("library") == v),
|
|
235
|
+
("params", lambda v: meta.get("params") is not None and all(
|
|
236
|
+
(meta["params"] >= parse_count(x[4:])) if x.startswith("min:") else (meta["params"] <= parse_count(x[4:]))
|
|
237
|
+
for x in v.split(","))),
|
|
238
|
+
]
|
|
239
|
+
else:
|
|
240
|
+
checks += [
|
|
241
|
+
("task", lambda v: f"task_categories:{v}" in tags),
|
|
242
|
+
("size", lambda v: f"size_categories:{v}" in tags),
|
|
243
|
+
]
|
|
244
|
+
return all(test(f[key]) for key, test in checks if f.get(key))
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def load_meta() -> dict:
|
|
248
|
+
try:
|
|
249
|
+
with open(META_FILE) as fh:
|
|
250
|
+
return json.load(fh)
|
|
251
|
+
except (OSError, ValueError):
|
|
252
|
+
return {}
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def save_meta(meta: dict) -> None:
|
|
256
|
+
try:
|
|
257
|
+
os.makedirs(os.path.dirname(META_FILE), exist_ok=True)
|
|
258
|
+
with open(META_FILE, "w") as fh:
|
|
259
|
+
json.dump(meta, fh)
|
|
260
|
+
except OSError:
|
|
261
|
+
pass
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def fmt_params(n: Optional[int]) -> str:
|
|
265
|
+
if not n:
|
|
266
|
+
return ""
|
|
267
|
+
for unit, k in (("T", 10**12), ("B", 10**9), ("M", 10**6), ("K", 10**3)):
|
|
268
|
+
if n >= k:
|
|
269
|
+
return f"{n / k:.1f}{unit}".replace(".0" + unit, unit)
|
|
270
|
+
return str(n)
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def split_patterns(s: str) -> Optional[list[str]]:
|
|
274
|
+
return s.split() or None
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
# ---------------------------------------------------------------- 공용 모달
|
|
278
|
+
|
|
279
|
+
class Confirm(ModalScreen[bool]):
|
|
280
|
+
"""예/아니오. require 를 주면 그 이름을 그대로 입력해야 확인된다 (되돌릴 수 없는 삭제)."""
|
|
281
|
+
|
|
282
|
+
BINDINGS = [Binding("escape", "dismiss(False)", "Cancel"), Binding("y", "yes", "Yes")]
|
|
283
|
+
|
|
284
|
+
def __init__(self, message: str, detail: str = "", require: str = "") -> None:
|
|
285
|
+
super().__init__()
|
|
286
|
+
self.message, self.detail, self.require = message, detail, require
|
|
287
|
+
|
|
288
|
+
def compose(self) -> ComposeResult:
|
|
289
|
+
with Vertical(id="dialog"):
|
|
290
|
+
yield Static(Text(self.message, style="bold"))
|
|
291
|
+
if self.detail:
|
|
292
|
+
with VerticalScroll(id="confirm-detail"):
|
|
293
|
+
yield Static(Text(self.detail))
|
|
294
|
+
if self.require:
|
|
295
|
+
yield Label(Text.assemble("Type ", (self.require, "bold"), " to confirm"), id="require-label")
|
|
296
|
+
yield Input(id="require")
|
|
297
|
+
with Horizontal(classes="buttons"):
|
|
298
|
+
yield Button("Delete" if self.require else "Yes (y)", variant="error", id="yes",
|
|
299
|
+
disabled=bool(self.require))
|
|
300
|
+
yield Button("Cancel (esc)", id="no")
|
|
301
|
+
|
|
302
|
+
def on_mount(self) -> None:
|
|
303
|
+
if self.require:
|
|
304
|
+
self.query_one("#require").focus()
|
|
305
|
+
|
|
306
|
+
def action_yes(self) -> None:
|
|
307
|
+
if not self.require:
|
|
308
|
+
self.dismiss(True)
|
|
309
|
+
|
|
310
|
+
@on(Input.Changed, "#require")
|
|
311
|
+
def typed(self, e: Input.Changed) -> None:
|
|
312
|
+
self.query_one("#yes", Button).disabled = e.value != self.require
|
|
313
|
+
|
|
314
|
+
@on(Input.Submitted, "#require")
|
|
315
|
+
def submitted(self, e: Input.Submitted) -> None:
|
|
316
|
+
if e.value == self.require:
|
|
317
|
+
self.dismiss(True)
|
|
318
|
+
|
|
319
|
+
@on(Button.Pressed)
|
|
320
|
+
def pressed(self, e: Button.Pressed) -> None:
|
|
321
|
+
self.dismiss(e.button.id == "yes")
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
class TaskLog(ModalScreen[None]):
|
|
325
|
+
"""오래 걸리는 작업의 진행 로그. background=True 면 끝나기 전에도 닫을 수 있고 작업은 계속된다."""
|
|
326
|
+
|
|
327
|
+
BINDINGS = [Binding("escape,q", "close", "Close")]
|
|
328
|
+
|
|
329
|
+
def __init__(self, title: str, background: bool = False) -> None:
|
|
330
|
+
super().__init__()
|
|
331
|
+
self.title_text = title
|
|
332
|
+
self.background = background
|
|
333
|
+
self.done = False
|
|
334
|
+
|
|
335
|
+
def compose(self) -> ComposeResult:
|
|
336
|
+
with Vertical(id="tasklog"):
|
|
337
|
+
yield Static(Text(self.title_text, style="bold"))
|
|
338
|
+
yield Static(id="progress")
|
|
339
|
+
yield RichLog(wrap=True, markup=False)
|
|
340
|
+
yield Button("Hide (esc), keeps running" if self.background else "Close (esc)", id="close",
|
|
341
|
+
disabled=not self.background)
|
|
342
|
+
|
|
343
|
+
def write(self, line: str, style: str = "") -> None:
|
|
344
|
+
self.query_one(RichLog).write(Text(line, style=style))
|
|
345
|
+
|
|
346
|
+
def progress(self, t: Text) -> None:
|
|
347
|
+
self.query_one("#progress", Static).update(t)
|
|
348
|
+
|
|
349
|
+
def finish(self) -> None:
|
|
350
|
+
self.done = True
|
|
351
|
+
b = self.query_one("#close", Button)
|
|
352
|
+
b.label, b.disabled = "Close (esc)", False
|
|
353
|
+
b.focus()
|
|
354
|
+
|
|
355
|
+
def action_close(self) -> None:
|
|
356
|
+
if self.done or self.background:
|
|
357
|
+
self.dismiss()
|
|
358
|
+
|
|
359
|
+
@on(Button.Pressed, "#close")
|
|
360
|
+
def close_pressed(self) -> None:
|
|
361
|
+
self.action_close()
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
# ---------------------------------------------------------------- 다운로드
|
|
365
|
+
|
|
366
|
+
class DownloadForm(ModalScreen[Optional[dict]]):
|
|
367
|
+
"""받을 파일(패턴), 리비전, 저장 위치를 고른다. 파일 목록과 크기는 고른 리비전 기준으로 보여준다.
|
|
368
|
+
실제로 받을 파일은 snapshot_download 가 같은 패턴으로 다시 정한다 (여기 목록은 미리보기)."""
|
|
369
|
+
|
|
370
|
+
BINDINGS = [Binding("ctrl+r", "submit", "Download"), Binding("escape", "dismiss(None)", "Cancel")]
|
|
371
|
+
|
|
372
|
+
def __init__(self, repo_id: str, repo_type: str, files: list, cache_dir: str) -> None:
|
|
373
|
+
super().__init__()
|
|
374
|
+
self.repo_id, self.repo_type, self.files, self.cache_dir = repo_id, repo_type, files, cache_dir
|
|
375
|
+
self.loaded_rev = "main" # self.files 가 어느 리비전의 목록인지
|
|
376
|
+
self.load_error = ""
|
|
377
|
+
|
|
378
|
+
def compose(self) -> ComposeResult:
|
|
379
|
+
with Vertical(id="dialog", classes="wide"):
|
|
380
|
+
yield Static(Text(f"Download {self.repo_type}: {self.repo_id}", style="bold"))
|
|
381
|
+
yield Label("Include (space-separated patterns, empty = all files)")
|
|
382
|
+
yield Input(placeholder="e.g. *.safetensors *.json tokenizer*", id="dl-include")
|
|
383
|
+
yield Label("Exclude")
|
|
384
|
+
yield Input(placeholder="e.g. *.bin *.pth original/*", id="dl-exclude")
|
|
385
|
+
yield Label("Revision (branch, tag or commit; Enter loads its file list)")
|
|
386
|
+
yield Input("main", id="dl-revision")
|
|
387
|
+
yield Label("Save to")
|
|
388
|
+
yield Select([(f"HF cache ({self.cache_dir})", "cache"), ("A local folder (plain files)", "local")],
|
|
389
|
+
value="cache", allow_blank=False, id="dl-dest")
|
|
390
|
+
yield Input(placeholder="Folder, e.g. ./models/" + self.repo_id.split("/")[-1], id="dl-local",
|
|
391
|
+
classes="hidden")
|
|
392
|
+
yield Static(id="dl-summary")
|
|
393
|
+
with Horizontal(classes="buttons"):
|
|
394
|
+
yield Button("Download (ctrl+r)", variant="primary", id="ok")
|
|
395
|
+
yield Button("Cancel (esc)", id="no")
|
|
396
|
+
|
|
397
|
+
def on_mount(self) -> None:
|
|
398
|
+
self.update_summary()
|
|
399
|
+
self.query_one("#dl-include").focus()
|
|
400
|
+
|
|
401
|
+
def revision(self) -> str:
|
|
402
|
+
return self.query_one("#dl-revision", Input).value.strip() or "main"
|
|
403
|
+
|
|
404
|
+
def patterns(self) -> tuple[Optional[list[str]], Optional[list[str]]]:
|
|
405
|
+
return (split_patterns(self.query_one("#dl-include", Input).value),
|
|
406
|
+
split_patterns(self.query_one("#dl-exclude", Input).value))
|
|
407
|
+
|
|
408
|
+
def selected_files(self) -> list:
|
|
409
|
+
allow, ignore = self.patterns()
|
|
410
|
+
return list(filter_repo_objects(self.files, key=lambda f: f.rfilename,
|
|
411
|
+
allow_patterns=allow, ignore_patterns=ignore))
|
|
412
|
+
|
|
413
|
+
@on(Input.Submitted, "#dl-revision")
|
|
414
|
+
def revision_entered(self) -> None:
|
|
415
|
+
if self.revision() != self.loaded_rev:
|
|
416
|
+
self.load_revision(self.revision())
|
|
417
|
+
|
|
418
|
+
@work(thread=True, exclusive=True, group="dl-files")
|
|
419
|
+
def load_revision(self, rev: str) -> None:
|
|
420
|
+
self.app.call_from_thread(self.query_one("#dl-summary", Static).update,
|
|
421
|
+
Text(f"Loading the file list of {rev}…", style="dim"))
|
|
422
|
+
try:
|
|
423
|
+
info = self.app.api.repo_info(self.repo_id, repo_type=self.repo_type, revision=rev,
|
|
424
|
+
files_metadata=True)
|
|
425
|
+
files, error = info.siblings or [], ""
|
|
426
|
+
except (HfHubHTTPError, OSError, ValueError) as e:
|
|
427
|
+
files, error = [], str(e).splitlines()[0]
|
|
428
|
+
self.app.call_from_thread(self.revision_loaded, rev, files, error)
|
|
429
|
+
|
|
430
|
+
def revision_loaded(self, rev: str, files: list, error: str) -> None:
|
|
431
|
+
if rev != self.revision():
|
|
432
|
+
return # 그사이 다른 리비전을 입력했다
|
|
433
|
+
self.load_error = error
|
|
434
|
+
if not error:
|
|
435
|
+
self.files, self.loaded_rev = files, rev
|
|
436
|
+
self.update_summary()
|
|
437
|
+
|
|
438
|
+
@on(Input.Changed)
|
|
439
|
+
@on(Select.Changed)
|
|
440
|
+
def update_summary(self) -> None:
|
|
441
|
+
local = self.query_one("#dl-dest", Select).value == "local"
|
|
442
|
+
self.query_one("#dl-local").set_class(not local, "hidden")
|
|
443
|
+
ok = self.query_one("#ok", Button)
|
|
444
|
+
rev = self.revision()
|
|
445
|
+
if rev != self.loaded_rev:
|
|
446
|
+
ok.disabled = True
|
|
447
|
+
msg = (f"Revision {rev} not found: {self.load_error}" if self.load_error
|
|
448
|
+
else f"Press Enter in Revision to load the file list of {rev}.")
|
|
449
|
+
self.query_one("#dl-summary", Static).update(Text(msg, style="red" if self.load_error else "yellow"))
|
|
450
|
+
return
|
|
451
|
+
ok.disabled = False
|
|
452
|
+
files = self.selected_files()
|
|
453
|
+
total = sum(f.size or 0 for f in files)
|
|
454
|
+
t = Text()
|
|
455
|
+
t.append(f"{len(files)} of {len(self.files)} files · {fmt_bytes(total)}", style="bold")
|
|
456
|
+
t.append(f" ({self.loaded_rev})", style="dim")
|
|
457
|
+
big = sorted(files, key=lambda f: -(f.size or 0))[:6]
|
|
458
|
+
for f in big:
|
|
459
|
+
t.append(f"\n {fmt_bytes(f.size or 0):>9} {f.rfilename}", style="dim")
|
|
460
|
+
if len(files) > len(big):
|
|
461
|
+
t.append(f"\n … {len(files) - len(big)} more", style="dim")
|
|
462
|
+
if local:
|
|
463
|
+
free = shutil.disk_usage(self.target_parent()).free
|
|
464
|
+
else:
|
|
465
|
+
free = shutil.disk_usage(self.cache_dir if os.path.isdir(self.cache_dir) else "/").free
|
|
466
|
+
t.append(f"\nFree space there: {fmt_bytes(free)}", style="red" if free < total else "")
|
|
467
|
+
self.query_one("#dl-summary", Static).update(t)
|
|
468
|
+
|
|
469
|
+
def target_parent(self) -> str:
|
|
470
|
+
p = Path(self.query_one("#dl-local", Input).value.strip() or ".").expanduser().resolve()
|
|
471
|
+
while not p.exists():
|
|
472
|
+
p = p.parent
|
|
473
|
+
return str(p)
|
|
474
|
+
|
|
475
|
+
@on(Button.Pressed, "#ok")
|
|
476
|
+
def action_submit(self) -> None:
|
|
477
|
+
if self.revision() != self.loaded_rev:
|
|
478
|
+
self.load_revision(self.revision())
|
|
479
|
+
return
|
|
480
|
+
files = self.selected_files()
|
|
481
|
+
if not files:
|
|
482
|
+
self.notify("No files match the patterns.", severity="warning")
|
|
483
|
+
return
|
|
484
|
+
local = None
|
|
485
|
+
if self.query_one("#dl-dest", Select).value == "local":
|
|
486
|
+
local = self.query_one("#dl-local", Input).value.strip()
|
|
487
|
+
if not local:
|
|
488
|
+
self.notify("Enter a folder to save to.", severity="warning")
|
|
489
|
+
return
|
|
490
|
+
local = str(Path(local).expanduser().resolve())
|
|
491
|
+
allow, ignore = self.patterns()
|
|
492
|
+
self.dismiss({"allow": allow, "ignore": ignore, "revision": self.loaded_rev, "local_dir": local,
|
|
493
|
+
"count": len(files), "total": sum(f.size or 0 for f in files)})
|
|
494
|
+
|
|
495
|
+
@on(Button.Pressed, "#no")
|
|
496
|
+
def cancel(self) -> None:
|
|
497
|
+
self.dismiss(None)
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
def progress_bars(bars: dict) -> type:
|
|
501
|
+
"""snapshot_download 의 tqdm_class 로 넘긴다. 화면에 그리지 않고, 파일 수/바이트 막대를 bars 에 모은다."""
|
|
502
|
+
|
|
503
|
+
class Bar(tqdm):
|
|
504
|
+
monitor_interval = 0 # 화면에 안 그리므로 감시 스레드가 필요 없다
|
|
505
|
+
|
|
506
|
+
def __init__(self, *args, **kwargs):
|
|
507
|
+
kwargs["disable"] = False
|
|
508
|
+
kwargs["file"] = io.StringIO()
|
|
509
|
+
super().__init__(*args, **kwargs)
|
|
510
|
+
desc = kwargs.get("desc") or ""
|
|
511
|
+
if desc.startswith("Fetching"):
|
|
512
|
+
bars["files"] = self
|
|
513
|
+
elif desc.startswith("Reconstruct"):
|
|
514
|
+
bars["bytes"] = self # 디스크에 쓴 바이트
|
|
515
|
+
|
|
516
|
+
# tqdm 은 기본으로 multiprocessing 잠금을 만드는데, TUI 실행 중에는 그 보조 프로세스를 띄울 수 없다
|
|
517
|
+
Bar.set_lock(threading.RLock())
|
|
518
|
+
return Bar
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
# ---------------------------------------------------------------- 허브 필터
|
|
522
|
+
|
|
523
|
+
class HubFilterForm(ModalScreen[Optional[dict]]):
|
|
524
|
+
"""허깅페이스 웹사이트의 필터처럼 용도, 파라미터 수, 라이브러리 등을 고른다."""
|
|
525
|
+
|
|
526
|
+
BINDINGS = [Binding("ctrl+r", "apply", "Apply"), Binding("escape", "dismiss(None)", "Cancel")]
|
|
527
|
+
|
|
528
|
+
def __init__(self, title: str, fields: list, current: dict) -> None:
|
|
529
|
+
super().__init__()
|
|
530
|
+
self.title_text, self.fields, self.current = title, fields, current
|
|
531
|
+
|
|
532
|
+
def compose(self) -> ComposeResult:
|
|
533
|
+
with Vertical(id="dialog", classes="wide"):
|
|
534
|
+
yield Static(Text(self.title_text, style="bold"))
|
|
535
|
+
with Grid(id="hf-grid"):
|
|
536
|
+
for key, label, choices in self.fields:
|
|
537
|
+
with Vertical(classes="hf-field"):
|
|
538
|
+
yield Label(label)
|
|
539
|
+
if choices is None:
|
|
540
|
+
yield Input(self.current.get(key, ""), placeholder="e.g. 4-bit code", id=f"hf-{key}")
|
|
541
|
+
elif key == "sort":
|
|
542
|
+
yield Select(choices, value=self.current.get(key) or choices[0][1],
|
|
543
|
+
allow_blank=False, id=f"hf-{key}")
|
|
544
|
+
else:
|
|
545
|
+
yield Select(choices, value=self.current.get(key) or Select.NULL,
|
|
546
|
+
prompt="Any", id=f"hf-{key}")
|
|
547
|
+
with Horizontal(classes="buttons"):
|
|
548
|
+
yield Button("Apply (ctrl+r)", variant="primary", id="ok")
|
|
549
|
+
yield Button("Clear all", id="clear")
|
|
550
|
+
yield Button("Cancel (esc)", id="no")
|
|
551
|
+
|
|
552
|
+
def values(self) -> dict:
|
|
553
|
+
out = {}
|
|
554
|
+
for key, _, choices in self.fields:
|
|
555
|
+
w = self.query_one(f"#hf-{key}")
|
|
556
|
+
value = w.value.strip() if isinstance(w, Input) else w.value
|
|
557
|
+
if value and value is not Select.NULL:
|
|
558
|
+
out[key] = value
|
|
559
|
+
return out
|
|
560
|
+
|
|
561
|
+
@on(Input.Submitted)
|
|
562
|
+
@on(Button.Pressed, "#ok")
|
|
563
|
+
def action_apply(self) -> None:
|
|
564
|
+
self.dismiss(self.values())
|
|
565
|
+
|
|
566
|
+
@on(Button.Pressed, "#clear")
|
|
567
|
+
def clear(self) -> None:
|
|
568
|
+
self.dismiss({})
|
|
569
|
+
|
|
570
|
+
@on(Button.Pressed, "#no")
|
|
571
|
+
def cancel(self) -> None:
|
|
572
|
+
self.dismiss(None)
|
|
573
|
+
|
|
574
|
+
|
|
575
|
+
# ---------------------------------------------------------------- 새 저장소
|
|
576
|
+
|
|
577
|
+
class NewRepoForm(ModalScreen[Optional[dict]]):
|
|
578
|
+
BINDINGS = [Binding("ctrl+r", "submit", "Create"), Binding("escape", "dismiss(None)", "Cancel")]
|
|
579
|
+
|
|
580
|
+
def __init__(self, owners: list[str]) -> None:
|
|
581
|
+
super().__init__()
|
|
582
|
+
self.owners = owners
|
|
583
|
+
|
|
584
|
+
def compose(self) -> ComposeResult:
|
|
585
|
+
with Vertical(id="dialog"):
|
|
586
|
+
yield Static(Text("New repository", style="bold"))
|
|
587
|
+
with Horizontal(classes="form-row"):
|
|
588
|
+
yield Select([(o, o) for o in self.owners], value=self.owners[0], allow_blank=False, id="nr-owner")
|
|
589
|
+
yield Input(placeholder="name", id="nr-name")
|
|
590
|
+
yield Select([(label[:-1], v) for label, v in REPO_TYPES], value="model", allow_blank=False,
|
|
591
|
+
id="nr-type")
|
|
592
|
+
yield Checkbox("Private", True, id="nr-private")
|
|
593
|
+
with Horizontal(classes="buttons"):
|
|
594
|
+
yield Button("Create (ctrl+r)", variant="primary", id="ok")
|
|
595
|
+
yield Button("Cancel (esc)", id="no")
|
|
596
|
+
|
|
597
|
+
@on(Input.Submitted)
|
|
598
|
+
@on(Button.Pressed, "#ok")
|
|
599
|
+
def action_submit(self) -> None:
|
|
600
|
+
name = self.query_one("#nr-name", Input).value.strip()
|
|
601
|
+
if not name:
|
|
602
|
+
self.notify("Enter a name.", severity="warning")
|
|
603
|
+
return
|
|
604
|
+
self.dismiss({"repo_id": f"{self.query_one('#nr-owner', Select).value}/{name}",
|
|
605
|
+
"repo_type": self.query_one("#nr-type", Select).value,
|
|
606
|
+
"private": self.query_one("#nr-private", Checkbox).value})
|
|
607
|
+
|
|
608
|
+
@on(Button.Pressed, "#no")
|
|
609
|
+
def cancel(self) -> None:
|
|
610
|
+
self.dismiss(None)
|
|
611
|
+
|
|
612
|
+
|
|
613
|
+
# ---------------------------------------------------------------- 설정
|
|
614
|
+
|
|
615
|
+
class Page(VerticalScroll):
|
|
616
|
+
"""탭 하나를 채우는 스크롤 영역. 좌우 화살표는 가로 스크롤 대신 탭 이동에 쓴다."""
|
|
617
|
+
|
|
618
|
+
BINDINGS = [Binding("left", "app.prev_tab", show=False), Binding("right", "app.next_tab", show=False)]
|
|
619
|
+
|
|
620
|
+
|
|
621
|
+
class SettingsPane(Page):
|
|
622
|
+
"""계정, 캐시 폴더. 바꾸면 바로 ~/.config/thf/config.json 에 저장."""
|
|
623
|
+
|
|
624
|
+
def __init__(self) -> None:
|
|
625
|
+
super().__init__(id="settings")
|
|
626
|
+
self.user: Optional[dict] = None
|
|
627
|
+
|
|
628
|
+
def compose(self) -> ComposeResult:
|
|
629
|
+
yield Label("Hugging Face account", classes="section")
|
|
630
|
+
yield Static(id="acct-status")
|
|
631
|
+
with Horizontal(classes="buttons", id="acct-buttons"):
|
|
632
|
+
yield Button("Log in", variant="primary", id="acct-login")
|
|
633
|
+
yield Button("Switch token", id="acct-switch")
|
|
634
|
+
yield Button("Log out", id="acct-logout")
|
|
635
|
+
yield Static(Text("Runs hf auth login / switch / logout in this terminal, then comes back.", style="dim"))
|
|
636
|
+
|
|
637
|
+
yield Label("Cache folder (empty = HF_HUB_CACHE)", classes="section")
|
|
638
|
+
with Horizontal(classes="form-row"):
|
|
639
|
+
yield Input(load_config().get("cache_dir", ""), placeholder=constants.HF_HUB_CACHE, id="set-cache")
|
|
640
|
+
yield Button("Apply", id="set-cache-apply")
|
|
641
|
+
|
|
642
|
+
yield Static(Text(f"Saved to {CONFIG_FILE}", style="dim"), id="settings-file")
|
|
643
|
+
|
|
644
|
+
def on_mount(self) -> None:
|
|
645
|
+
self.show_account(None)
|
|
646
|
+
|
|
647
|
+
# ------------------------------------------------ 계정
|
|
648
|
+
|
|
649
|
+
def show_account(self, user: Optional[dict]) -> None:
|
|
650
|
+
"""로그인 상태를 보여준다. 앱이 계정 정보를 다시 읽을 때마다 부른다."""
|
|
651
|
+
self.user = user
|
|
652
|
+
t = Text()
|
|
653
|
+
env_token = bool(os.environ.get("HF_TOKEN"))
|
|
654
|
+
if user:
|
|
655
|
+
t.append("Logged in as ")
|
|
656
|
+
t.append(user["name"], style="bold")
|
|
657
|
+
token = (user.get("auth") or {}).get("accessToken") or {}
|
|
658
|
+
if token:
|
|
659
|
+
t.append(f" · token {token.get('displayName', '')} ({token.get('role', '')})", style="dim")
|
|
660
|
+
if env_token:
|
|
661
|
+
t.append("\nThe token comes from the HF_TOKEN environment variable; unset it to log out.",
|
|
662
|
+
style="dim")
|
|
663
|
+
elif get_token():
|
|
664
|
+
t.append("A token is saved but could not be used (invalid, expired or offline).", style="yellow")
|
|
665
|
+
else:
|
|
666
|
+
t.append("Not logged in. Needed for your own repos, private and gated models.", style="dim")
|
|
667
|
+
self.query_one("#acct-status", Static).update(t)
|
|
668
|
+
has_token = bool(get_token())
|
|
669
|
+
login = self.query_one("#acct-login", Button)
|
|
670
|
+
login.label = "Log in again" if has_token else "Log in"
|
|
671
|
+
self.query_one("#acct-switch").set_class(not has_token, "hidden")
|
|
672
|
+
self.query_one("#acct-logout").set_class(not has_token, "hidden")
|
|
673
|
+
|
|
674
|
+
@on(Button.Pressed, "#acct-login")
|
|
675
|
+
def do_login(self) -> None:
|
|
676
|
+
# 이미 로그인돼 있으면 hf auth login 이 그냥 끝나므로 --force 로 다시 로그인하게 한다
|
|
677
|
+
self.app.run_hf("auth", "login", *(["--force"] if get_token() else []))
|
|
678
|
+
|
|
679
|
+
@on(Button.Pressed, "#acct-switch")
|
|
680
|
+
def do_switch(self) -> None:
|
|
681
|
+
self.app.run_hf("auth", "switch")
|
|
682
|
+
|
|
683
|
+
@on(Button.Pressed, "#acct-logout")
|
|
684
|
+
def do_logout(self) -> None:
|
|
685
|
+
self.app.run_hf("auth", "logout")
|
|
686
|
+
|
|
687
|
+
# ------------------------------------------------ 캐시 폴더
|
|
688
|
+
|
|
689
|
+
@on(Input.Submitted, "#set-cache")
|
|
690
|
+
@on(Button.Pressed, "#set-cache-apply")
|
|
691
|
+
def apply_cache_dir(self) -> None:
|
|
692
|
+
cache = self.query_one("#set-cache", Input).value.strip()
|
|
693
|
+
if cache and not os.path.isdir(os.path.expanduser(cache)):
|
|
694
|
+
self.notify(f"Folder not found: {cache}", severity="warning")
|
|
695
|
+
return
|
|
696
|
+
self.app.set_cache_dir(cache)
|
|
697
|
+
|
|
698
|
+
# ---------------------------------------------------------------- 탭별 표
|
|
699
|
+
|
|
700
|
+
class CacheTable(DataTable):
|
|
701
|
+
"""Models / Datasets / Spaces 탭이 같이 쓰는 캐시 목록."""
|
|
702
|
+
|
|
703
|
+
BINDINGS = [
|
|
704
|
+
Binding("left", "app.prev_tab", show=False),
|
|
705
|
+
Binding("right", "app.next_tab", show=False),
|
|
706
|
+
Binding("slash", "app.focus_filter", "Filter"),
|
|
707
|
+
Binding("f", "app.local_filters", "Filters"),
|
|
708
|
+
Binding("p", "app.update_cached", "Pull latest"),
|
|
709
|
+
Binding("x", "app.delete_old_revisions", "Remove old revisions"),
|
|
710
|
+
Binding("d", "app.delete_cached", "Delete"),
|
|
711
|
+
]
|
|
712
|
+
|
|
713
|
+
|
|
714
|
+
class HubTable(DataTable):
|
|
715
|
+
BINDINGS = [
|
|
716
|
+
Binding("p", "app.download", "Download"),
|
|
717
|
+
Binding("slash", "app.focus_search", "Search"),
|
|
718
|
+
Binding("f", "app.hub_filters", "Filters"),
|
|
719
|
+
Binding("left", "app.prev_tab", show=False),
|
|
720
|
+
Binding("right", "app.next_tab", show=False),
|
|
721
|
+
]
|
|
722
|
+
|
|
723
|
+
|
|
724
|
+
class RepoTable(DataTable):
|
|
725
|
+
BINDINGS = [
|
|
726
|
+
Binding("left", "app.prev_tab", show=False),
|
|
727
|
+
Binding("right", "app.next_tab", show=False),
|
|
728
|
+
Binding("slash", "app.focus_filter", "Filter"),
|
|
729
|
+
Binding("n", "app.new_repo", "New"),
|
|
730
|
+
Binding("p", "app.download_mine", "Download"),
|
|
731
|
+
Binding("v", "app.toggle_visibility", "Public/Private"),
|
|
732
|
+
Binding("d", "app.delete_repo", "Delete"),
|
|
733
|
+
]
|
|
734
|
+
|
|
735
|
+
|
|
736
|
+
# ---------------------------------------------------------------- 앱
|
|
737
|
+
|
|
738
|
+
class HfTUI(App):
|
|
739
|
+
TITLE = "thf"
|
|
740
|
+
CSS = """
|
|
741
|
+
.body { height: 1fr; }
|
|
742
|
+
CacheTable, HubTable, RepoTable { height: 1fr; }
|
|
743
|
+
.list { width: 3fr; }
|
|
744
|
+
.filterbar { height: auto; }
|
|
745
|
+
.filter { width: 1fr; }
|
|
746
|
+
.count { width: auto; padding: 1 1 0 1; color: $text-muted; }
|
|
747
|
+
.detail { width: 2fr; border-left: solid $primary; padding: 0 1; }
|
|
748
|
+
#hub-left { width: 3fr; }
|
|
749
|
+
.searchbar { height: auto; }
|
|
750
|
+
#hub-type { width: 16; }
|
|
751
|
+
#hub-query { width: 1fr; }
|
|
752
|
+
#cleanup { padding: 1 2; }
|
|
753
|
+
#cleanup-buttons { height: auto; padding: 0 2; }
|
|
754
|
+
#cleanup-buttons Button { margin: 1 1 0 0; }
|
|
755
|
+
Confirm, TaskLog, DownloadForm, NewRepoForm, HubFilterForm { align: center middle; }
|
|
756
|
+
#hf-grid { grid-size: 3; grid-gutter: 0 2; height: auto; }
|
|
757
|
+
.hf-field { height: auto; }
|
|
758
|
+
#hub-filters, .filter-summary { height: auto; padding: 0 1; color: $text-muted; }
|
|
759
|
+
#settings { padding: 0 2; }
|
|
760
|
+
#settings .section { margin-top: 1; color: $accent; }
|
|
761
|
+
#settings .form-row { width: 100; }
|
|
762
|
+
#settings .form-row Button { margin-left: 1; }
|
|
763
|
+
#settings-file { margin-top: 1; }
|
|
764
|
+
#dialog { width: 80; height: auto; max-height: 90%; border: thick $primary; background: $surface; padding: 1 2; }
|
|
765
|
+
#dialog.wide { width: 100; }
|
|
766
|
+
#dialog Label { margin-top: 1; color: $text-muted; }
|
|
767
|
+
#confirm-detail { height: auto; max-height: 20; margin-top: 1; }
|
|
768
|
+
#dl-summary { margin-top: 1; height: auto; }
|
|
769
|
+
.form-row { height: auto; }
|
|
770
|
+
.form-row Select { width: 30; }
|
|
771
|
+
.form-row Input { width: 1fr; }
|
|
772
|
+
.hidden { display: none; }
|
|
773
|
+
.buttons { height: auto; margin-top: 1; }
|
|
774
|
+
.buttons Button { margin-right: 2; }
|
|
775
|
+
#tasklog { width: 90%; height: 85%; border: thick $primary; background: $surface; padding: 1 2; }
|
|
776
|
+
#tasklog #progress { height: auto; margin: 1 0; }
|
|
777
|
+
#tasklog RichLog { height: 1fr; border: round $panel; }
|
|
778
|
+
"""
|
|
779
|
+
BINDINGS = [
|
|
780
|
+
Binding("q", "quit", "Quit"),
|
|
781
|
+
Binding("r", "refresh", "Refresh"),
|
|
782
|
+
Binding("s", "settings", "Settings"),
|
|
783
|
+
Binding("left", "prev_tab", show=False),
|
|
784
|
+
Binding("right", "next_tab", show=False),
|
|
785
|
+
Binding("escape", "leave_input", show=False),
|
|
786
|
+
]
|
|
787
|
+
|
|
788
|
+
def __init__(self, cache_dir: str, cli_cache_dir: bool = False) -> None:
|
|
789
|
+
super().__init__()
|
|
790
|
+
self.api = HfApi()
|
|
791
|
+
self.cache_dir = cache_dir
|
|
792
|
+
self.cli_cache_dir = cli_cache_dir # --cache-dir 로 정했으면 설정의 캐시 폴더보다 우선
|
|
793
|
+
self.cache = None # HFCacheInfo
|
|
794
|
+
self.cached: dict[str, object] = {} # "type:id" → CachedRepoInfo
|
|
795
|
+
self.hub: dict[str, tuple[str, object]] = {} # 검색 결과
|
|
796
|
+
self.hub_info: dict[str, object] = {} # repo_info 캐시
|
|
797
|
+
self.hub_filters: dict[str, dict] = {rt: {} for rt in CACHE_TYPES} # 타입별 허브 필터
|
|
798
|
+
self.local_filters: dict[str, dict] = {rt: {} for rt in CACHE_TYPES} # 받아 둔 목록의 필터
|
|
799
|
+
self.meta: dict[str, dict] = load_meta() # "type:id" → 허브 정보
|
|
800
|
+
self.meta_loading = (0, 0) # (받은 수, 전체)
|
|
801
|
+
self.my_repos: dict[str, tuple[str, object]] = {}
|
|
802
|
+
self.list_rows: dict[str, list] = {} # 목록 이름 → (key, cells, 필터용 글자) 전체. 필터는 여기서 고른다
|
|
803
|
+
self.user: Optional[dict] = None
|
|
804
|
+
|
|
805
|
+
def compose(self) -> ComposeResult:
|
|
806
|
+
yield Header()
|
|
807
|
+
with TabbedContent(initial="tab-model", id="main-tabs"):
|
|
808
|
+
for label, rt in REPO_TYPES:
|
|
809
|
+
with TabPane(label, id=f"tab-{rt}"):
|
|
810
|
+
with Horizontal(classes="body"):
|
|
811
|
+
with Vertical(classes="list"):
|
|
812
|
+
with Horizontal(classes="filterbar"):
|
|
813
|
+
yield Input(placeholder="Filter by name (/)", id=f"filter-{rt}", classes="filter")
|
|
814
|
+
yield Static(id=f"count-{rt}", classes="count")
|
|
815
|
+
yield Static(id=f"lfilters-{rt}", classes="filter-summary")
|
|
816
|
+
yield CacheTable(id=f"ctable-{rt}", cursor_type="row", zebra_stripes=True)
|
|
817
|
+
with VerticalScroll(classes="detail"):
|
|
818
|
+
yield Static(id=f"cache-detail-{rt}")
|
|
819
|
+
with TabPane("Hub", id="tab-hub"):
|
|
820
|
+
with Horizontal(classes="body"):
|
|
821
|
+
with Vertical(id="hub-left"):
|
|
822
|
+
with Horizontal(classes="searchbar"):
|
|
823
|
+
yield Select(REPO_TYPES, value="model", allow_blank=False, id="hub-type")
|
|
824
|
+
yield Input(placeholder="Search and press Enter (empty = everything that matches the filters)",
|
|
825
|
+
id="hub-query")
|
|
826
|
+
yield Static(id="hub-filters")
|
|
827
|
+
yield HubTable(id="htable", cursor_type="row", zebra_stripes=True)
|
|
828
|
+
with VerticalScroll(classes="detail"):
|
|
829
|
+
yield Static(id="hub-detail")
|
|
830
|
+
with TabPane("My repos", id="tab-mine"):
|
|
831
|
+
with Horizontal(classes="body"):
|
|
832
|
+
with Vertical(classes="list"):
|
|
833
|
+
with Horizontal(classes="filterbar"):
|
|
834
|
+
yield Input(placeholder="Filter by name, type, public/private (/)", id="filter-mine",
|
|
835
|
+
classes="filter")
|
|
836
|
+
yield Static(id="count-mine", classes="count")
|
|
837
|
+
yield RepoTable(id="rtable", cursor_type="row", zebra_stripes=True)
|
|
838
|
+
with VerticalScroll(classes="detail"):
|
|
839
|
+
yield Static(id="repo-detail")
|
|
840
|
+
with TabPane("Cleanup", id="tab-cleanup"):
|
|
841
|
+
with Page():
|
|
842
|
+
yield Static(id="cleanup")
|
|
843
|
+
with Horizontal(id="cleanup-buttons"):
|
|
844
|
+
yield Button("Remove old revisions", id="prune-revisions")
|
|
845
|
+
yield Button("Remove incomplete downloads", id="prune-incomplete")
|
|
846
|
+
yield Button("Remove orphaned shared files", id="prune-shared")
|
|
847
|
+
with TabPane("Settings", id="tab-settings"):
|
|
848
|
+
yield SettingsPane()
|
|
849
|
+
yield Footer()
|
|
850
|
+
|
|
851
|
+
def on_mount(self) -> None:
|
|
852
|
+
for rt in CACHE_TYPES:
|
|
853
|
+
self.query_one(f"#ctable-{rt}", DataTable).add_columns(
|
|
854
|
+
*(["Repo", "Task", "Params"] if rt == "model" else ["Repo"]), "Size", "Revisions", "Last used")
|
|
855
|
+
self.show_hub_filters()
|
|
856
|
+
self.query_one("#rtable", DataTable).add_columns("Repo", "Type", "Visibility", "Downloads", "Likes",
|
|
857
|
+
"Updated")
|
|
858
|
+
self.action_refresh_all()
|
|
859
|
+
self.query_one("#ctable-model").focus()
|
|
860
|
+
|
|
861
|
+
@on(TabbedContent.TabActivated)
|
|
862
|
+
def tab_switched(self, e: TabbedContent.TabActivated) -> None:
|
|
863
|
+
# 아래 키 안내는 포커스된 위젯 기준이라, 탭을 바꾸면 그 탭의 목록으로 포커스를 옮긴다
|
|
864
|
+
if e.tabbed_content.id != "main-tabs":
|
|
865
|
+
return
|
|
866
|
+
if e.pane.id == "tab-hub" and not self.hub:
|
|
867
|
+
self.search("", "model")
|
|
868
|
+
self.focus_tab(e.pane.id or "")
|
|
869
|
+
|
|
870
|
+
def focus_tab(self, tab: str) -> None:
|
|
871
|
+
target = {**{f"tab-{rt}": f"#ctable-{rt}" for rt in CACHE_TYPES}, "tab-hub": "#htable",
|
|
872
|
+
"tab-mine": "#rtable", "tab-cleanup": "#prune-revisions", "tab-settings": "#settings"}.get(tab)
|
|
873
|
+
if target:
|
|
874
|
+
self.query_one(target).focus()
|
|
875
|
+
|
|
876
|
+
def action_prev_tab(self) -> None:
|
|
877
|
+
self.move_tab(-1)
|
|
878
|
+
|
|
879
|
+
def action_next_tab(self) -> None:
|
|
880
|
+
self.move_tab(1)
|
|
881
|
+
|
|
882
|
+
def move_tab(self, step: int) -> None:
|
|
883
|
+
tabs = self.query_one("#main-tabs", TabbedContent)
|
|
884
|
+
ids = [p.id for p in tabs.query(TabPane) if p.parent.parent is tabs]
|
|
885
|
+
i = ids.index(tabs.active) + step
|
|
886
|
+
if 0 <= i < len(ids):
|
|
887
|
+
tabs.active = ids[i]
|
|
888
|
+
|
|
889
|
+
def action_leave_input(self) -> None:
|
|
890
|
+
"""입력칸에서 Esc: 그 탭의 목록으로 돌아가 단축키와 화살표를 다시 쓸 수 있게 한다."""
|
|
891
|
+
self.focus_tab(self.query_one("#main-tabs", TabbedContent).active)
|
|
892
|
+
|
|
893
|
+
# ------------------------------------------------ 필터
|
|
894
|
+
|
|
895
|
+
LIST_TABLES = {**{rt: f"ctable-{rt}" for rt in CACHE_TYPES}, "mine": "rtable"}
|
|
896
|
+
|
|
897
|
+
def cache_rows(self, rt: str) -> list:
|
|
898
|
+
"""받아 둔 rt 타입 리포를 정렬하고, 허브 정보 필터에 맞는 것만 (key, cells, 이름) 으로."""
|
|
899
|
+
f = self.local_filters[rt]
|
|
900
|
+
repos = [(k, r) for k, r in self.cached.items() if r.repo_type == rt]
|
|
901
|
+
order = {"size": lambda kr: -kr[1].size_on_disk, "used": lambda kr: -kr[1].last_accessed,
|
|
902
|
+
"unused": lambda kr: kr[1].last_accessed, "name": lambda kr: kr[1].repo_id.lower()}
|
|
903
|
+
repos.sort(key=order[f.get("sort") or "size"])
|
|
904
|
+
rows = []
|
|
905
|
+
for key, r in repos:
|
|
906
|
+
meta = self.meta.get(key)
|
|
907
|
+
if not meta_matches(rt, meta, f):
|
|
908
|
+
continue
|
|
909
|
+
ok = meta and meta.get("ok")
|
|
910
|
+
extra = [ok and meta.get("task") or "", fmt_params(ok and meta.get("params"))] if rt == "model" else []
|
|
911
|
+
rows.append((key, [r.repo_id, *extra, fmt_bytes(r.size_on_disk), len(r.revisions),
|
|
912
|
+
ago(r.last_accessed)], r.repo_id))
|
|
913
|
+
return rows
|
|
914
|
+
|
|
915
|
+
def fill_list(self, name: str) -> None:
|
|
916
|
+
"""name(model/dataset/space/mine) 목록을 필터 칸의 내용대로 다시 채운다."""
|
|
917
|
+
if name in CACHE_TYPES:
|
|
918
|
+
self.list_rows[name] = self.cache_rows(name)
|
|
919
|
+
self.show_local_filters(name)
|
|
920
|
+
rows = self.list_rows.get(name, [])
|
|
921
|
+
query = self.query_one(f"#filter-{name}", Input).value
|
|
922
|
+
shown = [(key, cells) for key, cells, text in rows if matches(text, query)]
|
|
923
|
+
self.refill(self.query_one(f"#{self.LIST_TABLES[name]}", DataTable), shown)
|
|
924
|
+
total = sum(1 for r in self.cached.values() if r.repo_type == name) if name in CACHE_TYPES else len(rows)
|
|
925
|
+
self.query_one(f"#count-{name}", Static).update(f"{len(shown)} of {total}" if len(shown) != total else "")
|
|
926
|
+
if name == "mine":
|
|
927
|
+
self.show_repo_detail()
|
|
928
|
+
else:
|
|
929
|
+
self.show_cache_detail(name)
|
|
930
|
+
|
|
931
|
+
def show_local_filters(self, rt: str) -> None:
|
|
932
|
+
f = self.local_filters[rt]
|
|
933
|
+
t = Text()
|
|
934
|
+
t.append("Filters: ", style="bold")
|
|
935
|
+
t.append(filter_summary(LOCAL_FILTERS[rt], f))
|
|
936
|
+
t.append(" (f to change)", style="dim")
|
|
937
|
+
done, total = self.meta_loading
|
|
938
|
+
if done < total:
|
|
939
|
+
t.append(f"\nLoading details from the Hub… {done}/{total}", style="yellow")
|
|
940
|
+
elif any(v for k, v in f.items() if k != "sort"):
|
|
941
|
+
missing = sum(1 for k, r in self.cached.items() if r.repo_type == rt
|
|
942
|
+
and not (self.meta.get(k) or {}).get("ok"))
|
|
943
|
+
if missing:
|
|
944
|
+
t.append(f"\n{missing} hidden: no details on the Hub (deleted, private or offline)", style="dim")
|
|
945
|
+
self.query_one(f"#lfilters-{rt}", Static).update(t)
|
|
946
|
+
|
|
947
|
+
def action_local_filters(self) -> None:
|
|
948
|
+
rt = self.cache_type()
|
|
949
|
+
|
|
950
|
+
def done(f: Optional[dict]) -> None:
|
|
951
|
+
if f is not None:
|
|
952
|
+
self.local_filters[rt] = f
|
|
953
|
+
self.fill_list(rt)
|
|
954
|
+
|
|
955
|
+
label = dict((v, k) for k, v in REPO_TYPES)[rt].lower()
|
|
956
|
+
self.push_screen(HubFilterForm(f"Filter downloaded {label} (details from the Hub)", LOCAL_FILTERS[rt],
|
|
957
|
+
self.local_filters[rt]), done)
|
|
958
|
+
|
|
959
|
+
@work(thread=True, exclusive=True, group="meta")
|
|
960
|
+
def fetch_meta(self) -> None:
|
|
961
|
+
"""받아 둔 리포의 허브 정보를 백그라운드에서 받는다. 저장해 둔 것이 새것이면 건너뛴다."""
|
|
962
|
+
now = time.time()
|
|
963
|
+
todo = [k for k in list(self.cached)
|
|
964
|
+
if now - (self.meta.get(k) or {}).get("t", 0) > (META_TTL if (self.meta.get(k) or {}).get("ok")
|
|
965
|
+
else META_RETRY)]
|
|
966
|
+
if not todo:
|
|
967
|
+
return
|
|
968
|
+
self.call_from_thread(self.meta_progress, 0, len(todo))
|
|
969
|
+
|
|
970
|
+
def get(key: str) -> dict:
|
|
971
|
+
rt, repo_id = key.split(":", 1)
|
|
972
|
+
try:
|
|
973
|
+
if rt == "model":
|
|
974
|
+
i = self.api.model_info(repo_id, expand=["pipeline_tag", "library_name", "tags", "safetensors",
|
|
975
|
+
"gated"])
|
|
976
|
+
return {"ok": True, "task": i.pipeline_tag, "library": i.library_name, "tags": i.tags or [],
|
|
977
|
+
"params": i.safetensors.total if i.safetensors else None, "gated": i.gated}
|
|
978
|
+
info = (self.api.dataset_info if rt == "dataset" else self.api.space_info)(
|
|
979
|
+
repo_id, expand=["tags", "gated"] if rt == "dataset" else ["tags"])
|
|
980
|
+
return {"ok": True, "tags": info.tags or [], "gated": getattr(info, "gated", False)}
|
|
981
|
+
except (HfHubHTTPError, OSError, ValueError):
|
|
982
|
+
return {"ok": False}
|
|
983
|
+
|
|
984
|
+
with ThreadPoolExecutor(max_workers=8) as pool:
|
|
985
|
+
jobs = {pool.submit(get, k): k for k in todo}
|
|
986
|
+
for n, job in enumerate(as_completed(jobs), 1):
|
|
987
|
+
self.meta[jobs[job]] = {**job.result(), "t": now}
|
|
988
|
+
self.call_from_thread(self.meta_progress, n, len(todo))
|
|
989
|
+
save_meta(self.meta)
|
|
990
|
+
|
|
991
|
+
def meta_progress(self, done: int, total: int) -> None:
|
|
992
|
+
self.meta_loading = (done, total)
|
|
993
|
+
if done == total or done % 5 == 0:
|
|
994
|
+
for rt in CACHE_TYPES:
|
|
995
|
+
self.fill_list(rt)
|
|
996
|
+
|
|
997
|
+
@on(Input.Changed, ".filter")
|
|
998
|
+
def filter_changed(self, e: Input.Changed) -> None:
|
|
999
|
+
self.fill_list(e.input.id.removeprefix("filter-"))
|
|
1000
|
+
|
|
1001
|
+
@on(Input.Submitted, ".filter")
|
|
1002
|
+
def filter_done(self, e: Input.Submitted) -> None:
|
|
1003
|
+
self.query_one(f"#{self.LIST_TABLES[e.input.id.removeprefix('filter-')]}").focus()
|
|
1004
|
+
|
|
1005
|
+
def action_focus_filter(self) -> None:
|
|
1006
|
+
tab = self.query_one("#main-tabs", TabbedContent).active.removeprefix("tab-")
|
|
1007
|
+
if tab in self.LIST_TABLES:
|
|
1008
|
+
self.query_one(f"#filter-{tab}").focus()
|
|
1009
|
+
|
|
1010
|
+
def action_focus_search(self) -> None:
|
|
1011
|
+
self.query_one("#hub-query").focus()
|
|
1012
|
+
|
|
1013
|
+
def check_action(self, action: str, parameters) -> Optional[bool]:
|
|
1014
|
+
# 입력칸에서는 q, s, 화살표를 글자 입력에 쓴다. Esc 는 입력칸에서만 의미가 있다
|
|
1015
|
+
typing = isinstance(self.focused, Input)
|
|
1016
|
+
# 창(모달)이 떠 있는 동안에는 뒤의 탭을 바꾸지 않는다
|
|
1017
|
+
if action in ("settings", "prev_tab", "next_tab", "leave_input", "refresh") and \
|
|
1018
|
+
isinstance(self.screen, ModalScreen):
|
|
1019
|
+
return False
|
|
1020
|
+
if action in ("quit", "settings", "prev_tab", "next_tab", "refresh"):
|
|
1021
|
+
return not typing
|
|
1022
|
+
if action == "leave_input":
|
|
1023
|
+
return typing
|
|
1024
|
+
return True
|
|
1025
|
+
|
|
1026
|
+
def action_settings(self) -> None:
|
|
1027
|
+
self.query_one("#main-tabs", TabbedContent).active = "tab-settings"
|
|
1028
|
+
|
|
1029
|
+
def save(self, **kw) -> bool:
|
|
1030
|
+
try:
|
|
1031
|
+
save_config(**kw)
|
|
1032
|
+
return True
|
|
1033
|
+
except OSError as e:
|
|
1034
|
+
self.notify(f"Could not save settings: {e}", severity="error", timeout=10)
|
|
1035
|
+
return False
|
|
1036
|
+
|
|
1037
|
+
def set_cache_dir(self, cache: str) -> None:
|
|
1038
|
+
if not self.save(cache_dir=cache):
|
|
1039
|
+
return
|
|
1040
|
+
if self.cli_cache_dir:
|
|
1041
|
+
self.notify("Saved. This session keeps the folder given with --cache-dir.", timeout=6)
|
|
1042
|
+
return
|
|
1043
|
+
self.cache_dir = os.path.abspath(os.path.expanduser(cache or constants.HF_HUB_CACHE))
|
|
1044
|
+
self.notify(f"Cache folder: {self.cache_dir}")
|
|
1045
|
+
self.load_cache()
|
|
1046
|
+
|
|
1047
|
+
def update_subtitle(self) -> None:
|
|
1048
|
+
who = self.user["name"] if self.user else "not logged in"
|
|
1049
|
+
self.sub_title = f"{who} · {self.cache_dir}"
|
|
1050
|
+
|
|
1051
|
+
# ------------------------------------------------ 새로고침
|
|
1052
|
+
|
|
1053
|
+
def action_refresh(self) -> None:
|
|
1054
|
+
"""r: 보고 있는 탭을 새로고침한다."""
|
|
1055
|
+
tab = self.query_one("#main-tabs", TabbedContent).active
|
|
1056
|
+
if tab == "tab-hub":
|
|
1057
|
+
self.search_submitted()
|
|
1058
|
+
elif tab in ("tab-mine", "tab-settings"):
|
|
1059
|
+
self.load_mine()
|
|
1060
|
+
else:
|
|
1061
|
+
self.load_cache()
|
|
1062
|
+
|
|
1063
|
+
def action_refresh_all(self) -> None:
|
|
1064
|
+
self.load_cache()
|
|
1065
|
+
self.load_mine()
|
|
1066
|
+
|
|
1067
|
+
@staticmethod
|
|
1068
|
+
def refill(table: DataTable, rows: list[tuple[str, list]]) -> None:
|
|
1069
|
+
"""커서 위치를 유지하며 다시 채운다."""
|
|
1070
|
+
keep = None
|
|
1071
|
+
if table.row_count:
|
|
1072
|
+
try:
|
|
1073
|
+
keep = table.coordinate_to_cell_key(table.cursor_coordinate).row_key.value
|
|
1074
|
+
except Exception:
|
|
1075
|
+
keep = None
|
|
1076
|
+
table.clear()
|
|
1077
|
+
for key, cells in rows:
|
|
1078
|
+
table.add_row(*[c if isinstance(c, Text) else Text(str(c)) for c in cells], key=key)
|
|
1079
|
+
keys = [k for k, _ in rows]
|
|
1080
|
+
if keep in keys:
|
|
1081
|
+
table.move_cursor(row=keys.index(keep))
|
|
1082
|
+
|
|
1083
|
+
def selected(self, table_id: str) -> Optional[str]:
|
|
1084
|
+
table = self.query_one(f"#{table_id}", DataTable)
|
|
1085
|
+
if not table.row_count:
|
|
1086
|
+
return None
|
|
1087
|
+
return table.coordinate_to_cell_key(table.cursor_coordinate).row_key.value
|
|
1088
|
+
|
|
1089
|
+
# ------------------------------------------------ 캐시
|
|
1090
|
+
|
|
1091
|
+
@work(thread=True, exclusive=True, group="cache")
|
|
1092
|
+
def load_cache(self) -> None:
|
|
1093
|
+
for rt in CACHE_TYPES:
|
|
1094
|
+
self.call_from_thread(self.query_one(f"#count-{rt}", Static).update, Text("Scanning…", style="yellow"))
|
|
1095
|
+
try:
|
|
1096
|
+
info = scan_cache_dir(self.cache_dir)
|
|
1097
|
+
except CacheNotFound:
|
|
1098
|
+
info = None
|
|
1099
|
+
orphans = {}
|
|
1100
|
+
if info and _shared_blobs is not None:
|
|
1101
|
+
try:
|
|
1102
|
+
orphans = _shared_blobs.unreferenced_shared_blobs(self.cache_dir)
|
|
1103
|
+
except OSError:
|
|
1104
|
+
pass
|
|
1105
|
+
self.call_from_thread(self.apply_cache, info, orphans)
|
|
1106
|
+
|
|
1107
|
+
def apply_cache(self, info, orphans: dict) -> None:
|
|
1108
|
+
self.cache, self.orphans = info, orphans
|
|
1109
|
+
self.cached = {f"{r.repo_type}:{r.repo_id}": r for r in info.repos} if info else {}
|
|
1110
|
+
tabs = self.query_one("#main-tabs", TabbedContent)
|
|
1111
|
+
for label, rt in REPO_TYPES:
|
|
1112
|
+
repos = [r for r in self.cached.values() if r.repo_type == rt]
|
|
1113
|
+
total = sum(r.size_on_disk for r in repos)
|
|
1114
|
+
self.fill_list(rt)
|
|
1115
|
+
tabs.get_tab(f"tab-{rt}").label = f"{label} {len(repos)}" + (f" · {fmt_bytes(total)}" if repos else "")
|
|
1116
|
+
self.fetch_meta()
|
|
1117
|
+
self.fill_cleanup()
|
|
1118
|
+
self.update_subtitle()
|
|
1119
|
+
|
|
1120
|
+
def cache_type(self) -> str:
|
|
1121
|
+
"""지금 보고 있는 캐시 탭의 타입 (model/dataset/space)."""
|
|
1122
|
+
active = self.query_one("#main-tabs", TabbedContent).active.removeprefix("tab-")
|
|
1123
|
+
return active if active in CACHE_TYPES else "model"
|
|
1124
|
+
|
|
1125
|
+
def selected_cached(self, warn: bool = True, rt: str = ""):
|
|
1126
|
+
key = self.selected(f"ctable-{rt or self.cache_type()}")
|
|
1127
|
+
r = self.cached.get(key) if key else None
|
|
1128
|
+
if r is None and warn:
|
|
1129
|
+
self.notify("Select a cached repo.", severity="warning")
|
|
1130
|
+
return r
|
|
1131
|
+
|
|
1132
|
+
@on(DataTable.RowHighlighted, "CacheTable")
|
|
1133
|
+
def cache_highlighted(self, e: DataTable.RowHighlighted) -> None:
|
|
1134
|
+
self.show_cache_detail(e.data_table.id.removeprefix("ctable-"))
|
|
1135
|
+
|
|
1136
|
+
def show_cache_detail(self, rt: str) -> None:
|
|
1137
|
+
box = self.query_one(f"#cache-detail-{rt}", Static)
|
|
1138
|
+
r = self.selected_cached(warn=False, rt=rt)
|
|
1139
|
+
if r is None:
|
|
1140
|
+
if self.list_rows.get(rt):
|
|
1141
|
+
box.update(Text("Nothing matches the filter.", style="dim"))
|
|
1142
|
+
else:
|
|
1143
|
+
box.update(Text(f"No {rt}s in {self.cache_dir}.\nSearch the Hub tab and press p to download.",
|
|
1144
|
+
style="dim"))
|
|
1145
|
+
return
|
|
1146
|
+
t = Text()
|
|
1147
|
+
t.append(f"{r.repo_id}\n", style="bold")
|
|
1148
|
+
t.append(f"Size {fmt_bytes(r.size_on_disk)} in {r.nb_files} files\n")
|
|
1149
|
+
meta = self.meta.get(f"{r.repo_type}:{r.repo_id}") or {}
|
|
1150
|
+
if meta.get("ok"):
|
|
1151
|
+
if meta.get("task"):
|
|
1152
|
+
t.append(f"Task {meta['task']}\n")
|
|
1153
|
+
if meta.get("library"):
|
|
1154
|
+
t.append(f"Library {meta['library']}\n")
|
|
1155
|
+
if meta.get("params"):
|
|
1156
|
+
t.append(f"Params {fmt_params(meta['params'])}\n")
|
|
1157
|
+
lic = [x.removeprefix("license:") for x in meta.get("tags", []) if x.startswith("license:")]
|
|
1158
|
+
if lic:
|
|
1159
|
+
t.append(f"License {', '.join(lic)}\n")
|
|
1160
|
+
t.append(f"Used {ago(r.last_accessed)} modified {ago(r.last_modified)}\n")
|
|
1161
|
+
t.append(f"Hub {repo_url(r.repo_id, r.repo_type)}\n")
|
|
1162
|
+
t.append("\nRevisions\n", style="bold")
|
|
1163
|
+
for rev in sorted(r.revisions, key=lambda v: -v.last_modified):
|
|
1164
|
+
refs = ", ".join(sorted(rev.refs))
|
|
1165
|
+
t.append(f" {rev.commit_hash[:10]} ", style="cyan")
|
|
1166
|
+
t.append(refs or "old (no branch points here)", style="" if refs else "dim")
|
|
1167
|
+
t.append(f" {fmt_bytes(rev.size_on_disk)} {len(rev.files)} files\n")
|
|
1168
|
+
main = next((v for v in r.revisions if "main" in v.refs), None) or \
|
|
1169
|
+
max(r.revisions, key=lambda v: v.last_modified, default=None)
|
|
1170
|
+
if main:
|
|
1171
|
+
t.append("\nLocal path\n", style="bold")
|
|
1172
|
+
t.append(f" {main.snapshot_path}\n", style="cyan")
|
|
1173
|
+
box.update(t)
|
|
1174
|
+
|
|
1175
|
+
def action_delete_cached(self) -> None:
|
|
1176
|
+
r = self.selected_cached()
|
|
1177
|
+
if r is None or self.cache is None:
|
|
1178
|
+
return
|
|
1179
|
+
strategy = self.cache.delete_revisions(*[v.commit_hash for v in r.revisions])
|
|
1180
|
+
detail = f"{r.repo_path}\nFrees {strategy.expected_freed_size_str}."
|
|
1181
|
+
if strategy.expected_freed_size < r.size_on_disk:
|
|
1182
|
+
detail += " Files shared with other repos are kept."
|
|
1183
|
+
detail += "\nAnyone else using this cache loses it too. It can be downloaded again from the Hub."
|
|
1184
|
+
self.push_screen(Confirm(f"Really delete {r.repo_id} from the cache?", detail),
|
|
1185
|
+
lambda ok: ok and self.run_delete(f"Delete {r.repo_id}", strategy))
|
|
1186
|
+
|
|
1187
|
+
def action_delete_old_revisions(self) -> None:
|
|
1188
|
+
r = self.selected_cached()
|
|
1189
|
+
if r is None or self.cache is None:
|
|
1190
|
+
return
|
|
1191
|
+
old = [v for v in r.revisions if not v.refs]
|
|
1192
|
+
if not old:
|
|
1193
|
+
self.notify("No old revisions: every cached revision is a current branch or tag.")
|
|
1194
|
+
return
|
|
1195
|
+
strategy = self.cache.delete_revisions(*[v.commit_hash for v in old])
|
|
1196
|
+
detail = "\n".join(f"{v.commit_hash[:10]} {fmt_bytes(v.size_on_disk)}" for v in old)
|
|
1197
|
+
detail += (f"\n\nFrees {strategy.expected_freed_size_str}. Files shared with the current revision are kept."
|
|
1198
|
+
"\nCode pinned to an old revision would download it again.")
|
|
1199
|
+
self.push_screen(Confirm(f"Really remove {len(old)} old revisions of {r.repo_id}?", detail),
|
|
1200
|
+
lambda ok: ok and self.run_delete(f"Remove old revisions of {r.repo_id}", strategy))
|
|
1201
|
+
|
|
1202
|
+
@work(thread=True)
|
|
1203
|
+
def run_delete(self, label: str, strategy) -> None:
|
|
1204
|
+
try:
|
|
1205
|
+
strategy.execute()
|
|
1206
|
+
self.call_from_thread(self.notify, f"{label}: done")
|
|
1207
|
+
except OSError as e:
|
|
1208
|
+
self.call_from_thread(self.notify, f"{label} failed: {e}", severity="error", timeout=10)
|
|
1209
|
+
self.call_from_thread(self.load_cache)
|
|
1210
|
+
|
|
1211
|
+
def action_update_cached(self) -> None:
|
|
1212
|
+
r = self.selected_cached()
|
|
1213
|
+
if r:
|
|
1214
|
+
self.open_download(r.repo_id, r.repo_type)
|
|
1215
|
+
|
|
1216
|
+
# ------------------------------------------------ 허브
|
|
1217
|
+
|
|
1218
|
+
@on(Input.Submitted, "#hub-query")
|
|
1219
|
+
@on(Select.Changed, "#hub-type")
|
|
1220
|
+
def search_submitted(self) -> None:
|
|
1221
|
+
self.show_hub_filters()
|
|
1222
|
+
self.search(self.query_one("#hub-query", Input).value.strip(), self.query_one("#hub-type", Select).value)
|
|
1223
|
+
|
|
1224
|
+
def show_hub_filters(self) -> None:
|
|
1225
|
+
rt = self.query_one("#hub-type", Select).value
|
|
1226
|
+
t = Text()
|
|
1227
|
+
t.append("Filters: ", style="bold")
|
|
1228
|
+
t.append(filter_summary(HUB_FILTERS[rt], self.hub_filters[rt]))
|
|
1229
|
+
t.append(" (f to change)", style="dim")
|
|
1230
|
+
self.query_one("#hub-filters", Static).update(t)
|
|
1231
|
+
|
|
1232
|
+
def action_hub_filters(self) -> None:
|
|
1233
|
+
rt = self.query_one("#hub-type", Select).value
|
|
1234
|
+
|
|
1235
|
+
def done(f: Optional[dict]) -> None:
|
|
1236
|
+
if f is not None:
|
|
1237
|
+
self.hub_filters[rt] = f
|
|
1238
|
+
self.search_submitted()
|
|
1239
|
+
|
|
1240
|
+
label = dict((v, k) for k, v in REPO_TYPES)[rt].lower()
|
|
1241
|
+
self.push_screen(HubFilterForm(f"Filter {label} on the Hub", HUB_FILTERS[rt], self.hub_filters[rt]), done)
|
|
1242
|
+
|
|
1243
|
+
@work(thread=True, exclusive=True, group="search")
|
|
1244
|
+
def search(self, query: str, repo_type: str) -> None:
|
|
1245
|
+
lister = {"model": self.api.list_models, "dataset": self.api.list_datasets,
|
|
1246
|
+
"space": self.api.list_spaces}[repo_type]
|
|
1247
|
+
kw = hub_query_args(repo_type, self.hub_filters[repo_type])
|
|
1248
|
+
if repo_type == "model":
|
|
1249
|
+
# 목록에 용도와 파라미터 수를 보여주려고 함께 받는다
|
|
1250
|
+
kw["expand"] = ["downloads", "likes", "lastModified", "pipeline_tag", "safetensors"]
|
|
1251
|
+
try:
|
|
1252
|
+
items = list(lister(search=query or None, limit=200, **kw))
|
|
1253
|
+
# 허브 검색은 owner/name 을 그대로 넣으면 못 찾는 경우가 있어 정확한 이름은 직접 조회한다
|
|
1254
|
+
if query.count("/") == 1 and not any(i.id == query for i in items):
|
|
1255
|
+
try:
|
|
1256
|
+
info = self.api.repo_info(query, repo_type=repo_type, files_metadata=True)
|
|
1257
|
+
self.hub_info[info.id] = info
|
|
1258
|
+
items.insert(0, info)
|
|
1259
|
+
except (HfHubHTTPError, OSError):
|
|
1260
|
+
pass
|
|
1261
|
+
except (HfHubHTTPError, OSError) as e:
|
|
1262
|
+
self.call_from_thread(self.notify, f"Search failed: {e}", severity="error", timeout=10)
|
|
1263
|
+
return
|
|
1264
|
+
self.call_from_thread(self.apply_search, repo_type, items, bool(query))
|
|
1265
|
+
|
|
1266
|
+
def apply_search(self, repo_type: str, items: list, focus: bool) -> None:
|
|
1267
|
+
self.hub = {i.id: (repo_type, i) for i in items}
|
|
1268
|
+
table = self.query_one("#htable", DataTable)
|
|
1269
|
+
# 용도와 파라미터 수는 모델에만 있다
|
|
1270
|
+
columns = ["Repo", "Task", "Params", "Downloads", "Likes", "Updated"] if repo_type == "model" \
|
|
1271
|
+
else ["Repo", "Downloads", "Likes", "Updated"]
|
|
1272
|
+
if [str(c.label) for c in table.columns.values()] != columns:
|
|
1273
|
+
table.clear(columns=True)
|
|
1274
|
+
table.add_columns(*columns)
|
|
1275
|
+
rows = []
|
|
1276
|
+
for i in items:
|
|
1277
|
+
st = getattr(i, "safetensors", None)
|
|
1278
|
+
model_cells = [getattr(i, "pipeline_tag", None) or "", fmt_params(st.total if st else None)] \
|
|
1279
|
+
if repo_type == "model" else []
|
|
1280
|
+
rows.append((i.id, [i.id, *model_cells, fmt_count(getattr(i, "downloads", None)),
|
|
1281
|
+
fmt_count(getattr(i, "likes", None)), ago_dt(getattr(i, "last_modified", None))]))
|
|
1282
|
+
self.refill(table, rows)
|
|
1283
|
+
if focus and rows:
|
|
1284
|
+
table.focus()
|
|
1285
|
+
if not rows:
|
|
1286
|
+
self.query_one("#hub-detail", Static).update(Text("No results. Try fewer filters (f).", style="dim"))
|
|
1287
|
+
|
|
1288
|
+
@on(DataTable.RowHighlighted, "#htable")
|
|
1289
|
+
def show_hub_detail(self) -> None:
|
|
1290
|
+
repo_id = self.selected("htable")
|
|
1291
|
+
if not repo_id or repo_id not in self.hub:
|
|
1292
|
+
return
|
|
1293
|
+
repo_type, _ = self.hub[repo_id]
|
|
1294
|
+
if repo_id in self.hub_info:
|
|
1295
|
+
self.render_hub_detail(repo_id, repo_type, self.hub_info[repo_id])
|
|
1296
|
+
else:
|
|
1297
|
+
self.query_one("#hub-detail", Static).update(Text(f"{repo_id}\n\nLoading…", style="dim"))
|
|
1298
|
+
self.load_hub_info(repo_id, repo_type)
|
|
1299
|
+
|
|
1300
|
+
@work(thread=True, exclusive=True, group="hub-info")
|
|
1301
|
+
def load_hub_info(self, repo_id: str, repo_type: str) -> None:
|
|
1302
|
+
time.sleep(0.25) # 커서를 빠르게 움직이는 동안은 요청하지 않는다
|
|
1303
|
+
try:
|
|
1304
|
+
info = self.api.repo_info(repo_id, repo_type=repo_type, files_metadata=True)
|
|
1305
|
+
except (HfHubHTTPError, OSError) as e:
|
|
1306
|
+
info = e
|
|
1307
|
+
self.hub_info[repo_id] = info
|
|
1308
|
+
self.call_from_thread(self.hub_info_loaded, repo_id, repo_type)
|
|
1309
|
+
|
|
1310
|
+
def hub_info_loaded(self, repo_id: str, repo_type: str) -> None:
|
|
1311
|
+
if self.selected("htable") == repo_id:
|
|
1312
|
+
self.render_hub_detail(repo_id, repo_type, self.hub_info[repo_id])
|
|
1313
|
+
|
|
1314
|
+
def render_hub_detail(self, repo_id: str, repo_type: str, info) -> None:
|
|
1315
|
+
box = self.query_one("#hub-detail", Static)
|
|
1316
|
+
t = Text()
|
|
1317
|
+
t.append(f"{repo_id}\n", style="bold")
|
|
1318
|
+
if isinstance(info, Exception):
|
|
1319
|
+
t.append(f"\n{info}\n", style="red")
|
|
1320
|
+
box.update(t)
|
|
1321
|
+
return
|
|
1322
|
+
files = info.siblings or []
|
|
1323
|
+
card = info.card_data.to_dict() if getattr(info, "card_data", None) else {}
|
|
1324
|
+
t.append(f"Type {repo_type}\n")
|
|
1325
|
+
if getattr(info, "pipeline_tag", None):
|
|
1326
|
+
t.append(f"Task {info.pipeline_tag}\n")
|
|
1327
|
+
if getattr(info, "library_name", None):
|
|
1328
|
+
t.append(f"Library {info.library_name}\n")
|
|
1329
|
+
if getattr(info, "safetensors", None):
|
|
1330
|
+
t.append(f"Params {fmt_params(info.safetensors.total)}\n")
|
|
1331
|
+
if card.get("license"):
|
|
1332
|
+
t.append(f"License {card['license']}\n")
|
|
1333
|
+
t.append(f"Size {fmt_bytes(sum(f.size or 0 for f in files))} in {len(files)} files\n")
|
|
1334
|
+
t.append(f"Updated {ago_dt(info.last_modified)}\n")
|
|
1335
|
+
if getattr(info, "gated", False):
|
|
1336
|
+
t.append(f"Gated accept the terms at {repo_url(repo_id, repo_type)} first\n", style="yellow")
|
|
1337
|
+
if getattr(info, "private", False):
|
|
1338
|
+
t.append("Private\n", style="yellow")
|
|
1339
|
+
cached = self.cached.get(f"{repo_type}:{repo_id}")
|
|
1340
|
+
if cached:
|
|
1341
|
+
t.append(f"Cached {fmt_bytes(cached.size_on_disk)} (see the {dict((v, k) for k, v in REPO_TYPES)[repo_type]} tab)\n",
|
|
1342
|
+
style="green")
|
|
1343
|
+
t.append(f"Hub {repo_url(repo_id, repo_type)}\n")
|
|
1344
|
+
t.append("\nLargest files\n", style="bold")
|
|
1345
|
+
for f in sorted(files, key=lambda f: -(f.size or 0))[:12]:
|
|
1346
|
+
t.append(f" {fmt_bytes(f.size or 0):>9} ", style="cyan")
|
|
1347
|
+
t.append(f"{f.rfilename}\n")
|
|
1348
|
+
if len(files) > 12:
|
|
1349
|
+
t.append(f" … {len(files) - 12} more\n", style="dim")
|
|
1350
|
+
box.update(t)
|
|
1351
|
+
|
|
1352
|
+
@on(DataTable.RowSelected, "#htable")
|
|
1353
|
+
def action_download(self) -> None:
|
|
1354
|
+
repo_id = self.selected("htable")
|
|
1355
|
+
if repo_id and repo_id in self.hub:
|
|
1356
|
+
self.open_download(repo_id, self.hub[repo_id][0])
|
|
1357
|
+
|
|
1358
|
+
def open_download(self, repo_id: str, repo_type: str) -> None:
|
|
1359
|
+
info = self.hub_info.get(repo_id)
|
|
1360
|
+
if info is not None and not isinstance(info, Exception):
|
|
1361
|
+
self.push_screen(DownloadForm(repo_id, repo_type, info.siblings or [], self.cache_dir),
|
|
1362
|
+
lambda opts: opts and self.start_download(repo_id, repo_type, opts))
|
|
1363
|
+
else:
|
|
1364
|
+
self.notify("Loading file list…", timeout=2)
|
|
1365
|
+
self.fetch_for_download(repo_id, repo_type)
|
|
1366
|
+
|
|
1367
|
+
@work(thread=True)
|
|
1368
|
+
def fetch_for_download(self, repo_id: str, repo_type: str) -> None:
|
|
1369
|
+
try:
|
|
1370
|
+
self.hub_info[repo_id] = self.api.repo_info(repo_id, repo_type=repo_type, files_metadata=True)
|
|
1371
|
+
except (HfHubHTTPError, OSError) as e:
|
|
1372
|
+
self.call_from_thread(self.notify, f"{repo_id}: {e}", severity="error", timeout=10)
|
|
1373
|
+
return
|
|
1374
|
+
self.call_from_thread(self.open_download, repo_id, repo_type)
|
|
1375
|
+
|
|
1376
|
+
def start_download(self, repo_id: str, repo_type: str, opts: dict) -> None:
|
|
1377
|
+
log = TaskLog(f"Download: {repo_id}", background=True)
|
|
1378
|
+
self.push_screen(log)
|
|
1379
|
+
self.run_download(log, repo_id, repo_type, opts)
|
|
1380
|
+
|
|
1381
|
+
@work(thread=True)
|
|
1382
|
+
def run_download(self, log: TaskLog, repo_id: str, repo_type: str, opts: dict) -> None:
|
|
1383
|
+
def say(msg: str, style: str = "") -> None:
|
|
1384
|
+
self.call_from_thread(log.write, msg, style)
|
|
1385
|
+
|
|
1386
|
+
total, count = opts["total"], opts["count"]
|
|
1387
|
+
dest = opts["local_dir"] or self.cache_dir
|
|
1388
|
+
say(f"{count} files · {fmt_bytes(total)} · revision {opts['revision']}")
|
|
1389
|
+
if opts["allow"]:
|
|
1390
|
+
say(f"include {' '.join(opts['allow'])}", "dim")
|
|
1391
|
+
if opts["ignore"]:
|
|
1392
|
+
say(f"exclude {' '.join(opts['ignore'])}", "dim")
|
|
1393
|
+
say(f"→ {dest}", "dim")
|
|
1394
|
+
bars: dict = {}
|
|
1395
|
+
started = time.time()
|
|
1396
|
+
stop = threading.Event()
|
|
1397
|
+
|
|
1398
|
+
def show_progress() -> None:
|
|
1399
|
+
got = int(bars["bytes"].n) if "bytes" in bars else 0
|
|
1400
|
+
files = bars.get("files")
|
|
1401
|
+
done, of = (int(files.n), int(files.total or count)) if files else (0, count)
|
|
1402
|
+
speed = got / max(time.time() - started, 0.1)
|
|
1403
|
+
bar_n = min(int(40 * got / total), 40) if total else 0
|
|
1404
|
+
t = Text()
|
|
1405
|
+
t.append("█" * bar_n, style="cyan")
|
|
1406
|
+
t.append("░" * (40 - bar_n), style="dim")
|
|
1407
|
+
t.append(f" {fmt_bytes(got)} / {fmt_bytes(total)} · {done}/{of} files · {fmt_bytes(speed)}/s")
|
|
1408
|
+
self.call_from_thread(log.progress, t)
|
|
1409
|
+
|
|
1410
|
+
def ticker() -> None:
|
|
1411
|
+
while not stop.wait(0.5):
|
|
1412
|
+
show_progress()
|
|
1413
|
+
|
|
1414
|
+
threading.Thread(target=ticker, daemon=True).start()
|
|
1415
|
+
kw = dict(repo_type=repo_type, revision=opts["revision"], allow_patterns=opts["allow"],
|
|
1416
|
+
ignore_patterns=opts["ignore"], tqdm_class=progress_bars(bars))
|
|
1417
|
+
if opts["local_dir"]:
|
|
1418
|
+
kw["local_dir"] = opts["local_dir"]
|
|
1419
|
+
else:
|
|
1420
|
+
kw["cache_dir"] = self.cache_dir
|
|
1421
|
+
try:
|
|
1422
|
+
# 커밋을 한 번 정하고 그 커밋의 파일만 받는다. 이미 받은 파일은 건너뛰므로 다시 실행하면 이어 받는다
|
|
1423
|
+
path = snapshot_download(repo_id, **kw)
|
|
1424
|
+
error = None
|
|
1425
|
+
except Exception as e: # 네트워크, 권한(gated), 디스크 부족 등
|
|
1426
|
+
path, error = None, e
|
|
1427
|
+
stop.set()
|
|
1428
|
+
show_progress()
|
|
1429
|
+
if error:
|
|
1430
|
+
say(f"\n✗ Failed: {error}", "bold red")
|
|
1431
|
+
say("Files that finished are kept. Download again to continue.", "dim")
|
|
1432
|
+
self.call_from_thread(self.notify, f"Download {repo_id} failed", severity="error", timeout=10)
|
|
1433
|
+
else:
|
|
1434
|
+
say(f"\nDone in {time.time() - started:.0f}s: {path}", "bold green")
|
|
1435
|
+
self.call_from_thread(self.notify, f"Downloaded {repo_id}")
|
|
1436
|
+
self.call_from_thread(log.finish)
|
|
1437
|
+
self.call_from_thread(self.load_cache)
|
|
1438
|
+
|
|
1439
|
+
# ------------------------------------------------ 내 저장소
|
|
1440
|
+
|
|
1441
|
+
@work(thread=True, exclusive=True, group="mine")
|
|
1442
|
+
def load_mine(self) -> None:
|
|
1443
|
+
# 목록에 loading 을 켜면 포커스가 빠져 키가 안 먹으므로, 필터 칸 옆에 글자로 알린다
|
|
1444
|
+
self.call_from_thread(self.query_one("#count-mine", Static).update, Text("Loading…", style="yellow"))
|
|
1445
|
+
if not get_token():
|
|
1446
|
+
self.call_from_thread(self.apply_mine, None, [])
|
|
1447
|
+
return
|
|
1448
|
+
try:
|
|
1449
|
+
user = self.api.whoami()
|
|
1450
|
+
rows = []
|
|
1451
|
+
for owner in [user["name"]] + [o["name"] for o in user.get("orgs", [])]:
|
|
1452
|
+
for repo_type, lister in (("model", self.api.list_models), ("dataset", self.api.list_datasets),
|
|
1453
|
+
("space", self.api.list_spaces)):
|
|
1454
|
+
rows += [(repo_type, i) for i in lister(author=owner)]
|
|
1455
|
+
except (HfHubHTTPError, OSError) as e:
|
|
1456
|
+
self.call_from_thread(self.notify, f"Could not load your repos: {e}", severity="error", timeout=10)
|
|
1457
|
+
user, rows = None, []
|
|
1458
|
+
self.call_from_thread(self.apply_mine, user, rows)
|
|
1459
|
+
|
|
1460
|
+
def apply_mine(self, user: Optional[dict], items: list) -> None:
|
|
1461
|
+
self.user = user
|
|
1462
|
+
self.my_repos = {f"{rt}:{i.id}": (rt, i) for rt, i in items}
|
|
1463
|
+
rows = []
|
|
1464
|
+
for key, (rt, i) in sorted(self.my_repos.items(), key=lambda kv: (kv[1][1].id.lower(), kv[1][0])):
|
|
1465
|
+
vis = Text("private", style="yellow") if i.private else Text("public")
|
|
1466
|
+
rows.append((key, [i.id, rt, vis, fmt_count(getattr(i, "downloads", None)),
|
|
1467
|
+
fmt_count(getattr(i, "likes", None)), ago_dt(i.last_modified)],
|
|
1468
|
+
f"{i.id} {rt} {vis.plain}"))
|
|
1469
|
+
self.list_rows["mine"] = rows
|
|
1470
|
+
self.fill_list("mine")
|
|
1471
|
+
self.update_subtitle()
|
|
1472
|
+
self.query_one(SettingsPane).show_account(user)
|
|
1473
|
+
|
|
1474
|
+
def owner_names(self) -> list[str]:
|
|
1475
|
+
if not self.user:
|
|
1476
|
+
return []
|
|
1477
|
+
return [self.user["name"]] + [o["name"] for o in self.user.get("orgs", [])]
|
|
1478
|
+
|
|
1479
|
+
def selected_mine(self, warn: bool = True):
|
|
1480
|
+
key = self.selected("rtable")
|
|
1481
|
+
r = self.my_repos.get(key) if key else None
|
|
1482
|
+
if r is None and warn:
|
|
1483
|
+
self.notify("Select a repository." if self.user else "Log in first (s → Settings).", severity="warning")
|
|
1484
|
+
return r
|
|
1485
|
+
|
|
1486
|
+
@on(DataTable.RowHighlighted, "#rtable")
|
|
1487
|
+
def show_repo_detail(self) -> None:
|
|
1488
|
+
box = self.query_one("#repo-detail", Static)
|
|
1489
|
+
if not self.user:
|
|
1490
|
+
box.update(Text("Not logged in.\nPress s and log in under Settings to see and manage your repositories.",
|
|
1491
|
+
style="dim"))
|
|
1492
|
+
return
|
|
1493
|
+
t = Text()
|
|
1494
|
+
t.append(f"{self.user['name']}", style="bold")
|
|
1495
|
+
if self.user.get("fullname"):
|
|
1496
|
+
t.append(f" {self.user['fullname']}")
|
|
1497
|
+
orgs = self.owner_names()[1:]
|
|
1498
|
+
t.append(f"\nOrgs {', '.join(orgs) or 'none'}\n")
|
|
1499
|
+
token = (self.user.get("auth") or {}).get("accessToken") or {}
|
|
1500
|
+
if token:
|
|
1501
|
+
t.append(f"Token {token.get('displayName', '')} ({token.get('role', '')})\n")
|
|
1502
|
+
r = self.selected_mine(warn=False)
|
|
1503
|
+
if r is None:
|
|
1504
|
+
t.append("\nNothing matches the filter.\n" if self.list_rows.get("mine")
|
|
1505
|
+
else "\nNo repositories yet. Press n to create one.\n", style="dim")
|
|
1506
|
+
box.update(t)
|
|
1507
|
+
return
|
|
1508
|
+
rt, i = r
|
|
1509
|
+
t.append(f"\n{i.id}\n", style="bold")
|
|
1510
|
+
t.append(f"Type {rt}\n")
|
|
1511
|
+
t.append("Visibility ")
|
|
1512
|
+
t.append("private\n" if i.private else "public\n", style="yellow" if i.private else "")
|
|
1513
|
+
if getattr(i, "downloads", None) is not None:
|
|
1514
|
+
t.append(f"Downloads {i.downloads:,} (last 30 days)\n")
|
|
1515
|
+
t.append(f"Likes {getattr(i, 'likes', 0) or 0}\n")
|
|
1516
|
+
t.append(f"Updated {ago_dt(i.last_modified)}\n")
|
|
1517
|
+
t.append(f"Hub {repo_url(i.id, rt)}\n")
|
|
1518
|
+
t.append("\nClone\n", style="bold")
|
|
1519
|
+
t.append(f" git clone {repo_url(i.id, rt)}\n", style="cyan")
|
|
1520
|
+
box.update(t)
|
|
1521
|
+
|
|
1522
|
+
def action_download_mine(self) -> None:
|
|
1523
|
+
r = self.selected_mine()
|
|
1524
|
+
if r:
|
|
1525
|
+
self.open_download(r[1].id, r[0])
|
|
1526
|
+
|
|
1527
|
+
def action_new_repo(self) -> None:
|
|
1528
|
+
if not self.user:
|
|
1529
|
+
self.notify("Log in first (s → Settings).", severity="warning")
|
|
1530
|
+
return
|
|
1531
|
+
self.push_screen(NewRepoForm(self.owner_names()), lambda opts: opts and self.hub_call(
|
|
1532
|
+
f"Create {opts['repo_id']}", self.api.create_repo, opts["repo_id"],
|
|
1533
|
+
repo_type=opts["repo_type"], private=opts["private"]))
|
|
1534
|
+
|
|
1535
|
+
def action_toggle_visibility(self) -> None:
|
|
1536
|
+
r = self.selected_mine()
|
|
1537
|
+
if not r:
|
|
1538
|
+
return
|
|
1539
|
+
rt, i = r
|
|
1540
|
+
to = "public" if i.private else "private"
|
|
1541
|
+
detail = ("Anyone will be able to see and download it." if to == "public"
|
|
1542
|
+
else "Only you and members of the owning organization will be able to see it.")
|
|
1543
|
+
self.push_screen(Confirm(f"Make {i.id} {to}?", detail), lambda ok: ok and self.hub_call(
|
|
1544
|
+
f"{i.id} is now {to}", self.api.update_repo_settings, i.id, repo_type=rt, private=not i.private))
|
|
1545
|
+
|
|
1546
|
+
def action_delete_repo(self) -> None:
|
|
1547
|
+
r = self.selected_mine()
|
|
1548
|
+
if not r:
|
|
1549
|
+
return
|
|
1550
|
+
rt, i = r
|
|
1551
|
+
self.push_screen(
|
|
1552
|
+
Confirm(f"Delete {rt} {i.id} from the Hub?",
|
|
1553
|
+
"This permanently deletes the repository, its files and its history.\n"
|
|
1554
|
+
"Local copies in the cache are not touched. This cannot be undone.", require=i.id),
|
|
1555
|
+
lambda ok: ok and self.hub_call(f"Delete {i.id}", self.api.delete_repo, i.id, repo_type=rt))
|
|
1556
|
+
|
|
1557
|
+
@work(thread=True)
|
|
1558
|
+
def hub_call(self, label: str, fn, *args, **kwargs) -> None:
|
|
1559
|
+
try:
|
|
1560
|
+
fn(*args, **kwargs)
|
|
1561
|
+
self.call_from_thread(self.notify, f"{label}: done")
|
|
1562
|
+
except (HfHubHTTPError, OSError, ValueError) as e:
|
|
1563
|
+
self.call_from_thread(self.notify, f"{label} failed: {e}", severity="error", timeout=10)
|
|
1564
|
+
self.call_from_thread(self.load_mine)
|
|
1565
|
+
|
|
1566
|
+
# ------------------------------------------------ 로그인
|
|
1567
|
+
|
|
1568
|
+
def run_hf(self, *args: str) -> None:
|
|
1569
|
+
"""hf CLI 를 이 터미널에서 그대로 실행한다 (로그인 방식, 질문, 토큰 저장은 모두 hf 가 한다)."""
|
|
1570
|
+
# thf 와 같은 huggingface_hub 를 쓰도록 같은 파이썬으로 실행한다
|
|
1571
|
+
# --format human: 에이전트로 오인되지 않게 해서, 로그인 방법(브라우저/토큰) 선택 메뉴가 항상 나오게 한다
|
|
1572
|
+
cmd = [sys.executable, "-m", "huggingface_hub.cli.hf", *args, "--format", "human"]
|
|
1573
|
+
# Ctrl+C 는 hf 만 멈추게 한다. thf 까지 받으면 돌아오자마자 종료된다
|
|
1574
|
+
# (파이썬 핸들러는 exec 때 기본값으로 돌아가므로 hf 에는 영향이 없다)
|
|
1575
|
+
previous = signal.signal(signal.SIGINT, lambda *_: None)
|
|
1576
|
+
try:
|
|
1577
|
+
with self.suspend():
|
|
1578
|
+
print(f"\n$ hf {' '.join(args)}\n", flush=True)
|
|
1579
|
+
rc = subprocess.run(cmd).returncode
|
|
1580
|
+
try:
|
|
1581
|
+
input(f"\n{'Done' if rc == 0 else f'Exited with {rc}'}. Press Enter to return to thf.")
|
|
1582
|
+
except EOFError:
|
|
1583
|
+
pass
|
|
1584
|
+
finally:
|
|
1585
|
+
signal.signal(signal.SIGINT, previous)
|
|
1586
|
+
self.load_mine()
|
|
1587
|
+
|
|
1588
|
+
# ------------------------------------------------ 정리
|
|
1589
|
+
|
|
1590
|
+
def fill_cleanup(self) -> None:
|
|
1591
|
+
box = self.query_one("#cleanup", Static)
|
|
1592
|
+
info = self.cache
|
|
1593
|
+
t = Text()
|
|
1594
|
+
t.append("Hugging Face cache\n", style="bold")
|
|
1595
|
+
t.append(f" {self.cache_dir}\n", style="cyan")
|
|
1596
|
+
if info is None:
|
|
1597
|
+
t.append(" (does not exist yet)\n", style="dim")
|
|
1598
|
+
box.update(t)
|
|
1599
|
+
return
|
|
1600
|
+
try:
|
|
1601
|
+
du = shutil.disk_usage(self.cache_dir)
|
|
1602
|
+
t.append(f" {fmt_bytes(info.size_on_disk)} used by the cache · {fmt_bytes(du.free)} free on this disk"
|
|
1603
|
+
f" ({du.used * 100 // du.total}% full)\n")
|
|
1604
|
+
except OSError:
|
|
1605
|
+
t.append(f" {fmt_bytes(info.size_on_disk)} used by the cache\n")
|
|
1606
|
+
|
|
1607
|
+
old = self.old_revisions()
|
|
1608
|
+
incomplete = list(info.incomplete_files)
|
|
1609
|
+
orphans = getattr(self, "orphans", {})
|
|
1610
|
+
t.append("\nCan be removed\n", style="bold")
|
|
1611
|
+
t.append(f" Old revisions {len(old)}\n")
|
|
1612
|
+
t.append(f" Incomplete downloads {len(incomplete)}, {fmt_bytes(sum(f.size_on_disk for f in incomplete))}\n")
|
|
1613
|
+
t.append(f" Orphaned shared files {len(orphans)}, {fmt_bytes(sum(orphans.values()))}\n")
|
|
1614
|
+
t.append(" Incomplete downloads may still be in progress; remove them only when no download is running.\n",
|
|
1615
|
+
style="dim")
|
|
1616
|
+
|
|
1617
|
+
# version.txt 는 라이브러리가 캐시에 두는 파일이라 문제로 보이지 않게 뺀다
|
|
1618
|
+
warnings = [w for w in info.warnings if "version.txt" not in str(w)]
|
|
1619
|
+
if warnings:
|
|
1620
|
+
t.append("\nProblems found while scanning\n", style="bold")
|
|
1621
|
+
for w in warnings[:20]:
|
|
1622
|
+
t.append(f" {w}\n", style="yellow")
|
|
1623
|
+
box.update(t)
|
|
1624
|
+
|
|
1625
|
+
def old_revisions(self) -> list:
|
|
1626
|
+
return [v for r in self.cached.values() for v in r.revisions if not v.refs]
|
|
1627
|
+
|
|
1628
|
+
@on(Button.Pressed)
|
|
1629
|
+
def cleanup(self, e: Button.Pressed) -> None:
|
|
1630
|
+
bid = e.button.id or ""
|
|
1631
|
+
if not bid.startswith("prune-") or self.cache is None:
|
|
1632
|
+
return
|
|
1633
|
+
if bid == "prune-revisions":
|
|
1634
|
+
old = self.old_revisions()
|
|
1635
|
+
if not old:
|
|
1636
|
+
self.notify("No cached repo has old revisions.")
|
|
1637
|
+
return
|
|
1638
|
+
strategy = self.cache.delete_revisions(*[v.commit_hash for v in old])
|
|
1639
|
+
self.push_screen(
|
|
1640
|
+
Confirm(f"Really remove {len(old)} old revisions?",
|
|
1641
|
+
"Revisions no branch or tag points to any more (left behind after updates).\n"
|
|
1642
|
+
f"Frees {strategy.expected_freed_size_str}. Anyone else using this cache loses them too."),
|
|
1643
|
+
lambda ok: ok and self.run_delete("Remove old revisions", strategy))
|
|
1644
|
+
elif bid == "prune-incomplete":
|
|
1645
|
+
files = list(self.cache.incomplete_files)
|
|
1646
|
+
if not files:
|
|
1647
|
+
self.notify("No incomplete downloads.")
|
|
1648
|
+
return
|
|
1649
|
+
self.push_screen(
|
|
1650
|
+
Confirm(f"Really remove {len(files)} incomplete downloads?",
|
|
1651
|
+
"A download that is still running (yours or anyone else's on this cache) will fail.\n\n"
|
|
1652
|
+
+ "\n".join(str(f.file_path) for f in files[:30])),
|
|
1653
|
+
lambda ok: ok and self.remove_files("Remove incomplete downloads", [f.file_path for f in files]))
|
|
1654
|
+
elif bid == "prune-shared":
|
|
1655
|
+
orphans = getattr(self, "orphans", {})
|
|
1656
|
+
if not orphans:
|
|
1657
|
+
self.notify("No orphaned shared files.")
|
|
1658
|
+
return
|
|
1659
|
+
self.push_screen(
|
|
1660
|
+
Confirm(f"Really remove {len(orphans)} orphaned shared files ({fmt_bytes(sum(orphans.values()))})?",
|
|
1661
|
+
"Files in the shared store that no cached repo refers to any more "
|
|
1662
|
+
"(left behind when a repo folder was deleted by hand)."),
|
|
1663
|
+
lambda ok: ok and self.sweep_orphans(list(orphans)))
|
|
1664
|
+
|
|
1665
|
+
@work(thread=True)
|
|
1666
|
+
def remove_files(self, label: str, paths: list[Path]) -> None:
|
|
1667
|
+
errors = 0
|
|
1668
|
+
for p in paths:
|
|
1669
|
+
try:
|
|
1670
|
+
p.unlink()
|
|
1671
|
+
except FileNotFoundError:
|
|
1672
|
+
pass
|
|
1673
|
+
except OSError:
|
|
1674
|
+
errors += 1
|
|
1675
|
+
self.call_from_thread(self.notify, f"{label}: done" + (f" ({errors} could not be removed)" if errors else ""))
|
|
1676
|
+
self.call_from_thread(self.load_cache)
|
|
1677
|
+
|
|
1678
|
+
@work(thread=True)
|
|
1679
|
+
def sweep_orphans(self, paths: list[Path]) -> None:
|
|
1680
|
+
for p in paths:
|
|
1681
|
+
try:
|
|
1682
|
+
_shared_blobs.sweep_shared_blob(p, cache_dir=self.cache_dir)
|
|
1683
|
+
except OSError:
|
|
1684
|
+
pass
|
|
1685
|
+
self.call_from_thread(self.notify, "Remove orphaned shared files: done")
|
|
1686
|
+
self.call_from_thread(self.load_cache)
|
|
1687
|
+
|
|
1688
|
+
|
|
1689
|
+
def main() -> None:
|
|
1690
|
+
import argparse
|
|
1691
|
+
from . import __version__
|
|
1692
|
+
ap = argparse.ArgumentParser(prog="thf", description="A terminal UI for managing Hugging Face models and datasets.")
|
|
1693
|
+
ap.add_argument("--cache-dir", metavar="DIR",
|
|
1694
|
+
help="Hugging Face cache to manage (default: the one in Settings, "
|
|
1695
|
+
f"otherwise HF_HUB_CACHE, now {constants.HF_HUB_CACHE})")
|
|
1696
|
+
ap.add_argument("--version", action="version", version=f"thf {__version__}")
|
|
1697
|
+
args = ap.parse_args()
|
|
1698
|
+
|
|
1699
|
+
# 라이브러리의 진행 막대와 경고가 화면을 깨뜨리지 않게 한다
|
|
1700
|
+
disable_progress_bars()
|
|
1701
|
+
hf_logging.set_verbosity_error()
|
|
1702
|
+
cache_dir = args.cache_dir or load_config().get("cache_dir") or constants.HF_HUB_CACHE
|
|
1703
|
+
HfTUI(os.path.abspath(os.path.expanduser(cache_dir)), cli_cache_dir=bool(args.cache_dir)).run()
|
|
1704
|
+
|
|
1705
|
+
|
|
1706
|
+
if __name__ == "__main__":
|
|
1707
|
+
main()
|