atomgit 1.0.10__tar.gz → 1.0.11__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {atomgit-1.0.10 → atomgit-1.0.11}/PKG-INFO +1 -1
- {atomgit-1.0.10 → atomgit-1.0.11}/api.py +14 -66
- {atomgit-1.0.10 → atomgit-1.0.11}/atomgit.egg-info/PKG-INFO +1 -1
- {atomgit-1.0.10 → atomgit-1.0.11}/setup.py +1 -1
- {atomgit-1.0.10 → atomgit-1.0.11}/MANIFEST.in +0 -0
- {atomgit-1.0.10 → atomgit-1.0.11}/README.md +0 -0
- {atomgit-1.0.10 → atomgit-1.0.11}/__init__.py +0 -0
- {atomgit-1.0.10 → atomgit-1.0.11}/__main__.py +0 -0
- {atomgit-1.0.10 → atomgit-1.0.11}/atomgit.egg-info/SOURCES.txt +0 -0
- {atomgit-1.0.10 → atomgit-1.0.11}/atomgit.egg-info/dependency_links.txt +0 -0
- {atomgit-1.0.10 → atomgit-1.0.11}/atomgit.egg-info/entry_points.txt +0 -0
- {atomgit-1.0.10 → atomgit-1.0.11}/atomgit.egg-info/requires.txt +0 -0
- {atomgit-1.0.10 → atomgit-1.0.11}/atomgit.egg-info/top_level.txt +0 -0
- {atomgit-1.0.10 → atomgit-1.0.11}/atomgit_hub.py +0 -0
- {atomgit-1.0.10 → atomgit-1.0.11}/cli.py +0 -0
- {atomgit-1.0.10 → atomgit-1.0.11}/config.py +0 -0
- {atomgit-1.0.10 → atomgit-1.0.11}/rate_limiter.py +0 -0
- {atomgit-1.0.10 → atomgit-1.0.11}/requirements.txt +0 -0
- {atomgit-1.0.10 → atomgit-1.0.11}/setup.cfg +0 -0
- {atomgit-1.0.10 → atomgit-1.0.11}/tests/test_download_real.py +0 -0
- {atomgit-1.0.10 → atomgit-1.0.11}/tests/test_endpoint_config.py +0 -0
- {atomgit-1.0.10 → atomgit-1.0.11}/tests/test_repo_id_normalize.py +0 -0
- {atomgit-1.0.10 → atomgit-1.0.11}/tests/test_repo_name_validation.py +0 -0
- {atomgit-1.0.10 → atomgit-1.0.11}/utils.py +0 -0
|
@@ -15,7 +15,12 @@ os.makedirs(cache_dir, exist_ok=True)
|
|
|
15
15
|
os.environ["HF_HOME"] = cache_dir
|
|
16
16
|
|
|
17
17
|
|
|
18
|
-
from huggingface_hub import hf_hub_download, upload_folder, create_repo,
|
|
18
|
+
from huggingface_hub import hf_hub_download, upload_folder, create_repo, constants as hf_constants
|
|
19
|
+
|
|
20
|
+
try:
|
|
21
|
+
from . import atomgit_hub
|
|
22
|
+
except ImportError:
|
|
23
|
+
import atomgit_hub
|
|
19
24
|
|
|
20
25
|
try:
|
|
21
26
|
from .rate_limiter import install_hf_rate_limiter
|
|
@@ -62,53 +67,6 @@ class HuggingFaceAPI:
|
|
|
62
67
|
# 二层或单层格式直接返回
|
|
63
68
|
return repo_id
|
|
64
69
|
|
|
65
|
-
def _probe_once(self, probe_id: str, token: Optional[str] = None):
|
|
66
|
-
"""探测一次仓库是否可访问,返回 (可访问, 原因)。
|
|
67
|
-
|
|
68
|
-
原因取值:ok / not_found / unauthorized / status=<code> / 探测失败
|
|
69
|
-
"""
|
|
70
|
-
endpoint = os.environ.get("HF_ENDPOINT", "https://hub.atomgit.com").rstrip("/")
|
|
71
|
-
headers = {"User-Agent": "atomgit-cli"}
|
|
72
|
-
if token:
|
|
73
|
-
headers["Authorization"] = f"Bearer {token}"
|
|
74
|
-
|
|
75
|
-
status = None
|
|
76
|
-
# 默认分支可能是 main 或 master,404 时再试另一个
|
|
77
|
-
for revision in ("main", "master"):
|
|
78
|
-
request = urllib.request.Request(
|
|
79
|
-
f"{endpoint}/api/models/{probe_id}/revision/{revision}", headers=headers
|
|
80
|
-
)
|
|
81
|
-
try:
|
|
82
|
-
with urllib.request.urlopen(request, timeout=15):
|
|
83
|
-
return True, "ok"
|
|
84
|
-
except urllib.error.HTTPError as e:
|
|
85
|
-
status = e.code
|
|
86
|
-
if status != 404:
|
|
87
|
-
break
|
|
88
|
-
except Exception as e:
|
|
89
|
-
# 网络抖动不阻断下载,交给 SDK 自己重试
|
|
90
|
-
return True, f"探测失败: {e}"
|
|
91
|
-
if status == 404:
|
|
92
|
-
return False, "not_found"
|
|
93
|
-
if status in (401, 403):
|
|
94
|
-
return False, "unauthorized"
|
|
95
|
-
return True, f"status={status}"
|
|
96
|
-
|
|
97
|
-
def _remote_repo_accessible(self, repo_id: str, token: Optional[str] = None):
|
|
98
|
-
"""确认远端仓库真的可访问。
|
|
99
|
-
|
|
100
|
-
huggingface_hub 的 snapshot_download 在远端 404 且 local_dir 已存在时,
|
|
101
|
-
会打印 "Returning existing local_dir ..." 并正常返回,
|
|
102
|
-
导致"下载不存在的仓库"被误报成功,所以下载前先探测一次。
|
|
103
|
-
"""
|
|
104
|
-
accessible, reason = self._probe_once(repo_id, None)
|
|
105
|
-
if accessible or reason == "not_found":
|
|
106
|
-
return accessible, reason
|
|
107
|
-
# 匿名探测失败时,若本地有令牌就再用令牌试一次
|
|
108
|
-
if token:
|
|
109
|
-
return self._probe_once(repo_id, token)
|
|
110
|
-
return accessible, reason
|
|
111
|
-
|
|
112
70
|
def login(self, token: str) -> bool:
|
|
113
71
|
"""登录验证"""
|
|
114
72
|
if not token or len(token) < 10:
|
|
@@ -262,20 +220,8 @@ class HuggingFaceAPI:
|
|
|
262
220
|
def download_repo(self, repo_id: str, local_path: Path = None, force_download: bool = False) -> bool:
|
|
263
221
|
"""下载仓库 - 使用Hugging Face Hub SDK,支持公开仓库无需token和断点续传"""
|
|
264
222
|
try:
|
|
265
|
-
#
|
|
223
|
+
# repo_id 交给 atomgit_hub 归一化(三层 -> dash),探测也在那边完成
|
|
266
224
|
normalized_repo_id = self._normalize_repo_id(repo_id)
|
|
267
|
-
|
|
268
|
-
# 远端不可访问且本地目录已存在时,SDK 会静默回退到本地目录并正常返回,
|
|
269
|
-
# 会把"上次残留"误报成下载成功,所以先探测远端仓库是否真的存在
|
|
270
|
-
credentials = config.get_credentials()
|
|
271
|
-
accessible, reason = self._remote_repo_accessible(
|
|
272
|
-
normalized_repo_id, credentials['token'] if credentials else None)
|
|
273
|
-
if not accessible:
|
|
274
|
-
if reason == "unauthorized":
|
|
275
|
-
print(f"❌ 仓库无权访问(令牌无效或权限不足): {repo_id}")
|
|
276
|
-
else:
|
|
277
|
-
print(f"❌ 仓库不存在: {repo_id}")
|
|
278
|
-
return False
|
|
279
225
|
|
|
280
226
|
if local_path is None:
|
|
281
227
|
local_path = Path.cwd() / repo_id.split('/')[-1]
|
|
@@ -283,14 +229,16 @@ class HuggingFaceAPI:
|
|
|
283
229
|
# 创建本地目录
|
|
284
230
|
local_path.mkdir(parents=True, exist_ok=True)
|
|
285
231
|
|
|
286
|
-
# 首先尝试不使用token下载(适用于公开仓库)
|
|
287
232
|
credentials = config.get_credentials()
|
|
288
233
|
try:
|
|
289
|
-
|
|
290
|
-
|
|
234
|
+
# 复用 atomgit_hub:内部会先探测仓库可访问性,令牌无效但仓库公开时自动降级匿名,
|
|
235
|
+
# 避免 huggingface_hub 在 403/404 时静默回退本地目录导致假成功
|
|
236
|
+
atomgit_hub.snapshot_download(
|
|
237
|
+
repo_id=repo_id,
|
|
291
238
|
local_dir=str(local_path),
|
|
292
|
-
force_download=force_download,
|
|
293
|
-
token=credentials['token'] if credentials and 'token' in credentials else None
|
|
239
|
+
force_download=force_download,
|
|
240
|
+
token=credentials['token'] if credentials and 'token' in credentials else None,
|
|
241
|
+
max_workers=8,
|
|
294
242
|
)
|
|
295
243
|
print(f"✅ 仓库下载成功")
|
|
296
244
|
return True
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|