atomgit 1.0.6__tar.gz → 1.0.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {atomgit-1.0.6 → atomgit-1.0.8}/PKG-INFO +1 -1
- {atomgit-1.0.6 → atomgit-1.0.8}/__init__.py +1 -1
- {atomgit-1.0.6 → atomgit-1.0.8}/atomgit.egg-info/PKG-INFO +1 -1
- {atomgit-1.0.6 → atomgit-1.0.8}/atomgit_hub.py +73 -34
- {atomgit-1.0.6 → atomgit-1.0.8}/cli.py +1 -1
- {atomgit-1.0.6 → atomgit-1.0.8}/setup.py +1 -1
- {atomgit-1.0.6 → atomgit-1.0.8}/MANIFEST.in +0 -0
- {atomgit-1.0.6 → atomgit-1.0.8}/README.md +0 -0
- {atomgit-1.0.6 → atomgit-1.0.8}/__main__.py +0 -0
- {atomgit-1.0.6 → atomgit-1.0.8}/api.py +0 -0
- {atomgit-1.0.6 → atomgit-1.0.8}/atomgit.egg-info/SOURCES.txt +0 -0
- {atomgit-1.0.6 → atomgit-1.0.8}/atomgit.egg-info/dependency_links.txt +0 -0
- {atomgit-1.0.6 → atomgit-1.0.8}/atomgit.egg-info/entry_points.txt +0 -0
- {atomgit-1.0.6 → atomgit-1.0.8}/atomgit.egg-info/requires.txt +0 -0
- {atomgit-1.0.6 → atomgit-1.0.8}/atomgit.egg-info/top_level.txt +0 -0
- {atomgit-1.0.6 → atomgit-1.0.8}/config.py +0 -0
- {atomgit-1.0.6 → atomgit-1.0.8}/rate_limiter.py +0 -0
- {atomgit-1.0.6 → atomgit-1.0.8}/requirements.txt +0 -0
- {atomgit-1.0.6 → atomgit-1.0.8}/setup.cfg +0 -0
- {atomgit-1.0.6 → atomgit-1.0.8}/test.py +0 -0
- {atomgit-1.0.6 → atomgit-1.0.8}/tests/test_rate_limiter.py +0 -0
- {atomgit-1.0.6 → atomgit-1.0.8}/utils.py +0 -0
|
@@ -22,11 +22,14 @@ os.makedirs(cache_dir, exist_ok=True)
|
|
|
22
22
|
os.environ["HF_HOME"] = cache_dir
|
|
23
23
|
|
|
24
24
|
from huggingface_hub import snapshot_download as hf_snapshot_download
|
|
25
|
-
from huggingface_hub import hf_hub_download, upload_folder as hf_upload_folder, create_repo
|
|
25
|
+
from huggingface_hub import hf_hub_download, upload_file as hf_upload_file, upload_folder as hf_upload_folder, create_repo
|
|
26
26
|
try:
|
|
27
27
|
from .rate_limiter import install_hf_rate_limiter
|
|
28
28
|
except ImportError:
|
|
29
|
-
|
|
29
|
+
try:
|
|
30
|
+
from rate_limiter import install_hf_rate_limiter
|
|
31
|
+
except ImportError:
|
|
32
|
+
from atomgit.rate_limiter import install_hf_rate_limiter
|
|
30
33
|
|
|
31
34
|
install_hf_rate_limiter()
|
|
32
35
|
|
|
@@ -288,18 +291,21 @@ def upload_folder(
|
|
|
288
291
|
upload_timeout: float = 300.0,
|
|
289
292
|
) -> str:
|
|
290
293
|
"""
|
|
291
|
-
|
|
292
|
-
|
|
294
|
+
上传文件或文件夹到AtomGit Hub
|
|
295
|
+
|
|
296
|
+
自动判断传入路径是文件还是文件夹,两者都支持。
|
|
297
|
+
|
|
293
298
|
参数:
|
|
294
|
-
folder_path (str 或 Path):
|
|
299
|
+
folder_path (str 或 Path): 本地文件或文件夹路径
|
|
295
300
|
repo_id (str): 仓库ID
|
|
296
301
|
token (str, 可选): 认证token
|
|
297
302
|
repo_type (str, 可选): 仓库类型
|
|
298
303
|
revision (str, 可选): 分支名
|
|
299
304
|
commit_message (str, 可选): 提交消息
|
|
300
305
|
commit_description (str, 可选): 提交描述
|
|
301
|
-
path_in_repo (str, 可选):
|
|
302
|
-
|
|
306
|
+
path_in_repo (str, 可选): 在仓库中的路径,默认为根目录。
|
|
307
|
+
若传入的是文件,文件名会自动拼接到该路径末尾。
|
|
308
|
+
ignore_patterns (List[str], 可选): 要忽略的文件模式(仅文件夹上传生效)
|
|
303
309
|
upload_timeout (float, 可选): 上传超时时间(秒),默认60秒(1分钟)。
|
|
304
310
|
对于大文件,服务器处理响应可能需要较长时间。
|
|
305
311
|
|
|
@@ -310,6 +316,7 @@ def upload_folder(
|
|
|
310
316
|
>>> upload_folder("./my-model/", "username/repo-name")
|
|
311
317
|
>>> upload_folder("./data/", "username/repo", path_in_repo="datasets/")
|
|
312
318
|
>>> upload_folder("./big-files/", "username/repo", upload_timeout=1200.0) # 20分钟超时
|
|
319
|
+
>>> upload_folder("./weights.pth", "username/repo", path_in_repo="checkpoints/") # 上传单个文件
|
|
313
320
|
"""
|
|
314
321
|
# 标准化仓库ID
|
|
315
322
|
normalized_repo_id = _normalize_repo_id(repo_id)
|
|
@@ -318,10 +325,10 @@ def upload_folder(
|
|
|
318
325
|
folder_path = Path(folder_path)
|
|
319
326
|
|
|
320
327
|
if not folder_path.exists():
|
|
321
|
-
raise FileNotFoundError(f"
|
|
322
|
-
|
|
323
|
-
if not folder_path.is_dir():
|
|
324
|
-
raise
|
|
328
|
+
raise FileNotFoundError(f"路径不存在: {folder_path}")
|
|
329
|
+
|
|
330
|
+
if not folder_path.is_file() and not folder_path.is_dir():
|
|
331
|
+
raise ValueError(f"路径既不是文件也不是目录: {folder_path}")
|
|
325
332
|
|
|
326
333
|
# 如果没有提供token,尝试使用保存的token
|
|
327
334
|
if token is None:
|
|
@@ -329,36 +336,62 @@ def upload_folder(
|
|
|
329
336
|
if token is None:
|
|
330
337
|
raise Exception("上传需要认证token,请先使用 'atomgit login' 登录,或提供token参数。")
|
|
331
338
|
|
|
332
|
-
#
|
|
339
|
+
# 使用 Monkey Patch 方式临时修改 huggingface_hub 的默认超时配置
|
|
340
|
+
from huggingface_hub import constants as hf_constants
|
|
333
341
|
import tempfile
|
|
334
342
|
import shutil
|
|
335
343
|
|
|
336
|
-
if path_in_repo == "./" or path_in_repo == "." or path_in_repo == "":
|
|
337
|
-
# 如果要上传到根目录,直接使用源文件夹
|
|
338
|
-
upload_path = str(folder_path)
|
|
339
|
-
else:
|
|
340
|
-
# 如果要上传到特定路径,需要重新组织目录结构
|
|
341
|
-
with tempfile.TemporaryDirectory() as temp_dir:
|
|
342
|
-
temp_path = Path(temp_dir)
|
|
343
|
-
|
|
344
|
-
# 创建目标路径
|
|
345
|
-
target_path = temp_path / path_in_repo.strip('./')
|
|
346
|
-
target_path.parent.mkdir(parents=True, exist_ok=True)
|
|
347
|
-
|
|
348
|
-
# 复制整个目录树
|
|
349
|
-
shutil.copytree(folder_path, target_path, dirs_exist_ok=True)
|
|
350
|
-
|
|
351
|
-
upload_path = str(temp_path)
|
|
352
|
-
# 使用 Monkey Patch 方式临时修改 huggingface_hub 的默认超时配置
|
|
353
|
-
from huggingface_hub import constants as hf_constants
|
|
354
|
-
|
|
355
344
|
# 保存原始超时配置
|
|
356
345
|
original_timeout = hf_constants.DEFAULT_REQUEST_TIMEOUT
|
|
357
|
-
|
|
346
|
+
|
|
358
347
|
# 临时修改超时配置
|
|
359
348
|
hf_constants.DEFAULT_REQUEST_TIMEOUT = upload_timeout
|
|
360
|
-
|
|
349
|
+
|
|
350
|
+
temp_dir = None
|
|
361
351
|
try:
|
|
352
|
+
# 上传单个文件
|
|
353
|
+
if folder_path.is_file():
|
|
354
|
+
# path_in_repo 按目录语义处理(与文件夹分支一致,去掉首尾 ./ 和 /),
|
|
355
|
+
# 文件名自动拼接到末尾
|
|
356
|
+
clean_path = path_in_repo.strip().strip("./")
|
|
357
|
+
if not clean_path:
|
|
358
|
+
target_path = folder_path.name
|
|
359
|
+
else:
|
|
360
|
+
target_path = clean_path + "/" + folder_path.name
|
|
361
|
+
commit_msg = commit_message or f"Upload file {folder_path.name}"
|
|
362
|
+
result = hf_upload_file(
|
|
363
|
+
repo_id=normalized_repo_id,
|
|
364
|
+
path_or_fileobj=str(folder_path),
|
|
365
|
+
path_in_repo=target_path,
|
|
366
|
+
token=token,
|
|
367
|
+
commit_message=commit_msg
|
|
368
|
+
)
|
|
369
|
+
return result
|
|
370
|
+
|
|
371
|
+
# 上传文件夹:直接使用原始目录,或者创建临时目录来重新组织结构
|
|
372
|
+
if path_in_repo == "./" or path_in_repo == "." or path_in_repo == "":
|
|
373
|
+
# 如果要上传到根目录,直接使用源文件夹
|
|
374
|
+
upload_path = str(folder_path)
|
|
375
|
+
else:
|
|
376
|
+
# 如果要上传到特定路径,需要重新组织目录结构。
|
|
377
|
+
# 注意:不能提前退出 with TemporaryDirectory(),否则上传时临时目录已被删除。
|
|
378
|
+
temp_dir = tempfile.mkdtemp(prefix="atomgit_upload_")
|
|
379
|
+
try:
|
|
380
|
+
temp_path = Path(temp_dir)
|
|
381
|
+
|
|
382
|
+
# 创建目标路径
|
|
383
|
+
target_path = temp_path / path_in_repo.strip('./')
|
|
384
|
+
target_path.parent.mkdir(parents=True, exist_ok=True)
|
|
385
|
+
|
|
386
|
+
# 复制整个目录树
|
|
387
|
+
shutil.copytree(folder_path, target_path, dirs_exist_ok=True)
|
|
388
|
+
|
|
389
|
+
except Exception:
|
|
390
|
+
shutil.rmtree(temp_dir, ignore_errors=True)
|
|
391
|
+
raise
|
|
392
|
+
|
|
393
|
+
upload_path = str(temp_path)
|
|
394
|
+
|
|
362
395
|
# 使用huggingface_hub的upload_folder上传
|
|
363
396
|
commit_msg = commit_message or f"Upload folder {folder_path.name}"
|
|
364
397
|
result = hf_upload_folder(
|
|
@@ -367,7 +400,7 @@ def upload_folder(
|
|
|
367
400
|
token=token,
|
|
368
401
|
commit_message=commit_msg
|
|
369
402
|
)
|
|
370
|
-
|
|
403
|
+
|
|
371
404
|
return result
|
|
372
405
|
|
|
373
406
|
except Exception as e:
|
|
@@ -378,6 +411,12 @@ def upload_folder(
|
|
|
378
411
|
raise Exception(f"仓库不存在:{repo_id}")
|
|
379
412
|
else:
|
|
380
413
|
raise Exception(f"上传失败:{error_msg}")
|
|
414
|
+
finally:
|
|
415
|
+
# 恢复原始超时配置
|
|
416
|
+
hf_constants.DEFAULT_REQUEST_TIMEOUT = original_timeout
|
|
417
|
+
# 清理临时目录(仅重新组织目录结构时创建)
|
|
418
|
+
if temp_dir is not None:
|
|
419
|
+
shutil.rmtree(temp_dir, ignore_errors=True)
|
|
381
420
|
|
|
382
421
|
|
|
383
422
|
def create_repository(
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|