atomgit 1.0.7__tar.gz → 1.0.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {atomgit-1.0.7 → atomgit-1.0.8}/PKG-INFO +1 -1
- {atomgit-1.0.7 → atomgit-1.0.8}/__init__.py +1 -1
- {atomgit-1.0.7 → atomgit-1.0.8}/atomgit.egg-info/PKG-INFO +1 -1
- {atomgit-1.0.7 → atomgit-1.0.8}/atomgit_hub.py +66 -40
- {atomgit-1.0.7 → atomgit-1.0.8}/cli.py +1 -1
- {atomgit-1.0.7 → atomgit-1.0.8}/setup.py +1 -1
- {atomgit-1.0.7 → atomgit-1.0.8}/MANIFEST.in +0 -0
- {atomgit-1.0.7 → atomgit-1.0.8}/README.md +0 -0
- {atomgit-1.0.7 → atomgit-1.0.8}/__main__.py +0 -0
- {atomgit-1.0.7 → atomgit-1.0.8}/api.py +0 -0
- {atomgit-1.0.7 → atomgit-1.0.8}/atomgit.egg-info/SOURCES.txt +0 -0
- {atomgit-1.0.7 → atomgit-1.0.8}/atomgit.egg-info/dependency_links.txt +0 -0
- {atomgit-1.0.7 → atomgit-1.0.8}/atomgit.egg-info/entry_points.txt +0 -0
- {atomgit-1.0.7 → atomgit-1.0.8}/atomgit.egg-info/requires.txt +0 -0
- {atomgit-1.0.7 → atomgit-1.0.8}/atomgit.egg-info/top_level.txt +0 -0
- {atomgit-1.0.7 → atomgit-1.0.8}/config.py +0 -0
- {atomgit-1.0.7 → atomgit-1.0.8}/rate_limiter.py +0 -0
- {atomgit-1.0.7 → atomgit-1.0.8}/requirements.txt +0 -0
- {atomgit-1.0.7 → atomgit-1.0.8}/setup.cfg +0 -0
- {atomgit-1.0.7 → atomgit-1.0.8}/test.py +0 -0
- {atomgit-1.0.7 → atomgit-1.0.8}/tests/test_rate_limiter.py +0 -0
- {atomgit-1.0.7 → atomgit-1.0.8}/utils.py +0 -0
|
@@ -22,7 +22,7 @@ os.makedirs(cache_dir, exist_ok=True)
|
|
|
22
22
|
os.environ["HF_HOME"] = cache_dir
|
|
23
23
|
|
|
24
24
|
from huggingface_hub import snapshot_download as hf_snapshot_download
|
|
25
|
-
from huggingface_hub import hf_hub_download, upload_folder as hf_upload_folder, create_repo
|
|
25
|
+
from huggingface_hub import hf_hub_download, upload_file as hf_upload_file, upload_folder as hf_upload_folder, create_repo
|
|
26
26
|
try:
|
|
27
27
|
from .rate_limiter import install_hf_rate_limiter
|
|
28
28
|
except ImportError:
|
|
@@ -291,18 +291,21 @@ def upload_folder(
|
|
|
291
291
|
upload_timeout: float = 300.0,
|
|
292
292
|
) -> str:
|
|
293
293
|
"""
|
|
294
|
-
|
|
295
|
-
|
|
294
|
+
上传文件或文件夹到AtomGit Hub
|
|
295
|
+
|
|
296
|
+
自动判断传入路径是文件还是文件夹,两者都支持。
|
|
297
|
+
|
|
296
298
|
参数:
|
|
297
|
-
folder_path (str 或 Path):
|
|
299
|
+
folder_path (str 或 Path): 本地文件或文件夹路径
|
|
298
300
|
repo_id (str): 仓库ID
|
|
299
301
|
token (str, 可选): 认证token
|
|
300
302
|
repo_type (str, 可选): 仓库类型
|
|
301
303
|
revision (str, 可选): 分支名
|
|
302
304
|
commit_message (str, 可选): 提交消息
|
|
303
305
|
commit_description (str, 可选): 提交描述
|
|
304
|
-
path_in_repo (str, 可选):
|
|
305
|
-
|
|
306
|
+
path_in_repo (str, 可选): 在仓库中的路径,默认为根目录。
|
|
307
|
+
若传入的是文件,文件名会自动拼接到该路径末尾。
|
|
308
|
+
ignore_patterns (List[str], 可选): 要忽略的文件模式(仅文件夹上传生效)
|
|
306
309
|
upload_timeout (float, 可选): 上传超时时间(秒),默认60秒(1分钟)。
|
|
307
310
|
对于大文件,服务器处理响应可能需要较长时间。
|
|
308
311
|
|
|
@@ -313,6 +316,7 @@ def upload_folder(
|
|
|
313
316
|
>>> upload_folder("./my-model/", "username/repo-name")
|
|
314
317
|
>>> upload_folder("./data/", "username/repo", path_in_repo="datasets/")
|
|
315
318
|
>>> upload_folder("./big-files/", "username/repo", upload_timeout=1200.0) # 20分钟超时
|
|
319
|
+
>>> upload_folder("./weights.pth", "username/repo", path_in_repo="checkpoints/") # 上传单个文件
|
|
316
320
|
"""
|
|
317
321
|
# 标准化仓库ID
|
|
318
322
|
normalized_repo_id = _normalize_repo_id(repo_id)
|
|
@@ -321,10 +325,10 @@ def upload_folder(
|
|
|
321
325
|
folder_path = Path(folder_path)
|
|
322
326
|
|
|
323
327
|
if not folder_path.exists():
|
|
324
|
-
raise FileNotFoundError(f"
|
|
325
|
-
|
|
326
|
-
if not folder_path.is_dir():
|
|
327
|
-
raise
|
|
328
|
+
raise FileNotFoundError(f"路径不存在: {folder_path}")
|
|
329
|
+
|
|
330
|
+
if not folder_path.is_file() and not folder_path.is_dir():
|
|
331
|
+
raise ValueError(f"路径既不是文件也不是目录: {folder_path}")
|
|
328
332
|
|
|
329
333
|
# 如果没有提供token,尝试使用保存的token
|
|
330
334
|
if token is None:
|
|
@@ -332,42 +336,62 @@ def upload_folder(
|
|
|
332
336
|
if token is None:
|
|
333
337
|
raise Exception("上传需要认证token,请先使用 'atomgit login' 登录,或提供token参数。")
|
|
334
338
|
|
|
335
|
-
#
|
|
339
|
+
# 使用 Monkey Patch 方式临时修改 huggingface_hub 的默认超时配置
|
|
340
|
+
from huggingface_hub import constants as hf_constants
|
|
336
341
|
import tempfile
|
|
337
342
|
import shutil
|
|
338
343
|
|
|
339
|
-
if path_in_repo == "./" or path_in_repo == "." or path_in_repo == "":
|
|
340
|
-
# 如果要上传到根目录,直接使用源文件夹
|
|
341
|
-
upload_path = str(folder_path)
|
|
342
|
-
else:
|
|
343
|
-
# 如果要上传到特定路径,需要重新组织目录结构。
|
|
344
|
-
# 注意:不能提前退出 with TemporaryDirectory(),否则上传时临时目录已被删除。
|
|
345
|
-
temp_dir = tempfile.mkdtemp(prefix="atomgit_upload_")
|
|
346
|
-
try:
|
|
347
|
-
temp_path = Path(temp_dir)
|
|
348
|
-
|
|
349
|
-
# 创建目标路径
|
|
350
|
-
target_path = temp_path / path_in_repo.strip('./')
|
|
351
|
-
target_path.parent.mkdir(parents=True, exist_ok=True)
|
|
352
|
-
|
|
353
|
-
# 复制整个目录树
|
|
354
|
-
shutil.copytree(folder_path, target_path, dirs_exist_ok=True)
|
|
355
|
-
|
|
356
|
-
except Exception:
|
|
357
|
-
shutil.rmtree(temp_dir, ignore_errors=True)
|
|
358
|
-
raise
|
|
359
|
-
|
|
360
|
-
upload_path = str(temp_path)
|
|
361
|
-
# 使用 Monkey Patch 方式临时修改 huggingface_hub 的默认超时配置
|
|
362
|
-
from huggingface_hub import constants as hf_constants
|
|
363
|
-
|
|
364
344
|
# 保存原始超时配置
|
|
365
345
|
original_timeout = hf_constants.DEFAULT_REQUEST_TIMEOUT
|
|
366
|
-
|
|
346
|
+
|
|
367
347
|
# 临时修改超时配置
|
|
368
348
|
hf_constants.DEFAULT_REQUEST_TIMEOUT = upload_timeout
|
|
369
|
-
|
|
349
|
+
|
|
350
|
+
temp_dir = None
|
|
370
351
|
try:
|
|
352
|
+
# 上传单个文件
|
|
353
|
+
if folder_path.is_file():
|
|
354
|
+
# path_in_repo 按目录语义处理(与文件夹分支一致,去掉首尾 ./ 和 /),
|
|
355
|
+
# 文件名自动拼接到末尾
|
|
356
|
+
clean_path = path_in_repo.strip().strip("./")
|
|
357
|
+
if not clean_path:
|
|
358
|
+
target_path = folder_path.name
|
|
359
|
+
else:
|
|
360
|
+
target_path = clean_path + "/" + folder_path.name
|
|
361
|
+
commit_msg = commit_message or f"Upload file {folder_path.name}"
|
|
362
|
+
result = hf_upload_file(
|
|
363
|
+
repo_id=normalized_repo_id,
|
|
364
|
+
path_or_fileobj=str(folder_path),
|
|
365
|
+
path_in_repo=target_path,
|
|
366
|
+
token=token,
|
|
367
|
+
commit_message=commit_msg
|
|
368
|
+
)
|
|
369
|
+
return result
|
|
370
|
+
|
|
371
|
+
# 上传文件夹:直接使用原始目录,或者创建临时目录来重新组织结构
|
|
372
|
+
if path_in_repo == "./" or path_in_repo == "." or path_in_repo == "":
|
|
373
|
+
# 如果要上传到根目录,直接使用源文件夹
|
|
374
|
+
upload_path = str(folder_path)
|
|
375
|
+
else:
|
|
376
|
+
# 如果要上传到特定路径,需要重新组织目录结构。
|
|
377
|
+
# 注意:不能提前退出 with TemporaryDirectory(),否则上传时临时目录已被删除。
|
|
378
|
+
temp_dir = tempfile.mkdtemp(prefix="atomgit_upload_")
|
|
379
|
+
try:
|
|
380
|
+
temp_path = Path(temp_dir)
|
|
381
|
+
|
|
382
|
+
# 创建目标路径
|
|
383
|
+
target_path = temp_path / path_in_repo.strip('./')
|
|
384
|
+
target_path.parent.mkdir(parents=True, exist_ok=True)
|
|
385
|
+
|
|
386
|
+
# 复制整个目录树
|
|
387
|
+
shutil.copytree(folder_path, target_path, dirs_exist_ok=True)
|
|
388
|
+
|
|
389
|
+
except Exception:
|
|
390
|
+
shutil.rmtree(temp_dir, ignore_errors=True)
|
|
391
|
+
raise
|
|
392
|
+
|
|
393
|
+
upload_path = str(temp_path)
|
|
394
|
+
|
|
371
395
|
# 使用huggingface_hub的upload_folder上传
|
|
372
396
|
commit_msg = commit_message or f"Upload folder {folder_path.name}"
|
|
373
397
|
result = hf_upload_folder(
|
|
@@ -376,7 +400,7 @@ def upload_folder(
|
|
|
376
400
|
token=token,
|
|
377
401
|
commit_message=commit_msg
|
|
378
402
|
)
|
|
379
|
-
|
|
403
|
+
|
|
380
404
|
return result
|
|
381
405
|
|
|
382
406
|
except Exception as e:
|
|
@@ -388,8 +412,10 @@ def upload_folder(
|
|
|
388
412
|
else:
|
|
389
413
|
raise Exception(f"上传失败:{error_msg}")
|
|
390
414
|
finally:
|
|
415
|
+
# 恢复原始超时配置
|
|
416
|
+
hf_constants.DEFAULT_REQUEST_TIMEOUT = original_timeout
|
|
391
417
|
# 清理临时目录(仅重新组织目录结构时创建)
|
|
392
|
-
if
|
|
418
|
+
if temp_dir is not None:
|
|
393
419
|
shutil.rmtree(temp_dir, ignore_errors=True)
|
|
394
420
|
|
|
395
421
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|