atomgit 1.0.2__tar.gz → 1.0.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {atomgit-1.0.2 → atomgit-1.0.4}/PKG-INFO +1 -1
- {atomgit-1.0.2 → atomgit-1.0.4}/__init__.py +1 -1
- {atomgit-1.0.2 → atomgit-1.0.4}/api.py +10 -5
- {atomgit-1.0.2 → atomgit-1.0.4}/atomgit.egg-info/PKG-INFO +1 -1
- {atomgit-1.0.2 → atomgit-1.0.4}/cli.py +7 -4
- {atomgit-1.0.2 → atomgit-1.0.4}/setup.py +1 -1
- {atomgit-1.0.2 → atomgit-1.0.4}/test.py +4 -2
- {atomgit-1.0.2 → atomgit-1.0.4}/README.md +0 -0
- {atomgit-1.0.2 → atomgit-1.0.4}/__main__.py +0 -0
- {atomgit-1.0.2 → atomgit-1.0.4}/atomgit.egg-info/SOURCES.txt +0 -0
- {atomgit-1.0.2 → atomgit-1.0.4}/atomgit.egg-info/dependency_links.txt +0 -0
- {atomgit-1.0.2 → atomgit-1.0.4}/atomgit.egg-info/entry_points.txt +0 -0
- {atomgit-1.0.2 → atomgit-1.0.4}/atomgit.egg-info/requires.txt +0 -0
- {atomgit-1.0.2 → atomgit-1.0.4}/atomgit.egg-info/top_level.txt +0 -0
- {atomgit-1.0.2 → atomgit-1.0.4}/atomgit_hub.py +0 -0
- {atomgit-1.0.2 → atomgit-1.0.4}/config.py +0 -0
- {atomgit-1.0.2 → atomgit-1.0.4}/setup.cfg +0 -0
- {atomgit-1.0.2 → atomgit-1.0.4}/utils.py +0 -0
|
@@ -13,7 +13,7 @@ os.makedirs(cache_dir, exist_ok=True)
|
|
|
13
13
|
os.environ["HF_HOME"] = cache_dir
|
|
14
14
|
|
|
15
15
|
|
|
16
|
-
from huggingface_hub import hf_hub_download, upload_folder, create_repo, snapshot_download,
|
|
16
|
+
from huggingface_hub import hf_hub_download, upload_folder, create_repo, snapshot_download, constants as hf_constants
|
|
17
17
|
|
|
18
18
|
try:
|
|
19
19
|
from .config import config
|
|
@@ -121,7 +121,8 @@ class HuggingFaceAPI:
|
|
|
121
121
|
return False
|
|
122
122
|
|
|
123
123
|
def upload_folder(self, file_path: Path, repo_id: str,
|
|
124
|
-
remote_path: str = None, message: str = None
|
|
124
|
+
remote_path: str = None, message: str = None,
|
|
125
|
+
upload_timeout: float = 300.0) -> bool:
|
|
125
126
|
"""上传文件 - 使用Hugging Face Hub SDK"""
|
|
126
127
|
try:
|
|
127
128
|
if not file_path.exists():
|
|
@@ -147,14 +148,16 @@ class HuggingFaceAPI:
|
|
|
147
148
|
# 复制文件到临时目录
|
|
148
149
|
import shutil
|
|
149
150
|
shutil.copy2(file_path, target_file)
|
|
150
|
-
# 使用
|
|
151
|
+
# 使用 Monkey Patch 方式临时修改 huggingface_hub 的默认超时配置
|
|
151
152
|
commit_message = message or "Upload folder using atomgit client"
|
|
153
|
+
hf_constants.DEFAULT_REQUEST_TIMEOUT = upload_timeout
|
|
152
154
|
upload_folder(
|
|
153
155
|
repo_id=repo_id,
|
|
154
156
|
folder_path=str(temp_dir),
|
|
155
157
|
token=credentials['token'],
|
|
156
158
|
commit_message=commit_message
|
|
157
159
|
)
|
|
160
|
+
|
|
158
161
|
return True
|
|
159
162
|
finally:
|
|
160
163
|
# 清理临时目录
|
|
@@ -166,7 +169,8 @@ class HuggingFaceAPI:
|
|
|
166
169
|
return False
|
|
167
170
|
|
|
168
171
|
def upload_directory(self, dir_path: Path, repo_id: str,
|
|
169
|
-
message: str = None, progress_callback=None
|
|
172
|
+
message: str = None, progress_callback=None,
|
|
173
|
+
upload_timeout: float = 300.0) -> bool:
|
|
170
174
|
"""上传目录 - 使用Hugging Face Hub SDK"""
|
|
171
175
|
try:
|
|
172
176
|
if not dir_path.exists() or not dir_path.is_dir():
|
|
@@ -178,8 +182,9 @@ class HuggingFaceAPI:
|
|
|
178
182
|
print("未找到登录凭证")
|
|
179
183
|
return False
|
|
180
184
|
|
|
181
|
-
#
|
|
185
|
+
# 使用 Monkey Patch 方式临时修改 huggingface_hub 的默认超时配置
|
|
182
186
|
commit_message = message or "Upload folder using atomgit client"
|
|
187
|
+
hf_constants.DEFAULT_REQUEST_TIMEOUT = upload_timeout
|
|
183
188
|
upload_folder(
|
|
184
189
|
repo_id=repo_id,
|
|
185
190
|
folder_path=str(dir_path),
|
|
@@ -37,7 +37,7 @@ except ImportError:
|
|
|
37
37
|
|
|
38
38
|
|
|
39
39
|
@click.group()
|
|
40
|
-
@click.version_option(version='1.0.
|
|
40
|
+
@click.version_option(version='1.0.4')
|
|
41
41
|
def cli():
|
|
42
42
|
"""AtomGit CLI - 基于Transformers和Hugging Face Hub的AtomGit平台模型文件上传下载工具"""
|
|
43
43
|
pass
|
|
@@ -158,7 +158,9 @@ def create(repo_name, repo_type, private):
|
|
|
158
158
|
@click.argument('path', type=click.Path(exists=True))
|
|
159
159
|
@click.option('--repo-id', required=True, help='目标仓库ID (username/repo-name)')
|
|
160
160
|
@click.option('--message', '-m', default='', help='上传说明')
|
|
161
|
-
|
|
161
|
+
@click.option('--timeout', '-t', 'timeout_sec', default=300, type=float,
|
|
162
|
+
help='上传超时时间(秒),默认300秒(5分钟)。大数据集建议增大此值')
|
|
163
|
+
def upload(path, repo_id, message, timeout_sec):
|
|
162
164
|
"""上传文件或目录到仓库"""
|
|
163
165
|
if not config.is_logged_in():
|
|
164
166
|
print_error("请先登录:atomgit login")
|
|
@@ -178,7 +180,7 @@ def upload(path, repo_id, message):
|
|
|
178
180
|
file_size = format_file_size(path.stat().st_size)
|
|
179
181
|
print_info(f"文件大小: {file_size}")
|
|
180
182
|
|
|
181
|
-
if api.upload_folder(path, repo_id, message=message):
|
|
183
|
+
if api.upload_folder(path, repo_id, message=message, upload_timeout=timeout_sec):
|
|
182
184
|
print_success(f"文件上传成功: {path.name}")
|
|
183
185
|
else:
|
|
184
186
|
print_error(f"文件上传失败: {path.name}")
|
|
@@ -191,8 +193,9 @@ def upload(path, repo_id, message):
|
|
|
191
193
|
print_info(f"正在上传目录: {path}")
|
|
192
194
|
print_info(f"文件数量: {file_count}")
|
|
193
195
|
print_info(f"目录大小: {dir_size}")
|
|
196
|
+
print_info(f"超时设置: {timeout_sec}秒")
|
|
194
197
|
|
|
195
|
-
if api.upload_directory(path, repo_id, message=message):
|
|
198
|
+
if api.upload_directory(path, repo_id, message=message, upload_timeout=timeout_sec):
|
|
196
199
|
print_success(f"目录上传成功: {path}")
|
|
197
200
|
else:
|
|
198
201
|
print_error(f"目录上传失败: {path}")
|
|
@@ -44,7 +44,8 @@ def test_upload_folder(repo_id: str, folder_path: str):
|
|
|
44
44
|
repo_id=repo_id,
|
|
45
45
|
token=None, # 使用保存的token
|
|
46
46
|
commit_message="Test upload with all parameters",
|
|
47
|
-
commit_description="Full parameter test for upload_folder function"
|
|
47
|
+
commit_description="Full parameter test for upload_folder function",
|
|
48
|
+
upload_timeout=100
|
|
48
49
|
)
|
|
49
50
|
print(f"✅ 上传文件夹成功: {result}")
|
|
50
51
|
except Exception as e:
|
|
@@ -129,9 +130,10 @@ def run_all_tests():
|
|
|
129
130
|
# 2. 测试upload
|
|
130
131
|
# test_upload_folder(repo_id=model, folder_path=model_folder_path)
|
|
131
132
|
# test_upload_folder(repo_id=dataset, folder_path=dataset_folder_path)
|
|
133
|
+
test_upload_folder(repo_id="yanlp/dataset-t1", folder_path="/Users/yanlp/csdn/IdeaProjects/gitcode/gitcode-hf-registry/dataset-t11")
|
|
132
134
|
|
|
133
135
|
# 3. 测试下载
|
|
134
|
-
test_snapshot_download(repo_id=model, local_dir="./test_downloads/model/snapshot_full")
|
|
136
|
+
# test_snapshot_download(repo_id=model, local_dir="./test_downloads/model/snapshot_full")
|
|
135
137
|
# test_snapshot_download(repo_id="hf_mirrors/Qwen/Qwen3-0.6B", local_dir="./test_downloads/dataset/snapshot_full2")
|
|
136
138
|
|
|
137
139
|
# 4. 测试下载文件
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|