atomgit 1.0.2__tar.gz → 1.0.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {atomgit-1.0.2 → atomgit-1.0.3}/PKG-INFO +1 -1
- {atomgit-1.0.2 → atomgit-1.0.3}/__init__.py +1 -1
- {atomgit-1.0.2 → atomgit-1.0.3}/api.py +18 -9
- {atomgit-1.0.2 → atomgit-1.0.3}/atomgit.egg-info/PKG-INFO +1 -1
- {atomgit-1.0.2 → atomgit-1.0.3}/cli.py +7 -4
- {atomgit-1.0.2 → atomgit-1.0.3}/setup.py +1 -1
- {atomgit-1.0.2 → atomgit-1.0.3}/test.py +4 -2
- {atomgit-1.0.2 → atomgit-1.0.3}/README.md +0 -0
- {atomgit-1.0.2 → atomgit-1.0.3}/__main__.py +0 -0
- {atomgit-1.0.2 → atomgit-1.0.3}/atomgit.egg-info/SOURCES.txt +0 -0
- {atomgit-1.0.2 → atomgit-1.0.3}/atomgit.egg-info/dependency_links.txt +0 -0
- {atomgit-1.0.2 → atomgit-1.0.3}/atomgit.egg-info/entry_points.txt +0 -0
- {atomgit-1.0.2 → atomgit-1.0.3}/atomgit.egg-info/requires.txt +0 -0
- {atomgit-1.0.2 → atomgit-1.0.3}/atomgit.egg-info/top_level.txt +0 -0
- {atomgit-1.0.2 → atomgit-1.0.3}/atomgit_hub.py +0 -0
- {atomgit-1.0.2 → atomgit-1.0.3}/config.py +0 -0
- {atomgit-1.0.2 → atomgit-1.0.3}/setup.cfg +0 -0
- {atomgit-1.0.2 → atomgit-1.0.3}/utils.py +0 -0
|
@@ -13,7 +13,12 @@ os.makedirs(cache_dir, exist_ok=True)
|
|
|
13
13
|
os.environ["HF_HOME"] = cache_dir
|
|
14
14
|
|
|
15
15
|
|
|
16
|
-
from huggingface_hub import hf_hub_download,
|
|
16
|
+
from huggingface_hub import hf_hub_download, create_repo, snapshot_download
|
|
17
|
+
|
|
18
|
+
try:
|
|
19
|
+
from .atomgit_hub import upload_folder as atomgit_upload_folder
|
|
20
|
+
except ImportError:
|
|
21
|
+
from atomgit_hub import upload_folder as atomgit_upload_folder
|
|
17
22
|
|
|
18
23
|
try:
|
|
19
24
|
from .config import config
|
|
@@ -121,7 +126,8 @@ class HuggingFaceAPI:
|
|
|
121
126
|
return False
|
|
122
127
|
|
|
123
128
|
def upload_folder(self, file_path: Path, repo_id: str,
|
|
124
|
-
remote_path: str = None, message: str = None
|
|
129
|
+
remote_path: str = None, message: str = None,
|
|
130
|
+
upload_timeout: float = 300.0) -> bool:
|
|
125
131
|
"""上传文件 - 使用Hugging Face Hub SDK"""
|
|
126
132
|
try:
|
|
127
133
|
if not file_path.exists():
|
|
@@ -147,13 +153,14 @@ class HuggingFaceAPI:
|
|
|
147
153
|
# 复制文件到临时目录
|
|
148
154
|
import shutil
|
|
149
155
|
shutil.copy2(file_path, target_file)
|
|
150
|
-
# 使用
|
|
156
|
+
# 使用AtomGit Hub SDK上传整个目录
|
|
151
157
|
commit_message = message or "Upload folder using atomgit client"
|
|
152
|
-
|
|
158
|
+
atomgit_upload_folder(
|
|
153
159
|
repo_id=repo_id,
|
|
154
160
|
folder_path=str(temp_dir),
|
|
155
161
|
token=credentials['token'],
|
|
156
|
-
commit_message=commit_message
|
|
162
|
+
commit_message=commit_message,
|
|
163
|
+
upload_timeout=upload_timeout
|
|
157
164
|
)
|
|
158
165
|
return True
|
|
159
166
|
finally:
|
|
@@ -166,7 +173,8 @@ class HuggingFaceAPI:
|
|
|
166
173
|
return False
|
|
167
174
|
|
|
168
175
|
def upload_directory(self, dir_path: Path, repo_id: str,
|
|
169
|
-
message: str = None, progress_callback=None
|
|
176
|
+
message: str = None, progress_callback=None,
|
|
177
|
+
upload_timeout: float = 300.0) -> bool:
|
|
170
178
|
"""上传目录 - 使用Hugging Face Hub SDK"""
|
|
171
179
|
try:
|
|
172
180
|
if not dir_path.exists() or not dir_path.is_dir():
|
|
@@ -178,13 +186,14 @@ class HuggingFaceAPI:
|
|
|
178
186
|
print("未找到登录凭证")
|
|
179
187
|
return False
|
|
180
188
|
|
|
181
|
-
# 直接使用
|
|
189
|
+
# 直接使用AtomGit Hub SDK上传目录
|
|
182
190
|
commit_message = message or "Upload folder using atomgit client"
|
|
183
|
-
|
|
191
|
+
atomgit_upload_folder(
|
|
184
192
|
repo_id=repo_id,
|
|
185
193
|
folder_path=str(dir_path),
|
|
186
194
|
token=credentials['token'],
|
|
187
|
-
commit_message=commit_message
|
|
195
|
+
commit_message=commit_message,
|
|
196
|
+
upload_timeout=upload_timeout
|
|
188
197
|
)
|
|
189
198
|
|
|
190
199
|
return True
|
|
@@ -37,7 +37,7 @@ except ImportError:
|
|
|
37
37
|
|
|
38
38
|
|
|
39
39
|
@click.group()
|
|
40
|
-
@click.version_option(version='1.0.
|
|
40
|
+
@click.version_option(version='1.0.3')
|
|
41
41
|
def cli():
|
|
42
42
|
"""AtomGit CLI - 基于Transformers和Hugging Face Hub的AtomGit平台模型文件上传下载工具"""
|
|
43
43
|
pass
|
|
@@ -158,7 +158,9 @@ def create(repo_name, repo_type, private):
|
|
|
158
158
|
@click.argument('path', type=click.Path(exists=True))
|
|
159
159
|
@click.option('--repo-id', required=True, help='目标仓库ID (username/repo-name)')
|
|
160
160
|
@click.option('--message', '-m', default='', help='上传说明')
|
|
161
|
-
|
|
161
|
+
@click.option('--timeout', '-t', 'timeout_sec', default=300, type=float,
|
|
162
|
+
help='上传超时时间(秒),默认300秒(5分钟)。大数据集建议增大此值')
|
|
163
|
+
def upload(path, repo_id, message, timeout_sec):
|
|
162
164
|
"""上传文件或目录到仓库"""
|
|
163
165
|
if not config.is_logged_in():
|
|
164
166
|
print_error("请先登录:atomgit login")
|
|
@@ -178,7 +180,7 @@ def upload(path, repo_id, message):
|
|
|
178
180
|
file_size = format_file_size(path.stat().st_size)
|
|
179
181
|
print_info(f"文件大小: {file_size}")
|
|
180
182
|
|
|
181
|
-
if api.upload_folder(path, repo_id, message=message):
|
|
183
|
+
if api.upload_folder(path, repo_id, message=message, upload_timeout=timeout_sec):
|
|
182
184
|
print_success(f"文件上传成功: {path.name}")
|
|
183
185
|
else:
|
|
184
186
|
print_error(f"文件上传失败: {path.name}")
|
|
@@ -191,8 +193,9 @@ def upload(path, repo_id, message):
|
|
|
191
193
|
print_info(f"正在上传目录: {path}")
|
|
192
194
|
print_info(f"文件数量: {file_count}")
|
|
193
195
|
print_info(f"目录大小: {dir_size}")
|
|
196
|
+
print_info(f"超时设置: {timeout_sec}秒")
|
|
194
197
|
|
|
195
|
-
if api.upload_directory(path, repo_id, message=message):
|
|
198
|
+
if api.upload_directory(path, repo_id, message=message, upload_timeout=timeout_sec):
|
|
196
199
|
print_success(f"目录上传成功: {path}")
|
|
197
200
|
else:
|
|
198
201
|
print_error(f"目录上传失败: {path}")
|
|
@@ -44,7 +44,8 @@ def test_upload_folder(repo_id: str, folder_path: str):
|
|
|
44
44
|
repo_id=repo_id,
|
|
45
45
|
token=None, # 使用保存的token
|
|
46
46
|
commit_message="Test upload with all parameters",
|
|
47
|
-
commit_description="Full parameter test for upload_folder function"
|
|
47
|
+
commit_description="Full parameter test for upload_folder function",
|
|
48
|
+
upload_timeout=100
|
|
48
49
|
)
|
|
49
50
|
print(f"✅ 上传文件夹成功: {result}")
|
|
50
51
|
except Exception as e:
|
|
@@ -129,9 +130,10 @@ def run_all_tests():
|
|
|
129
130
|
# 2. 测试upload
|
|
130
131
|
# test_upload_folder(repo_id=model, folder_path=model_folder_path)
|
|
131
132
|
# test_upload_folder(repo_id=dataset, folder_path=dataset_folder_path)
|
|
133
|
+
test_upload_folder(repo_id="yanlp/dataset-t1", folder_path="/Users/yanlp/csdn/IdeaProjects/gitcode/gitcode-hf-registry/dataset-t11")
|
|
132
134
|
|
|
133
135
|
# 3. 测试下载
|
|
134
|
-
test_snapshot_download(repo_id=model, local_dir="./test_downloads/model/snapshot_full")
|
|
136
|
+
# test_snapshot_download(repo_id=model, local_dir="./test_downloads/model/snapshot_full")
|
|
135
137
|
# test_snapshot_download(repo_id="hf_mirrors/Qwen/Qwen3-0.6B", local_dir="./test_downloads/dataset/snapshot_full2")
|
|
136
138
|
|
|
137
139
|
# 4. 测试下载文件
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|