xiaoshiai-hub 1.0.1__tar.gz → 1.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {xiaoshiai_hub-1.0.1/xiaoshiai_hub.egg-info → xiaoshiai_hub-1.1.1}/PKG-INFO +177 -50
- {xiaoshiai_hub-1.0.1 → xiaoshiai_hub-1.1.1}/README.md +176 -49
- {xiaoshiai_hub-1.0.1 → xiaoshiai_hub-1.1.1}/pyproject.toml +4 -4
- {xiaoshiai_hub-1.0.1 → xiaoshiai_hub-1.1.1}/setup.py +7 -7
- {xiaoshiai_hub-1.0.1 → xiaoshiai_hub-1.1.1}/xiaoshiai_hub/__init__.py +18 -3
- xiaoshiai_hub-1.1.1/xiaoshiai_hub/auth.py +163 -0
- xiaoshiai_hub-1.1.1/xiaoshiai_hub/cli.py +496 -0
- {xiaoshiai_hub-1.0.1 → xiaoshiai_hub-1.1.1}/xiaoshiai_hub/client.py +57 -10
- {xiaoshiai_hub-1.0.1 → xiaoshiai_hub-1.1.1}/xiaoshiai_hub/download.py +21 -3
- xiaoshiai_hub-1.1.1/xiaoshiai_hub/envelope_crypto.py +237 -0
- {xiaoshiai_hub-1.0.1 → xiaoshiai_hub-1.1.1}/xiaoshiai_hub/upload.py +50 -63
- {xiaoshiai_hub-1.0.1 → xiaoshiai_hub-1.1.1/xiaoshiai_hub.egg-info}/PKG-INFO +177 -50
- {xiaoshiai_hub-1.0.1 → xiaoshiai_hub-1.1.1}/xiaoshiai_hub.egg-info/SOURCES.txt +4 -0
- xiaoshiai_hub-1.1.1/xiaoshiai_hub.egg-info/entry_points.txt +2 -0
- {xiaoshiai_hub-1.0.1 → xiaoshiai_hub-1.1.1}/LICENSE +0 -0
- {xiaoshiai_hub-1.0.1 → xiaoshiai_hub-1.1.1}/MANIFEST.in +0 -0
- {xiaoshiai_hub-1.0.1 → xiaoshiai_hub-1.1.1}/requirements.txt +0 -0
- {xiaoshiai_hub-1.0.1 → xiaoshiai_hub-1.1.1}/setup.cfg +0 -0
- {xiaoshiai_hub-1.0.1 → xiaoshiai_hub-1.1.1}/xiaoshiai_hub/exceptions.py +0 -0
- {xiaoshiai_hub-1.0.1 → xiaoshiai_hub-1.1.1}/xiaoshiai_hub/types.py +0 -0
- {xiaoshiai_hub-1.0.1 → xiaoshiai_hub-1.1.1}/xiaoshiai_hub.egg-info/dependency_links.txt +0 -0
- {xiaoshiai_hub-1.0.1 → xiaoshiai_hub-1.1.1}/xiaoshiai_hub.egg-info/not-zip-safe +0 -0
- {xiaoshiai_hub-1.0.1 → xiaoshiai_hub-1.1.1}/xiaoshiai_hub.egg-info/requires.txt +0 -0
- {xiaoshiai_hub-1.0.1 → xiaoshiai_hub-1.1.1}/xiaoshiai_hub.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: xiaoshiai-hub
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.1.1
|
|
4
4
|
Summary: Python SDK for XiaoShi AI Hub - Upload, download, and manage AI models and datasets with xpai-enc encryption support
|
|
5
5
|
Home-page: https://github.com/poxiaoyun/moha-sdk
|
|
6
6
|
Author: XiaoShi AI
|
|
@@ -59,15 +59,16 @@ Dynamic: requires-python
|
|
|
59
59
|
[](https://pypi.org/project/xiaoshiai-hub/)
|
|
60
60
|
[](https://opensource.org/licenses/Apache-2.0)
|
|
61
61
|
|
|
62
|
-
XiaoShi AI Hub Python SDK 是一个功能强大的 Python 库,用于与 XiaoShi AI Hub 平台进行交互。它提供了简单易用的 API
|
|
62
|
+
XiaoShi AI Hub Python SDK 是一个功能强大的 Python 库,用于与 XiaoShi AI Hub 平台进行交互。它提供了简单易用的 API 和命令行工具,支持模型和数据集的上传、下载,并支持大型模型文件的透明加密功能。
|
|
63
63
|
|
|
64
64
|
|
|
65
65
|
## ✨ 特性
|
|
66
66
|
|
|
67
67
|
- 🚀 **简单易用** - 类似 Hugging Face Hub 的 API 设计,上手即用
|
|
68
|
+
- 🖥️ **命令行工具** - 提供 `moha` CLI,无需编写代码即可上传下载
|
|
68
69
|
- 📥 **下载功能** - 支持下载单个文件或整个仓库
|
|
69
70
|
- 📤 **上传功能** - 支持上传文件和文件夹到仓库
|
|
70
|
-
- 🔐 **智能加密** -
|
|
71
|
+
- 🔐 **智能加密** - 自动加密大型模型文件(≥5MB 的 .safetensors、.bin、.pt、.pth、.ckpt 文件)
|
|
71
72
|
- 🎯 **模式匹配** - 支持使用 allow/ignore 模式过滤文件
|
|
72
73
|
- 📊 **进度显示** - 下载和上传时显示进度条
|
|
73
74
|
- 🔑 **多种认证** - 支持用户名/密码和 Token 认证
|
|
@@ -84,19 +85,36 @@ XiaoShi AI Hub Python SDK 是一个功能强大的 Python 库,用于与 XiaoSh
|
|
|
84
85
|
pip install xiaoshiai-hub
|
|
85
86
|
```
|
|
86
87
|
|
|
87
|
-
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
## 🚀 快速开始
|
|
91
|
+
|
|
92
|
+
### 命令行工具 (CLI)
|
|
93
|
+
|
|
94
|
+
安装后即可使用 `moha` 命令行工具:
|
|
88
95
|
|
|
89
96
|
```bash
|
|
90
|
-
#
|
|
91
|
-
|
|
97
|
+
# 查看帮助
|
|
98
|
+
moha --help
|
|
92
99
|
|
|
93
|
-
#
|
|
94
|
-
|
|
100
|
+
# 上传文件夹到仓库
|
|
101
|
+
moha upload ./my_model org/my-model --username your-username --password your-password
|
|
102
|
+
|
|
103
|
+
# 上传单个文件
|
|
104
|
+
moha upload-file ./config.yaml org/my-model --username your-username --password your-password
|
|
105
|
+
|
|
106
|
+
# 下载整个仓库
|
|
107
|
+
moha download org/my-model --username your-username --password your-password
|
|
108
|
+
|
|
109
|
+
# 下载单个文件
|
|
110
|
+
moha download-file org/my-model config.yaml --username your-username --password your-password
|
|
95
111
|
```
|
|
96
112
|
|
|
97
|
-
|
|
113
|
+
详细的 CLI 使用说明请参考 [命令行工具](#-命令行工具-cli) 章节。
|
|
98
114
|
|
|
99
|
-
###
|
|
115
|
+
### Python API
|
|
116
|
+
|
|
117
|
+
#### 下载单个文件
|
|
100
118
|
|
|
101
119
|
```python
|
|
102
120
|
from xiaoshiai_hub import moha_hub_download
|
|
@@ -112,7 +130,7 @@ file_path = moha_hub_download(
|
|
|
112
130
|
print(f"文件已下载到: {file_path}")
|
|
113
131
|
```
|
|
114
132
|
|
|
115
|
-
|
|
133
|
+
#### 下载整个仓库
|
|
116
134
|
|
|
117
135
|
```python
|
|
118
136
|
from xiaoshiai_hub import snapshot_download
|
|
@@ -127,7 +145,7 @@ repo_path = snapshot_download(
|
|
|
127
145
|
print(f"仓库已下载到: {repo_path}")
|
|
128
146
|
```
|
|
129
147
|
|
|
130
|
-
|
|
148
|
+
#### 使用过滤器下载
|
|
131
149
|
|
|
132
150
|
```python
|
|
133
151
|
from xiaoshiai_hub import snapshot_download
|
|
@@ -142,7 +160,7 @@ repo_path = snapshot_download(
|
|
|
142
160
|
)
|
|
143
161
|
```
|
|
144
162
|
|
|
145
|
-
|
|
163
|
+
#### 上传文件
|
|
146
164
|
|
|
147
165
|
```python
|
|
148
166
|
from xiaoshiai_hub import upload_file
|
|
@@ -160,7 +178,7 @@ result = upload_file(
|
|
|
160
178
|
print(f"上传成功: {result}")
|
|
161
179
|
```
|
|
162
180
|
|
|
163
|
-
|
|
181
|
+
#### 上传文件夹
|
|
164
182
|
|
|
165
183
|
```python
|
|
166
184
|
from xiaoshiai_hub import upload_folder
|
|
@@ -178,7 +196,7 @@ result = upload_folder(
|
|
|
178
196
|
print(f"上传成功: {result}")
|
|
179
197
|
```
|
|
180
198
|
|
|
181
|
-
|
|
199
|
+
#### 加密上传
|
|
182
200
|
|
|
183
201
|
SDK 会自动加密大型模型文件(≥5MB 的 .safetensors、.bin、.pt、.pth、.ckpt 文件):
|
|
184
202
|
|
|
@@ -197,7 +215,7 @@ result = upload_file(
|
|
|
197
215
|
)
|
|
198
216
|
```
|
|
199
217
|
|
|
200
|
-
|
|
218
|
+
#### 上传文件夹(自动加密大文件)
|
|
201
219
|
|
|
202
220
|
```python
|
|
203
221
|
from xiaoshiai_hub import upload_folder
|
|
@@ -212,10 +230,10 @@ result = upload_folder(
|
|
|
212
230
|
username="your-username",
|
|
213
231
|
password="your-password",
|
|
214
232
|
)
|
|
215
|
-
|
|
233
|
+
|
|
216
234
|
```
|
|
217
235
|
|
|
218
|
-
|
|
236
|
+
#### 使用 HubClient API
|
|
219
237
|
|
|
220
238
|
```python
|
|
221
239
|
from xiaoshiai_hub import HubClient
|
|
@@ -244,7 +262,7 @@ for entry in content.entries:
|
|
|
244
262
|
|
|
245
263
|
## 🔐 加密功能
|
|
246
264
|
|
|
247
|
-
SDK
|
|
265
|
+
SDK 提供了智能加密功能,使用 AES-256-CTR 算法对大型模型文件进行加密。
|
|
248
266
|
|
|
249
267
|
### 自动加密规则
|
|
250
268
|
|
|
@@ -255,10 +273,6 @@ SDK 集成了 [xpai-enc](https://github.com/poxiaoyun/xpai-enc) 提供的智能
|
|
|
255
273
|
|
|
256
274
|
小文件和其他类型的文件(如配置文件、README 等)不会被加密,保持可读性。
|
|
257
275
|
|
|
258
|
-
### 加密清单文件
|
|
259
|
-
|
|
260
|
-
加密后会自动生成 `xpai_encryption_manifest.enc` 清单文件,记录哪些文件被加密了。此文件会自动上传到仓库。
|
|
261
|
-
|
|
262
276
|
### 使用加密功能
|
|
263
277
|
|
|
264
278
|
```python
|
|
@@ -275,7 +289,6 @@ result = upload_folder(
|
|
|
275
289
|
|
|
276
290
|
# 文件夹中的大型模型文件(如 model.safetensors)会被自动加密
|
|
277
291
|
# 小文件(如 config.json、README.md)保持原样
|
|
278
|
-
# 加密清单 xpai_encryption_manifest.enc 会自动上传
|
|
279
292
|
```
|
|
280
293
|
|
|
281
294
|
### 临时目录管理
|
|
@@ -300,10 +313,148 @@ result = upload_folder(
|
|
|
300
313
|
|
|
301
314
|
```bash
|
|
302
315
|
# Hub 服务端点
|
|
303
|
-
export MOHA_ENDPOINT="https://your-hub-url.com
|
|
316
|
+
export MOHA_ENDPOINT="https://your-hub-url.com"
|
|
317
|
+
|
|
318
|
+
# 认证信息(可选,避免每次输入)
|
|
319
|
+
export MOHA_USERNAME="your-username"
|
|
320
|
+
export MOHA_PASSWORD="your-password"
|
|
321
|
+
export MOHA_TOKEN="your-token"
|
|
322
|
+
|
|
323
|
+
# 加密密码(可选)
|
|
324
|
+
export MOHA_ENCRYPTION_PASSWORD="your-encryption-password"
|
|
325
|
+
```
|
|
326
|
+
|
|
327
|
+
## 🖥️ 命令行工具 (CLI)
|
|
328
|
+
|
|
329
|
+
SDK 提供了 `moha` 命令行工具,支持常见的上传下载操作。
|
|
330
|
+
|
|
331
|
+
### 基本用法
|
|
332
|
+
|
|
333
|
+
```bash
|
|
334
|
+
moha --help
|
|
304
335
|
```
|
|
305
336
|
|
|
306
|
-
|
|
337
|
+
### 上传文件夹
|
|
338
|
+
|
|
339
|
+
```bash
|
|
340
|
+
# 基本用法
|
|
341
|
+
moha upload-folder ./my_model org/my-model
|
|
342
|
+
|
|
343
|
+
# 使用别名 upload
|
|
344
|
+
moha upload ./my_model org/my-model
|
|
345
|
+
|
|
346
|
+
# 完整参数示例
|
|
347
|
+
moha upload ./my_model org/my-model \
|
|
348
|
+
--repo-type models \
|
|
349
|
+
--revision main \
|
|
350
|
+
--message "Upload model files" \
|
|
351
|
+
--ignore "*.log" \
|
|
352
|
+
--ignore ".git*" \
|
|
353
|
+
--username your-username \
|
|
354
|
+
--password your-password
|
|
355
|
+
|
|
356
|
+
# 启用加密
|
|
357
|
+
moha upload ./my_model org/my-model \
|
|
358
|
+
--encrypt \
|
|
359
|
+
--encryption-password "your-secret" \
|
|
360
|
+
--username your-username \
|
|
361
|
+
--password your-password
|
|
362
|
+
```
|
|
363
|
+
|
|
364
|
+
### 上传单个文件
|
|
365
|
+
|
|
366
|
+
```bash
|
|
367
|
+
# 基本用法(使用文件名作为仓库路径)
|
|
368
|
+
moha upload-file ./config.yaml org/my-model
|
|
369
|
+
|
|
370
|
+
# 指定仓库中的路径
|
|
371
|
+
moha upload-file ./config.yaml org/my-model \
|
|
372
|
+
--path-in-repo configs/config.yaml
|
|
373
|
+
|
|
374
|
+
# 完整参数示例
|
|
375
|
+
moha upload-file ./model.safetensors org/my-model \
|
|
376
|
+
--path-in-repo weights/model.safetensors \
|
|
377
|
+
--repo-type models \
|
|
378
|
+
--revision main \
|
|
379
|
+
--message "Upload model weights" \
|
|
380
|
+
--encrypt \
|
|
381
|
+
--encryption-password "your-secret" \
|
|
382
|
+
--username your-username \
|
|
383
|
+
--password your-password
|
|
384
|
+
```
|
|
385
|
+
|
|
386
|
+
### 下载仓库
|
|
387
|
+
|
|
388
|
+
```bash
|
|
389
|
+
# 基本用法
|
|
390
|
+
moha download org/my-model
|
|
391
|
+
|
|
392
|
+
# 使用别名 download-repo
|
|
393
|
+
moha download-repo org/my-model
|
|
394
|
+
|
|
395
|
+
# 完整参数示例
|
|
396
|
+
moha download org/my-model \
|
|
397
|
+
--local-dir ./downloaded_model \
|
|
398
|
+
--repo-type models \
|
|
399
|
+
--revision main \
|
|
400
|
+
--include "*.safetensors" \
|
|
401
|
+
--include "*.json" \
|
|
402
|
+
--ignore "*.log" \
|
|
403
|
+
--username your-username \
|
|
404
|
+
--password your-password
|
|
405
|
+
```
|
|
406
|
+
|
|
407
|
+
### 下载单个文件
|
|
408
|
+
|
|
409
|
+
```bash
|
|
410
|
+
# 基本用法
|
|
411
|
+
moha download-file org/my-model config.yaml
|
|
412
|
+
|
|
413
|
+
# 完整参数示例
|
|
414
|
+
moha download-file org/my-model model.safetensors \
|
|
415
|
+
--local-dir ./downloads \
|
|
416
|
+
--repo-type models \
|
|
417
|
+
--revision main \
|
|
418
|
+
--username your-username \
|
|
419
|
+
--password your-password
|
|
420
|
+
```
|
|
421
|
+
|
|
422
|
+
### CLI 参数说明
|
|
423
|
+
|
|
424
|
+
| 参数 | 说明 | 适用命令 |
|
|
425
|
+
|------|------|----------|
|
|
426
|
+
| `--repo-type, -t` | 仓库类型:`models` 或 `datasets`(默认:models) | 所有 |
|
|
427
|
+
| `--revision, -r` | 分支/标签/提交(默认:main) | 所有 |
|
|
428
|
+
| `--base-url` | API 基础 URL(默认:环境变量 MOHA_ENDPOINT) | 所有 |
|
|
429
|
+
| `--token` | 认证令牌 | 所有 |
|
|
430
|
+
| `--username` | 用户名 | 所有 |
|
|
431
|
+
| `--password` | 密码 | 所有 |
|
|
432
|
+
| `--message, -m` | 提交消息 | upload, upload-file |
|
|
433
|
+
| `--ignore, -i` | 忽略模式(可多次使用) | upload, download |
|
|
434
|
+
| `--include` | 包含模式(可多次使用) | download |
|
|
435
|
+
| `--encrypt, -e` | 启用加密 | upload, upload-file |
|
|
436
|
+
| `--encryption-password` | 加密密码 | upload, upload-file |
|
|
437
|
+
| `--path-in-repo, -p` | 仓库中的文件路径 | upload-file |
|
|
438
|
+
| `--temp-dir` | 加密临时目录 | upload |
|
|
439
|
+
| `--local-dir, -o` | 本地保存目录 | download, download-file |
|
|
440
|
+
| `--quiet, -q` | 禁用进度条 | download, download-file |
|
|
441
|
+
|
|
442
|
+
### 使用环境变量
|
|
443
|
+
|
|
444
|
+
可以通过环境变量设置认证信息,避免每次输入:
|
|
445
|
+
|
|
446
|
+
```bash
|
|
447
|
+
# 设置环境变量
|
|
448
|
+
export MOHA_USERNAME="your-username"
|
|
449
|
+
export MOHA_PASSWORD="your-password"
|
|
450
|
+
export MOHA_ENCRYPTION_PASSWORD="your-secret"
|
|
451
|
+
|
|
452
|
+
# 然后直接使用命令
|
|
453
|
+
moha upload ./my_model org/my-model --encrypt
|
|
454
|
+
moha download org/my-model
|
|
455
|
+
```
|
|
456
|
+
|
|
457
|
+
## 📋 使用场景
|
|
307
458
|
|
|
308
459
|
### 场景 1: 上传开源模型到私有 Hub
|
|
309
460
|
|
|
@@ -422,32 +573,8 @@ venv\Scripts\activate # Windows
|
|
|
422
573
|
|
|
423
574
|
# 安装依赖
|
|
424
575
|
pip install -r requirements.txt
|
|
425
|
-
|
|
426
|
-
# 安装 xpai-enc(用于加密功能)
|
|
427
|
-
pip install git+https://github.com/poxiaoyun/xpai-enc.git
|
|
428
576
|
```
|
|
429
577
|
|
|
430
|
-
## 🔗 相关项目
|
|
431
|
-
|
|
432
|
-
- **[xpai-enc](https://github.com/poxiaoyun/xpai-enc)** - 模型文件加密/解密工具,提供透明的加密支持
|
|
433
|
-
|
|
434
|
-
## 📝 更新日志
|
|
435
|
-
|
|
436
|
-
### v0.1.3 (最新)
|
|
437
|
-
|
|
438
|
-
- ✅ 添加仓库存在性检查(上传前自动验证)
|
|
439
|
-
- ✅ 集成 xpai-enc 加密功能
|
|
440
|
-
- ✅ 自动加密大型模型文件(≥5MB)
|
|
441
|
-
- ✅ 自动生成和上传加密清单文件
|
|
442
|
-
- ✅ 改进错误处理和提示信息
|
|
443
|
-
|
|
444
|
-
### v0.1.0
|
|
445
|
-
|
|
446
|
-
- 🎉 初始版本发布
|
|
447
|
-
- ✅ 基础下载功能
|
|
448
|
-
- ✅ 基础上传功能
|
|
449
|
-
- ✅ HubClient API
|
|
450
|
-
|
|
451
578
|
## 🤝 贡献
|
|
452
579
|
|
|
453
580
|
欢迎贡献!请随时提交 Issue 或 Pull Request。
|
|
@@ -4,15 +4,16 @@
|
|
|
4
4
|
[](https://pypi.org/project/xiaoshiai-hub/)
|
|
5
5
|
[](https://opensource.org/licenses/Apache-2.0)
|
|
6
6
|
|
|
7
|
-
XiaoShi AI Hub Python SDK 是一个功能强大的 Python 库,用于与 XiaoShi AI Hub 平台进行交互。它提供了简单易用的 API
|
|
7
|
+
XiaoShi AI Hub Python SDK 是一个功能强大的 Python 库,用于与 XiaoShi AI Hub 平台进行交互。它提供了简单易用的 API 和命令行工具,支持模型和数据集的上传、下载,并支持大型模型文件的透明加密功能。
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
## ✨ 特性
|
|
11
11
|
|
|
12
12
|
- 🚀 **简单易用** - 类似 Hugging Face Hub 的 API 设计,上手即用
|
|
13
|
+
- 🖥️ **命令行工具** - 提供 `moha` CLI,无需编写代码即可上传下载
|
|
13
14
|
- 📥 **下载功能** - 支持下载单个文件或整个仓库
|
|
14
15
|
- 📤 **上传功能** - 支持上传文件和文件夹到仓库
|
|
15
|
-
- 🔐 **智能加密** -
|
|
16
|
+
- 🔐 **智能加密** - 自动加密大型模型文件(≥5MB 的 .safetensors、.bin、.pt、.pth、.ckpt 文件)
|
|
16
17
|
- 🎯 **模式匹配** - 支持使用 allow/ignore 模式过滤文件
|
|
17
18
|
- 📊 **进度显示** - 下载和上传时显示进度条
|
|
18
19
|
- 🔑 **多种认证** - 支持用户名/密码和 Token 认证
|
|
@@ -29,19 +30,36 @@ XiaoShi AI Hub Python SDK 是一个功能强大的 Python 库,用于与 XiaoSh
|
|
|
29
30
|
pip install xiaoshiai-hub
|
|
30
31
|
```
|
|
31
32
|
|
|
32
|
-
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
## 🚀 快速开始
|
|
36
|
+
|
|
37
|
+
### 命令行工具 (CLI)
|
|
38
|
+
|
|
39
|
+
安装后即可使用 `moha` 命令行工具:
|
|
33
40
|
|
|
34
41
|
```bash
|
|
35
|
-
#
|
|
36
|
-
|
|
42
|
+
# 查看帮助
|
|
43
|
+
moha --help
|
|
37
44
|
|
|
38
|
-
#
|
|
39
|
-
|
|
45
|
+
# 上传文件夹到仓库
|
|
46
|
+
moha upload ./my_model org/my-model --username your-username --password your-password
|
|
47
|
+
|
|
48
|
+
# 上传单个文件
|
|
49
|
+
moha upload-file ./config.yaml org/my-model --username your-username --password your-password
|
|
50
|
+
|
|
51
|
+
# 下载整个仓库
|
|
52
|
+
moha download org/my-model --username your-username --password your-password
|
|
53
|
+
|
|
54
|
+
# 下载单个文件
|
|
55
|
+
moha download-file org/my-model config.yaml --username your-username --password your-password
|
|
40
56
|
```
|
|
41
57
|
|
|
42
|
-
|
|
58
|
+
详细的 CLI 使用说明请参考 [命令行工具](#-命令行工具-cli) 章节。
|
|
43
59
|
|
|
44
|
-
###
|
|
60
|
+
### Python API
|
|
61
|
+
|
|
62
|
+
#### 下载单个文件
|
|
45
63
|
|
|
46
64
|
```python
|
|
47
65
|
from xiaoshiai_hub import moha_hub_download
|
|
@@ -57,7 +75,7 @@ file_path = moha_hub_download(
|
|
|
57
75
|
print(f"文件已下载到: {file_path}")
|
|
58
76
|
```
|
|
59
77
|
|
|
60
|
-
|
|
78
|
+
#### 下载整个仓库
|
|
61
79
|
|
|
62
80
|
```python
|
|
63
81
|
from xiaoshiai_hub import snapshot_download
|
|
@@ -72,7 +90,7 @@ repo_path = snapshot_download(
|
|
|
72
90
|
print(f"仓库已下载到: {repo_path}")
|
|
73
91
|
```
|
|
74
92
|
|
|
75
|
-
|
|
93
|
+
#### 使用过滤器下载
|
|
76
94
|
|
|
77
95
|
```python
|
|
78
96
|
from xiaoshiai_hub import snapshot_download
|
|
@@ -87,7 +105,7 @@ repo_path = snapshot_download(
|
|
|
87
105
|
)
|
|
88
106
|
```
|
|
89
107
|
|
|
90
|
-
|
|
108
|
+
#### 上传文件
|
|
91
109
|
|
|
92
110
|
```python
|
|
93
111
|
from xiaoshiai_hub import upload_file
|
|
@@ -105,7 +123,7 @@ result = upload_file(
|
|
|
105
123
|
print(f"上传成功: {result}")
|
|
106
124
|
```
|
|
107
125
|
|
|
108
|
-
|
|
126
|
+
#### 上传文件夹
|
|
109
127
|
|
|
110
128
|
```python
|
|
111
129
|
from xiaoshiai_hub import upload_folder
|
|
@@ -123,7 +141,7 @@ result = upload_folder(
|
|
|
123
141
|
print(f"上传成功: {result}")
|
|
124
142
|
```
|
|
125
143
|
|
|
126
|
-
|
|
144
|
+
#### 加密上传
|
|
127
145
|
|
|
128
146
|
SDK 会自动加密大型模型文件(≥5MB 的 .safetensors、.bin、.pt、.pth、.ckpt 文件):
|
|
129
147
|
|
|
@@ -142,7 +160,7 @@ result = upload_file(
|
|
|
142
160
|
)
|
|
143
161
|
```
|
|
144
162
|
|
|
145
|
-
|
|
163
|
+
#### 上传文件夹(自动加密大文件)
|
|
146
164
|
|
|
147
165
|
```python
|
|
148
166
|
from xiaoshiai_hub import upload_folder
|
|
@@ -157,10 +175,10 @@ result = upload_folder(
|
|
|
157
175
|
username="your-username",
|
|
158
176
|
password="your-password",
|
|
159
177
|
)
|
|
160
|
-
|
|
178
|
+
|
|
161
179
|
```
|
|
162
180
|
|
|
163
|
-
|
|
181
|
+
#### 使用 HubClient API
|
|
164
182
|
|
|
165
183
|
```python
|
|
166
184
|
from xiaoshiai_hub import HubClient
|
|
@@ -189,7 +207,7 @@ for entry in content.entries:
|
|
|
189
207
|
|
|
190
208
|
## 🔐 加密功能
|
|
191
209
|
|
|
192
|
-
SDK
|
|
210
|
+
SDK 提供了智能加密功能,使用 AES-256-CTR 算法对大型模型文件进行加密。
|
|
193
211
|
|
|
194
212
|
### 自动加密规则
|
|
195
213
|
|
|
@@ -200,10 +218,6 @@ SDK 集成了 [xpai-enc](https://github.com/poxiaoyun/xpai-enc) 提供的智能
|
|
|
200
218
|
|
|
201
219
|
小文件和其他类型的文件(如配置文件、README 等)不会被加密,保持可读性。
|
|
202
220
|
|
|
203
|
-
### 加密清单文件
|
|
204
|
-
|
|
205
|
-
加密后会自动生成 `xpai_encryption_manifest.enc` 清单文件,记录哪些文件被加密了。此文件会自动上传到仓库。
|
|
206
|
-
|
|
207
221
|
### 使用加密功能
|
|
208
222
|
|
|
209
223
|
```python
|
|
@@ -220,7 +234,6 @@ result = upload_folder(
|
|
|
220
234
|
|
|
221
235
|
# 文件夹中的大型模型文件(如 model.safetensors)会被自动加密
|
|
222
236
|
# 小文件(如 config.json、README.md)保持原样
|
|
223
|
-
# 加密清单 xpai_encryption_manifest.enc 会自动上传
|
|
224
237
|
```
|
|
225
238
|
|
|
226
239
|
### 临时目录管理
|
|
@@ -245,10 +258,148 @@ result = upload_folder(
|
|
|
245
258
|
|
|
246
259
|
```bash
|
|
247
260
|
# Hub 服务端点
|
|
248
|
-
export MOHA_ENDPOINT="https://your-hub-url.com
|
|
261
|
+
export MOHA_ENDPOINT="https://your-hub-url.com"
|
|
262
|
+
|
|
263
|
+
# 认证信息(可选,避免每次输入)
|
|
264
|
+
export MOHA_USERNAME="your-username"
|
|
265
|
+
export MOHA_PASSWORD="your-password"
|
|
266
|
+
export MOHA_TOKEN="your-token"
|
|
267
|
+
|
|
268
|
+
# 加密密码(可选)
|
|
269
|
+
export MOHA_ENCRYPTION_PASSWORD="your-encryption-password"
|
|
270
|
+
```
|
|
271
|
+
|
|
272
|
+
## 🖥️ 命令行工具 (CLI)
|
|
273
|
+
|
|
274
|
+
SDK 提供了 `moha` 命令行工具,支持常见的上传下载操作。
|
|
275
|
+
|
|
276
|
+
### 基本用法
|
|
277
|
+
|
|
278
|
+
```bash
|
|
279
|
+
moha --help
|
|
249
280
|
```
|
|
250
281
|
|
|
251
|
-
|
|
282
|
+
### 上传文件夹
|
|
283
|
+
|
|
284
|
+
```bash
|
|
285
|
+
# 基本用法
|
|
286
|
+
moha upload-folder ./my_model org/my-model
|
|
287
|
+
|
|
288
|
+
# 使用别名 upload
|
|
289
|
+
moha upload ./my_model org/my-model
|
|
290
|
+
|
|
291
|
+
# 完整参数示例
|
|
292
|
+
moha upload ./my_model org/my-model \
|
|
293
|
+
--repo-type models \
|
|
294
|
+
--revision main \
|
|
295
|
+
--message "Upload model files" \
|
|
296
|
+
--ignore "*.log" \
|
|
297
|
+
--ignore ".git*" \
|
|
298
|
+
--username your-username \
|
|
299
|
+
--password your-password
|
|
300
|
+
|
|
301
|
+
# 启用加密
|
|
302
|
+
moha upload ./my_model org/my-model \
|
|
303
|
+
--encrypt \
|
|
304
|
+
--encryption-password "your-secret" \
|
|
305
|
+
--username your-username \
|
|
306
|
+
--password your-password
|
|
307
|
+
```
|
|
308
|
+
|
|
309
|
+
### 上传单个文件
|
|
310
|
+
|
|
311
|
+
```bash
|
|
312
|
+
# 基本用法(使用文件名作为仓库路径)
|
|
313
|
+
moha upload-file ./config.yaml org/my-model
|
|
314
|
+
|
|
315
|
+
# 指定仓库中的路径
|
|
316
|
+
moha upload-file ./config.yaml org/my-model \
|
|
317
|
+
--path-in-repo configs/config.yaml
|
|
318
|
+
|
|
319
|
+
# 完整参数示例
|
|
320
|
+
moha upload-file ./model.safetensors org/my-model \
|
|
321
|
+
--path-in-repo weights/model.safetensors \
|
|
322
|
+
--repo-type models \
|
|
323
|
+
--revision main \
|
|
324
|
+
--message "Upload model weights" \
|
|
325
|
+
--encrypt \
|
|
326
|
+
--encryption-password "your-secret" \
|
|
327
|
+
--username your-username \
|
|
328
|
+
--password your-password
|
|
329
|
+
```
|
|
330
|
+
|
|
331
|
+
### 下载仓库
|
|
332
|
+
|
|
333
|
+
```bash
|
|
334
|
+
# 基本用法
|
|
335
|
+
moha download org/my-model
|
|
336
|
+
|
|
337
|
+
# 使用别名 download-repo
|
|
338
|
+
moha download-repo org/my-model
|
|
339
|
+
|
|
340
|
+
# 完整参数示例
|
|
341
|
+
moha download org/my-model \
|
|
342
|
+
--local-dir ./downloaded_model \
|
|
343
|
+
--repo-type models \
|
|
344
|
+
--revision main \
|
|
345
|
+
--include "*.safetensors" \
|
|
346
|
+
--include "*.json" \
|
|
347
|
+
--ignore "*.log" \
|
|
348
|
+
--username your-username \
|
|
349
|
+
--password your-password
|
|
350
|
+
```
|
|
351
|
+
|
|
352
|
+
### 下载单个文件
|
|
353
|
+
|
|
354
|
+
```bash
|
|
355
|
+
# 基本用法
|
|
356
|
+
moha download-file org/my-model config.yaml
|
|
357
|
+
|
|
358
|
+
# 完整参数示例
|
|
359
|
+
moha download-file org/my-model model.safetensors \
|
|
360
|
+
--local-dir ./downloads \
|
|
361
|
+
--repo-type models \
|
|
362
|
+
--revision main \
|
|
363
|
+
--username your-username \
|
|
364
|
+
--password your-password
|
|
365
|
+
```
|
|
366
|
+
|
|
367
|
+
### CLI 参数说明
|
|
368
|
+
|
|
369
|
+
| 参数 | 说明 | 适用命令 |
|
|
370
|
+
|------|------|----------|
|
|
371
|
+
| `--repo-type, -t` | 仓库类型:`models` 或 `datasets`(默认:models) | 所有 |
|
|
372
|
+
| `--revision, -r` | 分支/标签/提交(默认:main) | 所有 |
|
|
373
|
+
| `--base-url` | API 基础 URL(默认:环境变量 MOHA_ENDPOINT) | 所有 |
|
|
374
|
+
| `--token` | 认证令牌 | 所有 |
|
|
375
|
+
| `--username` | 用户名 | 所有 |
|
|
376
|
+
| `--password` | 密码 | 所有 |
|
|
377
|
+
| `--message, -m` | 提交消息 | upload, upload-file |
|
|
378
|
+
| `--ignore, -i` | 忽略模式(可多次使用) | upload, download |
|
|
379
|
+
| `--include` | 包含模式(可多次使用) | download |
|
|
380
|
+
| `--encrypt, -e` | 启用加密 | upload, upload-file |
|
|
381
|
+
| `--encryption-password` | 加密密码 | upload, upload-file |
|
|
382
|
+
| `--path-in-repo, -p` | 仓库中的文件路径 | upload-file |
|
|
383
|
+
| `--temp-dir` | 加密临时目录 | upload |
|
|
384
|
+
| `--local-dir, -o` | 本地保存目录 | download, download-file |
|
|
385
|
+
| `--quiet, -q` | 禁用进度条 | download, download-file |
|
|
386
|
+
|
|
387
|
+
### 使用环境变量
|
|
388
|
+
|
|
389
|
+
可以通过环境变量设置认证信息,避免每次输入:
|
|
390
|
+
|
|
391
|
+
```bash
|
|
392
|
+
# 设置环境变量
|
|
393
|
+
export MOHA_USERNAME="your-username"
|
|
394
|
+
export MOHA_PASSWORD="your-password"
|
|
395
|
+
export MOHA_ENCRYPTION_PASSWORD="your-secret"
|
|
396
|
+
|
|
397
|
+
# 然后直接使用命令
|
|
398
|
+
moha upload ./my_model org/my-model --encrypt
|
|
399
|
+
moha download org/my-model
|
|
400
|
+
```
|
|
401
|
+
|
|
402
|
+
## 📋 使用场景
|
|
252
403
|
|
|
253
404
|
### 场景 1: 上传开源模型到私有 Hub
|
|
254
405
|
|
|
@@ -367,32 +518,8 @@ venv\Scripts\activate # Windows
|
|
|
367
518
|
|
|
368
519
|
# 安装依赖
|
|
369
520
|
pip install -r requirements.txt
|
|
370
|
-
|
|
371
|
-
# 安装 xpai-enc(用于加密功能)
|
|
372
|
-
pip install git+https://github.com/poxiaoyun/xpai-enc.git
|
|
373
521
|
```
|
|
374
522
|
|
|
375
|
-
## 🔗 相关项目
|
|
376
|
-
|
|
377
|
-
- **[xpai-enc](https://github.com/poxiaoyun/xpai-enc)** - 模型文件加密/解密工具,提供透明的加密支持
|
|
378
|
-
|
|
379
|
-
## 📝 更新日志
|
|
380
|
-
|
|
381
|
-
### v0.1.3 (最新)
|
|
382
|
-
|
|
383
|
-
- ✅ 添加仓库存在性检查(上传前自动验证)
|
|
384
|
-
- ✅ 集成 xpai-enc 加密功能
|
|
385
|
-
- ✅ 自动加密大型模型文件(≥5MB)
|
|
386
|
-
- ✅ 自动生成和上传加密清单文件
|
|
387
|
-
- ✅ 改进错误处理和提示信息
|
|
388
|
-
|
|
389
|
-
### v0.1.0
|
|
390
|
-
|
|
391
|
-
- 🎉 初始版本发布
|
|
392
|
-
- ✅ 基础下载功能
|
|
393
|
-
- ✅ 基础上传功能
|
|
394
|
-
- ✅ HubClient API
|
|
395
|
-
|
|
396
523
|
## 🤝 贡献
|
|
397
524
|
|
|
398
525
|
欢迎贡献!请随时提交 Issue 或 Pull Request。
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "xiaoshiai-hub"
|
|
7
|
-
version = "1.
|
|
7
|
+
version = "1.1.1"
|
|
8
8
|
description = "Python SDK for XiaoShi AI Hub - Upload, download, and manage AI models and datasets with xpai-enc encryption support"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.7"
|
|
@@ -54,9 +54,6 @@ dependencies = [
|
|
|
54
54
|
upload = [
|
|
55
55
|
"gitpython>=3.1.0",
|
|
56
56
|
]
|
|
57
|
-
|
|
58
|
-
# Encryption support (xpai-enc must be installed separately)
|
|
59
|
-
# Install with: pip install git+https://github.com/poxiaoyun/xpai-enc.git
|
|
60
57
|
encryption = []
|
|
61
58
|
|
|
62
59
|
# Development dependencies
|
|
@@ -83,6 +80,9 @@ Homepage = "https://github.com/poxiaoyun/moha-sdk"
|
|
|
83
80
|
Repository = "https://github.com/poxiaoyun/moha-sdk"
|
|
84
81
|
Issues = "https://github.com/poxiaoyun/moha-sdk/issues"
|
|
85
82
|
|
|
83
|
+
[project.scripts]
|
|
84
|
+
moha = "xiaoshiai_hub.cli:main"
|
|
85
|
+
|
|
86
86
|
[tool.black]
|
|
87
87
|
line-length = 100
|
|
88
88
|
target-version = ['py37', 'py38', 'py39', 'py310', 'py311']
|