hfms-bridge 0.4.0__tar.gz

hfms_bridge-0.4.0/.github/workflows/test.yml

@@ -0,0 +1,19 @@
+name: tests
+on: [push, pull_request]
+jobs:
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14']
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+        with:
+          python-version: ${{ matrix.python }}
+      - run: uv sync --locked
+      - run: uv run ruff check .
+      - run: uv run pytest
+      - run: uv build

hfms_bridge-0.4.0/.gitignore

@@ -0,0 +1,15 @@
+.venv/
+.hfms-bridge/
+__pycache__/
+*.py[cod]
+.pytest_cache/
+.ruff_cache/
+dist/
+*.egg-info/
+.env
+
+# OS
+.DS_Store
+
+# IDE
+.vscode

hfms_bridge-0.4.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 hfms-bridge contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

hfms_bridge-0.4.0/PKG-INFO

@@ -0,0 +1,99 @@
+Metadata-Version: 2.4
+Name: hfms-bridge
+Version: 0.4.0
+Summary: Stream models and datasets between HuggingFace and ModelScope
+Project-URL: Homepage, https://github.com/feixyz10/hfms-bridge/
+Project-URL: Source, https://github.com/feixyz10/hfms-bridge/
+Project-URL: Issues, https://github.com/feixyz10/hfms-bridge/issues
+Author: ExcitingMe
+License-Expression: MIT
+License-File: LICENSE
+Keywords: cli,datasets,huggingface,models,modelscope,streaming,transfer
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Utilities
+Requires-Python: >=3.9
+Requires-Dist: filelock<4,>=3.16
+Requires-Dist: requests<3,>=2.32.3
+Requires-Dist: tqdm<5,>=4.66
+Description-Content-Type: text/markdown
+
+# hfms-bridge
+
+Transfer models and datasets between **Hugging Face and ModelScope** with minimal local disk usage. File contents stream directly between services; only small metadata checkpoints are saved locally.
+
+- Both directions: `hfms-bridge hf2ms` and `hfms-bridge ms2hf`.
+- Parallel file transfers with overall and per-file progress bars.
+- Automatic destination creation and incremental updates.
+- File-level resume and recovery from interrupted source downloads.
+- Python 3.9+, Linux, macOS and Windows.
+
+## Install
+
+```bash
+pip install hfms-bridge
+# Or install as an isolated command:
+uv tool install hfms-bridge
+```
+
+## Transfer
+
+From Hugging Face to ModelScope:
+
+```bash
+hfms-bridge hf2ms org/source user/destination --ms-token "$MODELSCOPE_API_TOKEN"
+```
+
+From ModelScope to Hugging Face:
+
+```bash
+hfms-bridge ms2hf org/source user/destination --hf-token "$HF_TOKEN"
+```
+
+The destination token is required explicitly and needs write/create permission. Public sources normally need no token. For private sources, pass `--hf-token` or `--ms-token`, or set the source token environment variable (`HF_TOKEN` or `MODELSCOPE_API_TOKEN`).
+
+In PowerShell, use `$env:HF_TOKEN` or `$env:MODELSCOPE_API_TOKEN` instead of the shell variables above.
+
+## Common options
+
+```bash
+# Preview without downloading payloads or modifying repositories/cache
+hfms-bridge ms2hf org/source user/destination --hf-token "$HF_TOKEN" --dry-run
+
+# Select files and concurrency
+hfms-bridge hf2ms org/source user/destination --ms-token "$MODELSCOPE_API_TOKEN" \
+  --workers 4 --include '*.safetensors' '*.json' --exclude 'original/*'
+
+hfms-bridge hf2ms --help
+hfms-bridge ms2hf --help
+```
+
+`--repo-type auto` checks the source's model and dataset namespaces and selects the unique accessible match. If both exist, choose `--repo-type model` or `--repo-type dataset` explicitly. The destination uses the same type.
+
+`--hf-revision` defaults to `main`; `--ms-revision` defaults to `master`. Source revisions can be selected independently. A missing destination is created on its default branch; other destination branches must already exist. `--visibility auto|public|private` controls new repositories; auto creates private destinations for private, gated or unknown-visibility sources.
+
+Existing identical files are skipped; different content is overwritten by default. Choose `--on-conflict error` or `--on-conflict skip` to change that behavior. Files unrelated to the transfer are not deleted.
+
+## Resume and resource usage
+
+If `HF_ENDPOINT` or `--hf-endpoint` is `https://hf-mirror.com`, `ms2hf` sends destination requests directly to official HF (`https://huggingface.co`) to avoid upload redirects. Official HF must be reachable. Existing tasks can resume using the same command and cache; `hf2ms` downloads still use the mirror.
+
+Run the same command again after an error or interruption. Completed files are compared and skipped when identical. Source file versions are pinned for unfinished tasks. Changing `--workers` or `--progress` does not discard progress. `--restart` resolves the source again; it does not force identical remote blobs to upload again.
+
+No payload is cached on disk. Metadata checkpoints default to `~/.cache/hfms-bridge`, with a `256MiB` budget; change this with `--cache-dir` and `--cache-max-size`. Old completed tasks are evicted first, and active tasks are protected. Memory scales mainly with worker count and read buffers (`--chunk-size`, default `1MiB`).
+
+Interrupted source reads can continue through HTTP Range while the process is running. MS supports this path for LFS files; ordinary MS files restart their read. HF uploads can use server-negotiated multipart uploads, with each part streamed directly. **Destination upload failures and process restarts do not support file-internal resume:** unfinished files may restart from byte zero. Complete remote blobs may still be reused.
+
+For ordinary HF destination files, checking equality may require another source read. Large ordinary files are committed using streaming Base64. MS repository metadata files have a 1MiB inline limit. Network, permission, service file-size and rate limits still apply.
+
+
+Source and developer documentation: [English](https://github.com/feixyz10/hfms-bridge/blob/main/README.md), [简体中文](https://github.com/feixyz10/hfms-bridge/blob/main/README_CN.md).

hfms_bridge-0.4.0/README.md

@@ -0,0 +1,100 @@
+# hfms-bridge
+
+[User guide](docs/README.md) | **English** | [简体中文](README_CN.md)
+
+Stream models and datasets between HuggingsFace (HF) and ModelScope (MS), without saving file payloads to disk. Python 3.9+, Linux, macOS and Windows.
+
+## Development quick start
+
+```bash
+uv sync --locked
+uv run hfms-bridge hf2ms org/source user/destination --ms-token "$MODELSCOPE_API_TOKEN"
+uv run hfms-bridge ms2hf org/source user/destination --hf-token "$HF_TOKEN"
+uv run hfms-bridge ms2hf --help
+```
+
+The destination token must be explicit. Optional source tokens default to `HF_TOKEN` or `MODELSCOPE_API_TOKEN`. Private or gated sources require authorized read access.
+
+
+## Options and behavior
+
+Both directions share these options:
+
+| Option                                 | Default / behavior                                                                                                           |
+| -------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- |
+| `SOURCE DESTINATION`                 | Required `owner/repository` IDs                                                                                            |
+| `--repo-type auto\|model\|dataset`     | Probe the source model and dataset namespaces; select the unique accessible match; require an explicit choice when ambiguous |
+| `--hf-token TOKEN`                   | Required for `ms2hf`; otherwise optional, defaults to `HF_TOKEN`                                                         |
+| `--ms-token TOKEN`                   | Required for `hf2ms`; otherwise optional, defaults to `MODELSCOPE_API_TOKEN`                                             |
+| `--hf-revision REV`                  | `main`; source revision or existing destination branch                                                                     |
+| `--ms-revision REV`                  | `master`; source revision or existing destination branch                                                                   |
+| `--visibility auto\|public\|private`   | New destination only; auto mirrors public sources, otherwise private                                                         |
+| `--include GLOB [GLOB ...]`          | Repeatable case-sensitive repository-relative globs                                                                          |
+| `--exclude GLOB [GLOB ...]`          | Repeatable, takes precedence over include                                                                                    |
+| `--on-conflict overwrite\|error\|skip` | `overwrite`; different or unverifiable destination content                                                                 |
+| `--workers N`                        | `4`; concurrent file transfers, `1` for serial                                                                           |
+| `--attempts N`                       | `3`; maximum attempts per operation, including the first                                                                   |
+| `--chunk-size SIZE`                  | `1MiB`; streaming buffer per active file                                                                                   |
+| `--timeout SECONDS`                  | `120`; socket timeout                                                                                                      |
+| `--progress auto\|on\|off`             | `auto`; enabled on terminal stderr                                                                                         |
+| `--cache-dir PATH`                   | `~/.cache/hfms-bridge`                                                                                                     |
+| `--cache-max-size SIZE`              | `256MiB`; minimum `64KiB`, oldest completed tasks evicted first                                                          |
+| `--restart`                          | Ignore local progress and resolve the source again; identical remote data can still be reused                                |
+| `--dry-run`                          | Read-only plan, no payload downloads, remote writes or checkpoint changes                                                    |
+| `--hf-endpoint URL`                  | `HF_ENDPOINT` or `https://huggingface.co`                                                                                |
+| `--ms-endpoint URL`                  | `MODELSCOPE_ENDPOINT` or `https://modelscope.cn`                                                                         |
+
+Use `hfms-bridge --version` for the version. Sizes accept bytes, KiB/MiB/GiB or KB/MB/GB. Quote globs and place positional arguments before multi-value filters. A missing/inaccessible source produces an error; auto detection cannot discover repositories hidden from the current token.
+
+Missing destinations are created automatically on their platform's default branch. Other destination branches must already exist. Existing repositories are updated through per-file commits; unrelated destination files are not deleted. Visibility is only set on creation. MS creation uses the source license when available, otherwise `Other`; license and README files are transferred normally. HF repository creation does not synthesize a license card.
+
+Identical destination content is skipped using SHA-256 or, for ordinary HF files, Git blob SHA-1. Ordinary HF comparisons may require reading the source again. Dry runs do not download data to establish missing hashes, so some equality checks remain unknown. `--on-conflict error` checks all selected paths before remote writes; it does not provide an atomic transaction against concurrent external writers.
+
+## Architecture and recovery
+
+For `ms2hf`, configuring `https://hf-mirror.com` as the HF endpoint routes destination requests directly to `https://huggingface.co`. This avoids replaying streamed commits after a mirror redirect and requires access to official HF. The configured endpoint remains part of the checkpoint identity, so existing mirror-configured tasks resume with the same command. `hf2ms` source downloads still use the configured mirror.
+
+`hubs/hf.py` and `hubs/ms.py` implement source and destination adapters. `transfer.py`, `http.py`, `progress.py`, and `state.py` share scheduling, retry, checked streaming, progress and checkpoint logic across both directions.
+
+HF sources pin a commit. MS sources store a SQLite manifest containing each file's immutable revision, size and digest; the root listing marker is checked before and after enumeration. An unfinished run reuses this manifest even when the source branch changes. This is a file-revision snapshot, not a server-side atomic repository snapshot. No file payload is stored locally. Metadata disk usage grows with the number of files; the cache budget protects active tasks from eviction and fails if they cannot fit.
+
+Large payloads pass through bounded buffers. SHA-256 is computed while reading and validated against available source metadata before completion is recorded. Missing digests may require an extra source pass. HF preupload determines ordinary/LFS storage; ordinary file commits stream Base64 NDJSON, while LFS uses basic PUT or server-negotiated multipart upload. MS uses its existing LFS PUT/commit protocol. MS repository metadata files have a 1 MiB inline limit.
+
+Source connection loss can resume inside a running upload using a strict HTTP 206 Range response. MS Range recovery is enabled only for LFS files; ordinary MS files restart their read because their Range behavior is unreliable. The extension (`.pt`, `.jsonl`, etc.) does not decide this: the server's LFS metadata does.
+
+A destination upload failure retries the file. HF multipart parts are streamed sequentially, without buffering a whole part or persisting upload sessions; failed parts are not resumed independently. **Process exit or Ctrl-C resumes at file granularity, not within an unfinished file.** Completed blobs may be reused if the destination reports them present. Run the same command without `--restart` to continue; workers and progress settings can change without changing task identity.
+
+Threads overlap network I/O with isolated HTTP sessions. Only commits and short checkpoint operations serialize. Progress has one overall file counter and a temporary row per active file. Byte counts reflect bytes consumed by the uploader, not server acknowledgements. Source Range recovery preserves progress; a whole-file retry resets it. Exhausted retries stop scheduling new files while active workers finish. Ctrl-C waits for active network calls to return or time out before releasing resources.
+
+Credentials are excluded from checkpoints and stripped from cross-origin download redirects. Metadata redirects are restricted to trusted origins; streamed uploads and commits are not blindly replayed on redirects.
+
+## Python API
+
+```python
+import os
+from hfms_bridge import Transfer, TransferOptions
+
+result = Transfer(
+    TransferOptions(
+        source="org/source",
+        destination="user/destination",
+        direction="ms2hf",  # or "hf2ms"
+        workers=4,
+    ),
+    hf_token=os.environ["HF_TOKEN"],
+    ms_token=os.getenv("MODELSCOPE_API_TOKEN"),
+).run()
+```
+
+## Validation and build
+
+```bash
+uv sync --locked
+uv run ruff check .
+uv run pytest
+uv build
+```
+
+Tests use local HTTP servers for both directions, actual streaming requests, multipart completion, Range recovery, pinned-manifest recovery, conflict handling, cache limits, cancellation and progress. CI covers Python 3.9–3.14 on Linux/macOS/Windows. Public MS model/dataset downloads and HF metadata endpoints are checked separately; local server tests do not replace authenticated end-to-end service validation.
+
+Protocol references: [HF LFS implementation](https://github.com/huggingface/huggingface_hub/blob/main/src/huggingface_hub/lfs.py), [HF commits](https://github.com/huggingface/huggingface_hub/blob/main/src/huggingface_hub/_commit_api.py), [ModelScope REST/LFS](https://github.com/modelscope/modelscope_hub/blob/main/src/modelscope_hub/_legacy_api.py).

hfms_bridge-0.4.0/README_CN.md

@@ -0,0 +1,93 @@
+# hfms-bridge
+
+[用户指南（英文）](docs/README.md) | [English](README.md) | **简体中文**
+
+在 HuggingFace（HF）与 ModelScope（MS）之间流式传输模型和数据集，不把文件内容保存到本地。支持 Python 3.9+、Linux、macOS 和 Windows。
+
+## 开发环境与使用
+
+```bash
+uv sync --locked
+uv run hfms-bridge hf2ms org/source user/destination --ms-token "$MODELSCOPE_API_TOKEN"
+uv run hfms-bridge ms2hf org/source user/destination --hf-token "$HF_TOKEN"
+uv run hfms-bridge ms2hf --help
+```
+
+目标端 token 必须显式传入。源端 token 可选，默认读取 `HF_TOKEN` 或 `MODELSCOPE_API_TOKEN`；私有或受限源仓库需要具备读取权限的 token。
+
+
+## 参数与仓库行为
+
+| 参数                                   | 默认值 / 行为                                                                           |
+| -------------------------------------- | --------------------------------------------------------------------------------------- |
+| `SOURCE DESTINATION`                 | 必填，均为 `owner/repository`                                                         |
+| `--repo-type auto\|model\|dataset`     | auto 检查源端模型和数据集命名空间；只有一个可访问结果时选用它，两个均存在则要求显式指定 |
+| `--hf-token TOKEN`                   | ms2hf 必填；hf2ms 可选，默认读取 `HF_TOKEN`                                           |
+| `--ms-token TOKEN`                   | hf2ms 必填；ms2hf 可选，默认读取 `MODELSCOPE_API_TOKEN`                               |
+| `--hf-revision REV`                  | `main`；源端版本或已存在的目标分支                                                    |
+| `--ms-revision REV`                  | `master`；源端版本或已存在的目标分支                                                  |
+| `--visibility auto\|public\|private`   | 仅新仓库；auto 对公开源创建公开目标，其余创建私有目标                                   |
+| `--include GLOB [GLOB ...]`          | 可重复，匹配仓库相对路径，区分大小写                                                    |
+| `--exclude GLOB [GLOB ...]`          | 可重复，优先于 include                                                                  |
+| `--on-conflict overwrite\|error\|skip` | `overwrite`；处理内容不同或无法确认相同的目标文件                                     |
+| `--workers N`                        | `4`；文件并行数，设为 1 即串行                                                        |
+| `--attempts N`                       | `3`；每项操作尝试次数，包含首次                                                       |
+| `--chunk-size SIZE`                  | `1MiB`；每个活动文件的读取缓冲                                                        |
+| `--timeout SECONDS`                  | `120`；网络超时                                                                       |
+| `--progress auto\|on\|off`             | `auto`；终端 stderr 显示进度                                                          |
+| `--cache-dir PATH`                   | `~/.cache/hfms-bridge`                                                                |
+| `--cache-max-size SIZE`              | `256MiB`；最小 64KiB，优先清理最久未使用的已完成任务                                  |
+| `--restart`                          | 忽略本地进度并重新解析源版本，仍可复用远端相同内容                                      |
+| `--dry-run`                          | 只读预览，不下载文件内容、不写远端、不更改缓存                                          |
+| `--hf-endpoint URL`                  | `HF_ENDPOINT` 或 `https://huggingface.co`                                           |
+| `--ms-endpoint URL`                  | `MODELSCOPE_ENDPOINT` 或 `https://modelscope.cn`                                    |
+
+版本查看使用 `hfms-bridge --version`。大小支持字节及 KiB/MiB/GiB、KB/MB/GB。请给 glob 加引号，并将源和目标位置参数放在多值筛选参数前。自动识别只针对当前 token 可见的仓库。
+
+目标仓库不存在时自动创建，使用该平台默认分支；其他目标分支必须提前创建。已有仓库逐文件提交更新，不删除目标上的其他文件。可见性只在创建时设置。创建 MS 仓库时采用源许可证，缺失时使用 `Other`；HF 不自动生成许可证卡片，许可证文件和 README 正常传输。
+
+通过 SHA-256 或 HF 普通文件的 Git blob SHA-1 比较相同内容并跳过。后者可能需要再次读取源文件。dry-run 不为比较下载内容，因此部分结果为未知。`--on-conflict error` 在远端写入前检查所选路径，但不能对其他客户端的并发修改提供事务保证。
+
+## 实现与恢复语义
+
+`ms2hf` 配置 `https://hf-mirror.com` 时，目标端请求直接使用 `https://huggingface.co`，避免镜像跳转后重放已消耗的提交流，因此仍需能访问 HF 官方站点。检查点身份保留原始 endpoint 配置，已有任务使用原命令即可恢复。`hf2ms` 源端下载仍使用配置的镜像。
+
+`hubs/hf.py`、`hubs/ms.py` 提供平台适配器，调度、重试、流校验、进度和检查点由公共引擎复用。
+
+HF 源固定到 commit；MS 源把每个文件的不可变 revision、大小和摘要写入 SQLite 清单，枚举前后检查根列表标记。未完成任务恢复时复用清单，即使源分支已变化也保持文件版本。它是文件版本快照，不是服务端原子仓库快照。本地只保存元数据，不保存文件内容；元数据空间随文件数增长。缓存预算不会淘汰活动任务，放不下时会报错。
+
+文件内容通过有限缓冲传输，并在完成前校验可用的 SHA-256。缺少摘要可能增加一次读取。HF 由 preupload 接口决定普通/LFS 存储；普通文件使用流式 Base64 NDJSON 提交，LFS 使用单次 PUT 或服务端协商的分片上传。MS 使用现有 LFS PUT/commit 协议，仓库元数据文件有 1MiB 内联限制。
+
+源端断线可在当前上传过程中通过严格校验的 HTTP 206 Range 响应续读。MS 仅对 LFS 文件启用 Range，普通文件的 Range 行为不可靠，需重新读取。是否使用 LFS 取决于服务端元数据，与 `.pt`、`.jsonl` 等扩展名无关。
+
+目标端上传失败仍按文件重试。HF 分片顺序流式发送，不缓存整片，也不持久化分片会话；不单独续传失败分片。**进程退出或 Ctrl-C 后，只保证文件级恢复，未完成文件不能从中间字节继续。** 若目标端已保存完整 blob，则可协商复用。保持原命令、不加 `--restart` 即可恢复；修改 workers 或 progress 不影响任务身份。
+
+并行采用线程和独立 HTTP 会话，仅提交和短暂的 SQLite 操作串行。总进度统计完成或跳过文件，活动文件各有临时进度行。字节进度代表上传器读取量，不是服务端确认量；源端 Range 恢复不清零，整文件重试会清零。重试耗尽后停止派发新文件，等待活动任务完成。Ctrl-C 清理需等待正在执行的网络调用返回或超时。
+
+token 不写入检查点，跨域下载跳转会移除认证信息。元数据跳转限于受信任来源，不自动重放已消耗的上传或提交流。
+
+## Python API
+
+```python
+import os
+from hfms_bridge import Transfer, TransferOptions
+
+result = Transfer(
+    TransferOptions("org/source", "user/destination", direction="ms2hf", workers=4),
+    hf_token=os.environ["HF_TOKEN"],
+    ms_token=os.getenv("MODELSCOPE_API_TOKEN"),
+).run()
+```
+
+## 验证与构建
+
+```bash
+uv sync --locked
+uv run ruff check .
+uv run pytest
+uv build
+```
+
+本地 HTTP 服务测试覆盖双向流式请求、分片完成、Range 恢复、固定清单恢复、冲突处理、缓存限制、取消和进度。CI 配置 Python 3.9–3.14 与 Linux/macOS/Windows。公开 MS 模型/数据集下载和 HF 元数据接口另行只读验证；本地测试不能替代真实鉴权上传的端到端验证。
+
+协议参考：[HF LFS](https://github.com/huggingface/huggingface_hub/blob/main/src/huggingface_hub/lfs.py)、[HF commit](https://github.com/huggingface/huggingface_hub/blob/main/src/huggingface_hub/_commit_api.py)、[MS REST/LFS](https://github.com/modelscope/modelscope_hub/blob/main/src/modelscope_hub/_legacy_api.py)。

hfms_bridge-0.4.0/docs/README.md

@@ -0,0 +1,70 @@
+# hfms-bridge
+
+Transfer models and datasets between **Hugging Face and ModelScope** with minimal local disk usage. File contents stream directly between services; only small metadata checkpoints are saved locally.
+
+- Both directions: `hfms-bridge hf2ms` and `hfms-bridge ms2hf`.
+- Parallel file transfers with overall and per-file progress bars.
+- Automatic destination creation and incremental updates.
+- File-level resume and recovery from interrupted source downloads.
+- Python 3.9+, Linux, macOS and Windows.
+
+## Install
+
+```bash
+pip install hfms-bridge
+# Or install as an isolated command:
+uv tool install hfms-bridge
+```
+
+## Transfer
+
+From Hugging Face to ModelScope:
+
+```bash
+hfms-bridge hf2ms org/source user/destination --ms-token "$MODELSCOPE_API_TOKEN"
+```
+
+From ModelScope to Hugging Face:
+
+```bash
+hfms-bridge ms2hf org/source user/destination --hf-token "$HF_TOKEN"
+```
+
+The destination token is required explicitly and needs write/create permission. Public sources normally need no token. For private sources, pass `--hf-token` or `--ms-token`, or set the source token environment variable (`HF_TOKEN` or `MODELSCOPE_API_TOKEN`).
+
+In PowerShell, use `$env:HF_TOKEN` or `$env:MODELSCOPE_API_TOKEN` instead of the shell variables above.
+
+## Common options
+
+```bash
+# Preview without downloading payloads or modifying repositories/cache
+hfms-bridge ms2hf org/source user/destination --hf-token "$HF_TOKEN" --dry-run
+
+# Select files and concurrency
+hfms-bridge hf2ms org/source user/destination --ms-token "$MODELSCOPE_API_TOKEN" \
+  --workers 4 --include '*.safetensors' '*.json' --exclude 'original/*'
+
+hfms-bridge hf2ms --help
+hfms-bridge ms2hf --help
+```
+
+`--repo-type auto` checks the source's model and dataset namespaces and selects the unique accessible match. If both exist, choose `--repo-type model` or `--repo-type dataset` explicitly. The destination uses the same type.
+
+`--hf-revision` defaults to `main`; `--ms-revision` defaults to `master`. Source revisions can be selected independently. A missing destination is created on its default branch; other destination branches must already exist. `--visibility auto|public|private` controls new repositories; auto creates private destinations for private, gated or unknown-visibility sources.
+
+Existing identical files are skipped; different content is overwritten by default. Choose `--on-conflict error` or `--on-conflict skip` to change that behavior. Files unrelated to the transfer are not deleted.
+
+## Resume and resource usage
+
+If `HF_ENDPOINT` or `--hf-endpoint` is `https://hf-mirror.com`, `ms2hf` sends destination requests directly to official HF (`https://huggingface.co`) to avoid upload redirects. Official HF must be reachable. Existing tasks can resume using the same command and cache; `hf2ms` downloads still use the mirror.
+
+Run the same command again after an error or interruption. Completed files are compared and skipped when identical. Source file versions are pinned for unfinished tasks. Changing `--workers` or `--progress` does not discard progress. `--restart` resolves the source again; it does not force identical remote blobs to upload again.
+
+No payload is cached on disk. Metadata checkpoints default to `~/.cache/hfms-bridge`, with a `256MiB` budget; change this with `--cache-dir` and `--cache-max-size`. Old completed tasks are evicted first, and active tasks are protected. Memory scales mainly with worker count and read buffers (`--chunk-size`, default `1MiB`).
+
+Interrupted source reads can continue through HTTP Range while the process is running. MS supports this path for LFS files; ordinary MS files restart their read. HF uploads can use server-negotiated multipart uploads, with each part streamed directly. **Destination upload failures and process restarts do not support file-internal resume:** unfinished files may restart from byte zero. Complete remote blobs may still be reused.
+
+For ordinary HF destination files, checking equality may require another source read. Large ordinary files are committed using streaming Base64. MS repository metadata files have a 1MiB inline limit. Network, permission, service file-size and rate limits still apply.
+
+
+Source and developer documentation: [English](https://github.com/feixyz10/hfms-bridge/blob/main/README.md), [简体中文](https://github.com/feixyz10/hfms-bridge/blob/main/README_CN.md).

hfms_bridge-0.4.0/pyproject.toml

@@ -0,0 +1,63 @@
+[build-system]
+requires = ["hatchling>=1.27,<2"]
+build-backend = "hatchling.build"
+
+[project]
+name = "hfms-bridge"
+version = "0.4.0"
+description = "Stream models and datasets between HuggingFace and ModelScope"
+readme = "docs/README.md"
+authors = [{ name = "ExcitingMe" }]
+requires-python = ">=3.9"
+license = "MIT"
+license-files = ["LICENSE"]
+keywords = ["huggingface", "modelscope", "datasets", "models", "streaming", "transfer", "cli"]
+dependencies = ["requests>=2.32.3,<3", "filelock>=3.16,<4", "tqdm>=4.66,<5"]
+classifiers = [
+  "Environment :: Console",
+  "Intended Audience :: Developers",
+  "Topic :: Utilities",
+  "Programming Language :: Python :: 3 :: Only",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Programming Language :: Python :: 3.14",
+]
+
+[project.urls]
+Homepage = "https://github.com/feixyz10/hfms-bridge/"
+Source = "https://github.com/feixyz10/hfms-bridge/"
+Issues = "https://github.com/feixyz10/hfms-bridge/issues"
+
+[project.scripts]
+hfms-bridge = "hfms_bridge.cli:main"
+
+[dependency-groups]
+dev = ["pytest>=8,<9", "ruff>=0.11,<1"]
+
+[[tool.uv.index]]
+name = "pypi"
+url = "https://pypi.org/simple"
+default = true
+
+[tool.ruff]
+target-version = "py39"
+line-length = 100
+
+[tool.ruff.lint]
+select = ["E4", "E7", "E9", "F", "B"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+# Keep release metadata compatible with Twine 6 validation.
+[tool.hatch.build.targets.wheel]
+packages = ["src/hfms_bridge"]
+core-metadata-version = "2.4"
+
+[tool.hatch.build.targets.sdist]
+core-metadata-version = "2.4"

hfms_bridge-0.4.0/src/hfms_bridge/__init__.py

@@ -0,0 +1,6 @@
+"""Streaming repository transfer without storing payloads on disk."""
+
+from .transfer import Transfer, TransferOptions
+
+__all__ = ["Transfer", "TransferOptions"]
+__version__ = "0.4.0"

hfms_bridge-0.4.0/src/hfms_bridge/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())