unmuzzle 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- unmuzzle-0.1.0/LICENSE +21 -0
- unmuzzle-0.1.0/PKG-INFO +123 -0
- unmuzzle-0.1.0/README.md +107 -0
- unmuzzle-0.1.0/pyproject.toml +24 -0
- unmuzzle-0.1.0/setup.cfg +4 -0
- unmuzzle-0.1.0/tests/test_api.py +100 -0
- unmuzzle-0.1.0/tests/test_download.py +72 -0
- unmuzzle-0.1.0/tests/test_e2e.py +79 -0
- unmuzzle-0.1.0/tests/test_hfcache.py +26 -0
- unmuzzle-0.1.0/tests/test_index.py +54 -0
- unmuzzle-0.1.0/unmuzzle/__init__.py +3 -0
- unmuzzle-0.1.0/unmuzzle/api.py +206 -0
- unmuzzle-0.1.0/unmuzzle/cli.py +190 -0
- unmuzzle-0.1.0/unmuzzle/download.py +138 -0
- unmuzzle-0.1.0/unmuzzle/hfcache.py +54 -0
- unmuzzle-0.1.0/unmuzzle/index.py +116 -0
- unmuzzle-0.1.0/unmuzzle/mcp_server.py +80 -0
- unmuzzle-0.1.0/unmuzzle/publish.py +88 -0
- unmuzzle-0.1.0/unmuzzle/sign.py +64 -0
- unmuzzle-0.1.0/unmuzzle.egg-info/PKG-INFO +123 -0
- unmuzzle-0.1.0/unmuzzle.egg-info/SOURCES.txt +23 -0
- unmuzzle-0.1.0/unmuzzle.egg-info/dependency_links.txt +1 -0
- unmuzzle-0.1.0/unmuzzle.egg-info/entry_points.txt +3 -0
- unmuzzle-0.1.0/unmuzzle.egg-info/requires.txt +6 -0
- unmuzzle-0.1.0/unmuzzle.egg-info/top_level.txt +1 -0
unmuzzle-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Jiajun Zeng
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
unmuzzle-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: unmuzzle
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Censorship-resistant distribution for open-weight models. BitTorrent + HTTP mirrors, signed checksums, Hugging Face cache compatible.
|
|
5
|
+
Author: Jiajun Zeng
|
|
6
|
+
License: MIT
|
|
7
|
+
Keywords: llm,open-weights,bittorrent,distribution,censorship-resistant
|
|
8
|
+
Requires-Python: >=3.9
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Provides-Extra: dev
|
|
12
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
13
|
+
Provides-Extra: mcp
|
|
14
|
+
Requires-Dist: mcp>=1.2; extra == "mcp"
|
|
15
|
+
Dynamic: license-file
|
|
16
|
+
|
|
17
|
+
# unmuzzle
|
|
18
|
+
|
|
19
|
+
[](https://github.com/zengjiajun0623/unmuzzle-hub/actions/workflows/e2e-verify.yml)
|
|
20
|
+
|
|
21
|
+
Agent-native distribution for open-weight models.
|
|
22
|
+
|
|
23
|
+
AI agents are the users here. `unmuzzle` is built so an agent (Claude Code,
|
|
24
|
+
Codex, Kimi Code, any MCP client) can publish and download models end to end
|
|
25
|
+
with no human in the loop: plain-JSON index, `--json` on every command, an MCP
|
|
26
|
+
server, and no gates, auth flows, or license click-throughs that only a human
|
|
27
|
+
in a browser can complete.
|
|
28
|
+
|
|
29
|
+
It is also censorship-resistant by construction:
|
|
30
|
+
|
|
31
|
+
- **The index is plain JSON in a git repo.** Anyone can mirror or fork it.
|
|
32
|
+
- **Weights move over BitTorrent (web-seeded) and HTTP mirrors.** Web seeds
|
|
33
|
+
keep every torrent alive at zero peers; no central server to block or bill.
|
|
34
|
+
- **Every file is sha256-pinned and the manifest is minisign-signed.** The
|
|
35
|
+
signature, not the host, is the trust root. Any static host can be a mirror.
|
|
36
|
+
- **Installs land in the Hugging Face cache layout**, so `transformers` loads
|
|
37
|
+
them with zero code changes.
|
|
38
|
+
|
|
39
|
+
Zero dependencies. Python 3.9+.
|
|
40
|
+
|
|
41
|
+
## Install
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
pip install git+https://github.com/zengjiajun0623/unmuzzle-hub.git
|
|
45
|
+
# optional: aria2 for torrents, minisign for signature verification
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
## For agents
|
|
49
|
+
|
|
50
|
+
Read [AGENTS.md](AGENTS.md). It is the complete publish/download protocol.
|
|
51
|
+
Or add the MCP server:
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
pip install 'unmuzzle[mcp]'
|
|
55
|
+
claude mcp add unmuzzle -- unmuzzle-mcp
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
## For humans
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
unmuzzle list
|
|
62
|
+
unmuzzle info unmuzzle/qwen2.5-14b-abliterated
|
|
63
|
+
unmuzzle get unmuzzle/qwen2.5-14b-abliterated --require-signature --dest ./models
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
Every command takes `--json`.
|
|
67
|
+
|
|
68
|
+
## First release: unmuzzle/qwen2.5-14b-abliterated
|
|
69
|
+
|
|
70
|
+
Qwen2.5-14B-Instruct with the CCP-censorship refusal direction weight-baked
|
|
71
|
+
out (GGUF Q4_K_M, ~9 GB, Apache-2.0). Answers factually on Tiananmen, Xinjiang,
|
|
72
|
+
Taiwan, and similar topics. Runs in Ollama/llama.cpp on a 16 GB machine:
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
unmuzzle get unmuzzle/qwen2.5-14b-abliterated --require-signature --dest ./models
|
|
76
|
+
ollama create unmuzzle-qwen14b -f Modelfile # Modelfile in the HF mirror repo
|
|
77
|
+
ollama run unmuzzle-qwen14b
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Mirrors: Hugging Face HTTP + web-seeded torrent. Signed with the unmuzzle
|
|
81
|
+
minisign key.
|
|
82
|
+
|
|
83
|
+
## Publish a model
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
unmuzzle keygen # one-time signing identity
|
|
87
|
+
unmuzzle publish ./mymodel \
|
|
88
|
+
--name org/mymodel \
|
|
89
|
+
--http-base https://your-mirror/mymodel \
|
|
90
|
+
--torrent-url https://your-mirror/mymodel.torrent \
|
|
91
|
+
--magnet "magnet:?xt=..." \
|
|
92
|
+
--sign-key ~/.minisign/unmuzzle.key
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Then PR your `index/models/*.json` into this repo, or host your own index and
|
|
96
|
+
point clients at it with `--index` / `$UNMUZZLE_INDEX`. The full recipe,
|
|
97
|
+
including torrent creation with web seeds, is in [AGENTS.md](AGENTS.md).
|
|
98
|
+
Format and threat model: [SPEC.md](SPEC.md).
|
|
99
|
+
|
|
100
|
+
## Why not just Hugging Face / a mirror / a torrent site?
|
|
101
|
+
|
|
102
|
+
- **HF** gates models behind human browser flows and can take them down
|
|
103
|
+
(ERNIE ViLG, GEITje). In mainland China the site itself is blocked, which is
|
|
104
|
+
why hf-mirror.com exists. Right default for ordinary models, wrong single
|
|
105
|
+
point of failure for models someone wants suppressed, and unusable for
|
|
106
|
+
agents hitting gated repos.
|
|
107
|
+
- **Mirrors** (hf-mirror, ModelScope) reintroduce a single operator who can
|
|
108
|
+
censor, and you cannot verify what they serve back to you.
|
|
109
|
+
- **A bare torrent** has no discovery, no publisher identity, and dies at zero
|
|
110
|
+
seeds. unmuzzle adds the index, per-file sha256, publisher signatures, and
|
|
111
|
+
web seeds on top.
|
|
112
|
+
|
|
113
|
+
## Roadmap
|
|
114
|
+
|
|
115
|
+
- [x] first real release (unmuzzle/qwen2.5-14b-abliterated)
|
|
116
|
+
- [x] MCP server for agent-native publish/download
|
|
117
|
+
- [ ] standalone registry site generated from the index
|
|
118
|
+
- [ ] DHT-native peer discovery, built-in torrent client (drop aria2c)
|
|
119
|
+
- [ ] index federation: subscribe to multiple publisher indexes
|
|
120
|
+
|
|
121
|
+
## License
|
|
122
|
+
|
|
123
|
+
MIT
|
unmuzzle-0.1.0/README.md
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
# unmuzzle
|
|
2
|
+
|
|
3
|
+
[](https://github.com/zengjiajun0623/unmuzzle-hub/actions/workflows/e2e-verify.yml)
|
|
4
|
+
|
|
5
|
+
Agent-native distribution for open-weight models.
|
|
6
|
+
|
|
7
|
+
AI agents are the users here. `unmuzzle` is built so an agent (Claude Code,
|
|
8
|
+
Codex, Kimi Code, any MCP client) can publish and download models end to end
|
|
9
|
+
with no human in the loop: plain-JSON index, `--json` on every command, an MCP
|
|
10
|
+
server, and no gates, auth flows, or license click-throughs that only a human
|
|
11
|
+
in a browser can complete.
|
|
12
|
+
|
|
13
|
+
It is also censorship-resistant by construction:
|
|
14
|
+
|
|
15
|
+
- **The index is plain JSON in a git repo.** Anyone can mirror or fork it.
|
|
16
|
+
- **Weights move over BitTorrent (web-seeded) and HTTP mirrors.** Web seeds
|
|
17
|
+
keep every torrent alive at zero peers; no central server to block or bill.
|
|
18
|
+
- **Every file is sha256-pinned and the manifest is minisign-signed.** The
|
|
19
|
+
signature, not the host, is the trust root. Any static host can be a mirror.
|
|
20
|
+
- **Installs land in the Hugging Face cache layout**, so `transformers` loads
|
|
21
|
+
them with zero code changes.
|
|
22
|
+
|
|
23
|
+
Zero dependencies. Python 3.9+.
|
|
24
|
+
|
|
25
|
+
## Install
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
pip install git+https://github.com/zengjiajun0623/unmuzzle-hub.git
|
|
29
|
+
# optional: aria2 for torrents, minisign for signature verification
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
## For agents
|
|
33
|
+
|
|
34
|
+
Read [AGENTS.md](AGENTS.md). It is the complete publish/download protocol.
|
|
35
|
+
Or add the MCP server:
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
pip install 'unmuzzle[mcp]'
|
|
39
|
+
claude mcp add unmuzzle -- unmuzzle-mcp
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## For humans
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
unmuzzle list
|
|
46
|
+
unmuzzle info unmuzzle/qwen2.5-14b-abliterated
|
|
47
|
+
unmuzzle get unmuzzle/qwen2.5-14b-abliterated --require-signature --dest ./models
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Every command takes `--json`.
|
|
51
|
+
|
|
52
|
+
## First release: unmuzzle/qwen2.5-14b-abliterated
|
|
53
|
+
|
|
54
|
+
Qwen2.5-14B-Instruct with the CCP-censorship refusal direction weight-baked
|
|
55
|
+
out (GGUF Q4_K_M, ~9 GB, Apache-2.0). Answers factually on Tiananmen, Xinjiang,
|
|
56
|
+
Taiwan, and similar topics. Runs in Ollama/llama.cpp on a 16 GB machine:
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
unmuzzle get unmuzzle/qwen2.5-14b-abliterated --require-signature --dest ./models
|
|
60
|
+
ollama create unmuzzle-qwen14b -f Modelfile # Modelfile in the HF mirror repo
|
|
61
|
+
ollama run unmuzzle-qwen14b
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Mirrors: Hugging Face HTTP + web-seeded torrent. Signed with the unmuzzle
|
|
65
|
+
minisign key.
|
|
66
|
+
|
|
67
|
+
## Publish a model
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
unmuzzle keygen # one-time signing identity
|
|
71
|
+
unmuzzle publish ./mymodel \
|
|
72
|
+
--name org/mymodel \
|
|
73
|
+
--http-base https://your-mirror/mymodel \
|
|
74
|
+
--torrent-url https://your-mirror/mymodel.torrent \
|
|
75
|
+
--magnet "magnet:?xt=..." \
|
|
76
|
+
--sign-key ~/.minisign/unmuzzle.key
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Then PR your `index/models/*.json` into this repo, or host your own index and
|
|
80
|
+
point clients at it with `--index` / `$UNMUZZLE_INDEX`. The full recipe,
|
|
81
|
+
including torrent creation with web seeds, is in [AGENTS.md](AGENTS.md).
|
|
82
|
+
Format and threat model: [SPEC.md](SPEC.md).
|
|
83
|
+
|
|
84
|
+
## Why not just Hugging Face / a mirror / a torrent site?
|
|
85
|
+
|
|
86
|
+
- **HF** gates models behind human browser flows and can take them down
|
|
87
|
+
(ERNIE ViLG, GEITje). In mainland China the site itself is blocked, which is
|
|
88
|
+
why hf-mirror.com exists. Right default for ordinary models, wrong single
|
|
89
|
+
point of failure for models someone wants suppressed, and unusable for
|
|
90
|
+
agents hitting gated repos.
|
|
91
|
+
- **Mirrors** (hf-mirror, ModelScope) reintroduce a single operator who can
|
|
92
|
+
censor, and you cannot verify what they serve back to you.
|
|
93
|
+
- **A bare torrent** has no discovery, no publisher identity, and dies at zero
|
|
94
|
+
seeds. unmuzzle adds the index, per-file sha256, publisher signatures, and
|
|
95
|
+
web seeds on top.
|
|
96
|
+
|
|
97
|
+
## Roadmap
|
|
98
|
+
|
|
99
|
+
- [x] first real release (unmuzzle/qwen2.5-14b-abliterated)
|
|
100
|
+
- [x] MCP server for agent-native publish/download
|
|
101
|
+
- [ ] standalone registry site generated from the index
|
|
102
|
+
- [ ] DHT-native peer discovery, built-in torrent client (drop aria2c)
|
|
103
|
+
- [ ] index federation: subscribe to multiple publisher indexes
|
|
104
|
+
|
|
105
|
+
## License
|
|
106
|
+
|
|
107
|
+
MIT
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "unmuzzle"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Censorship-resistant distribution for open-weight models. BitTorrent + HTTP mirrors, signed checksums, Hugging Face cache compatible."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [{ name = "Jiajun Zeng" }]
|
|
13
|
+
keywords = ["llm", "open-weights", "bittorrent", "distribution", "censorship-resistant"]
|
|
14
|
+
|
|
15
|
+
[project.scripts]
|
|
16
|
+
unmuzzle = "unmuzzle.cli:main"
|
|
17
|
+
unmuzzle-mcp = "unmuzzle.mcp_server:main"
|
|
18
|
+
|
|
19
|
+
[project.optional-dependencies]
|
|
20
|
+
dev = ["pytest>=7"]
|
|
21
|
+
mcp = ["mcp>=1.2"]
|
|
22
|
+
|
|
23
|
+
[tool.setuptools.packages.find]
|
|
24
|
+
include = ["unmuzzle*"]
|
unmuzzle-0.1.0/setup.cfg
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
import pytest
|
|
5
|
+
|
|
6
|
+
from unmuzzle import api
|
|
7
|
+
from unmuzzle.cli import main
|
|
8
|
+
from unmuzzle.publish import build_entry, write_entry
|
|
9
|
+
|
|
10
|
+
from helpers import serve
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@pytest.fixture
|
|
14
|
+
def published(tmp_path):
|
|
15
|
+
model_dir = tmp_path / "model"
|
|
16
|
+
model_dir.mkdir()
|
|
17
|
+
(model_dir / "config.json").write_text("{}")
|
|
18
|
+
(model_dir / "weights.bin").write_bytes(os.urandom(150_000))
|
|
19
|
+
srv, base = serve(model_dir)
|
|
20
|
+
index_dir = tmp_path / "index"
|
|
21
|
+
write_entry(index_dir, build_entry(model_dir, "unmuzzle/api-test",
|
|
22
|
+
http_bases=[base], tags=["test"]))
|
|
23
|
+
yield str(index_dir / "index.json"), model_dir
|
|
24
|
+
srv.shutdown()
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def test_list_and_info(published):
|
|
28
|
+
index, _ = published
|
|
29
|
+
models = api.list_models(index=index)
|
|
30
|
+
assert [m["name"] for m in models] == ["unmuzzle/api-test"]
|
|
31
|
+
assert models[0]["revision"]
|
|
32
|
+
info = api.model_info("unmuzzle/api-test", index=index)
|
|
33
|
+
assert info["total_size"] > 0 and not info["signed"]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def test_get_and_verify_api(published, tmp_path, monkeypatch):
|
|
37
|
+
index, model_dir = published
|
|
38
|
+
monkeypatch.setenv("HF_HOME", str(tmp_path / "hf"))
|
|
39
|
+
result = api.get("unmuzzle/api-test", index=index, method="http", jobs=2)
|
|
40
|
+
assert result["hf_cache"] and result["files"] == 2
|
|
41
|
+
assert "local_files_only=True" in result["load_with"]
|
|
42
|
+
v = api.verify("unmuzzle/api-test", index=index)
|
|
43
|
+
assert v["ok"] and v["checked"] == 2
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def test_get_missing_model_errors(published):
|
|
47
|
+
index, _ = published
|
|
48
|
+
with pytest.raises(api.UnmuzzleError, match="not found"):
|
|
49
|
+
api.get("unmuzzle/nope", index=index)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def test_require_signature_on_unsigned_entry(published):
|
|
53
|
+
index, _ = published
|
|
54
|
+
with pytest.raises(api.UnmuzzleError, match="unsigned"):
|
|
55
|
+
api.get("unmuzzle/api-test", index=index, require_signature=True)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def test_cli_json_output(published, tmp_path, monkeypatch, capsys):
|
|
59
|
+
index, _ = published
|
|
60
|
+
monkeypatch.setenv("HF_HOME", str(tmp_path / "hf"))
|
|
61
|
+
rc = main(["list", "--json", "--index", index])
|
|
62
|
+
assert rc == 0
|
|
63
|
+
parsed = json.loads(capsys.readouterr().out)
|
|
64
|
+
assert parsed[0]["name"] == "unmuzzle/api-test"
|
|
65
|
+
|
|
66
|
+
rc = main(["get", "unmuzzle/api-test", "--json", "--index", index, "--jobs", "2"])
|
|
67
|
+
assert rc == 0
|
|
68
|
+
result = json.loads(capsys.readouterr().out)
|
|
69
|
+
assert result["hf_cache"] is True
|
|
70
|
+
|
|
71
|
+
rc = main(["get", "unmuzzle/nope", "--json", "--index", index])
|
|
72
|
+
assert rc == 1
|
|
73
|
+
assert "error" in json.loads(capsys.readouterr().err)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def test_keygen(tmp_path, monkeypatch):
|
|
77
|
+
import shutil
|
|
78
|
+
if not shutil.which("minisign"):
|
|
79
|
+
pytest.skip("minisign not installed")
|
|
80
|
+
key = tmp_path / "k.key"
|
|
81
|
+
result = api.keygen(str(key))
|
|
82
|
+
assert result["pubkey"] and key.exists()
|
|
83
|
+
with pytest.raises(api.UnmuzzleError, match="already exists"):
|
|
84
|
+
api.keygen(str(key))
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def test_sign_and_verify_roundtrip(published, tmp_path):
|
|
88
|
+
import shutil
|
|
89
|
+
if not shutil.which("minisign"):
|
|
90
|
+
pytest.skip("minisign not installed")
|
|
91
|
+
index, model_dir = published
|
|
92
|
+
key = tmp_path / "k.key"
|
|
93
|
+
api.keygen(str(key))
|
|
94
|
+
result = api.publish(str(model_dir), "unmuzzle/signed-test",
|
|
95
|
+
http_bases=["https://example.com/x"],
|
|
96
|
+
sign_key=str(key), index_dir=str(tmp_path / "idx2"))
|
|
97
|
+
assert result["signed"] and result["entry"]["publisher_pubkey"]
|
|
98
|
+
from unmuzzle.index import validate_entry
|
|
99
|
+
status = api.check_signature(validate_entry(result["entry"]), require=True)
|
|
100
|
+
assert status == {"signed": True, "verified": True}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import hashlib
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
import pytest
|
|
5
|
+
|
|
6
|
+
from unmuzzle.download import DownloadError, download_file, sha256_file
|
|
7
|
+
|
|
8
|
+
from helpers import serve
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@pytest.fixture
|
|
12
|
+
def server(tmp_path):
|
|
13
|
+
src = tmp_path / "src"
|
|
14
|
+
src.mkdir()
|
|
15
|
+
payload = os.urandom(300_000) # spans multiple chunks at test chunk size
|
|
16
|
+
(src / "weights.bin").write_bytes(payload)
|
|
17
|
+
srv, base = serve(src)
|
|
18
|
+
yield base, payload
|
|
19
|
+
srv.shutdown()
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def test_download_multichunk(server, tmp_path):
|
|
23
|
+
base, payload = server
|
|
24
|
+
dest = tmp_path / "out" / "weights.bin"
|
|
25
|
+
sha = hashlib.sha256(payload).hexdigest()
|
|
26
|
+
download_file([base], "weights.bin", dest, len(payload), sha,
|
|
27
|
+
jobs=4, chunk_size=64 * 1024)
|
|
28
|
+
assert dest.read_bytes() == payload
|
|
29
|
+
assert not (dest.parent / "weights.bin.parts").exists()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def test_download_resumes_from_parts(server, tmp_path):
|
|
33
|
+
base, payload = server
|
|
34
|
+
dest = tmp_path / "weights.bin"
|
|
35
|
+
sha = hashlib.sha256(payload).hexdigest()
|
|
36
|
+
chunk = 64 * 1024
|
|
37
|
+
# pre-seed the first chunk as if a previous run was interrupted
|
|
38
|
+
parts = tmp_path / "weights.bin.parts"
|
|
39
|
+
parts.mkdir()
|
|
40
|
+
(parts / "000000").write_bytes(payload[:chunk])
|
|
41
|
+
download_file([base], "weights.bin", dest, len(payload), sha,
|
|
42
|
+
jobs=2, chunk_size=chunk)
|
|
43
|
+
assert dest.read_bytes() == payload
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def test_checksum_mismatch_raises(server, tmp_path):
|
|
47
|
+
base, payload = server
|
|
48
|
+
dest = tmp_path / "weights.bin"
|
|
49
|
+
with pytest.raises(DownloadError, match="sha256 mismatch"):
|
|
50
|
+
download_file([base], "weights.bin", dest, len(payload), "0" * 64,
|
|
51
|
+
jobs=2, chunk_size=64 * 1024)
|
|
52
|
+
assert not dest.exists()
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def test_mirror_failover(server, tmp_path):
|
|
56
|
+
base, payload = server
|
|
57
|
+
dest = tmp_path / "weights.bin"
|
|
58
|
+
sha = hashlib.sha256(payload).hexdigest()
|
|
59
|
+
download_file(["http://127.0.0.1:1/dead", base], "weights.bin",
|
|
60
|
+
dest, len(payload), sha, jobs=2, chunk_size=128 * 1024)
|
|
61
|
+
assert dest.read_bytes() == payload
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def test_existing_verified_file_skips(server, tmp_path):
|
|
65
|
+
base, payload = server
|
|
66
|
+
dest = tmp_path / "weights.bin"
|
|
67
|
+
dest.write_bytes(payload)
|
|
68
|
+
sha = hashlib.sha256(payload).hexdigest()
|
|
69
|
+
assert sha256_file(dest) == sha
|
|
70
|
+
download_file(["http://127.0.0.1:1/dead"], "weights.bin", dest,
|
|
71
|
+
len(payload), sha) # would fail if it tried the network
|
|
72
|
+
assert dest.read_bytes() == payload
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
from unmuzzle import hfcache
|
|
5
|
+
from unmuzzle.cli import main
|
|
6
|
+
from unmuzzle.publish import build_entry, write_entry
|
|
7
|
+
|
|
8
|
+
from helpers import serve
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def make_model(tmp_path):
|
|
12
|
+
model_dir = tmp_path / "model"
|
|
13
|
+
model_dir.mkdir()
|
|
14
|
+
(model_dir / "config.json").write_text(json.dumps({"architectures": ["TestForCausalLM"]}))
|
|
15
|
+
(model_dir / "weights.bin").write_bytes(os.urandom(200_000))
|
|
16
|
+
(model_dir / ".DS_Store").write_bytes(b"junk") # must be skipped
|
|
17
|
+
return model_dir
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def test_publish_get_verify_e2e(tmp_path, monkeypatch, capsys):
|
|
21
|
+
model_dir = make_model(tmp_path)
|
|
22
|
+
|
|
23
|
+
# publisher: serve the model dir over http, build + sign-free index entry
|
|
24
|
+
srv, base = serve(model_dir)
|
|
25
|
+
index_dir = tmp_path / "index"
|
|
26
|
+
entry = build_entry(model_dir, "unmuzzle/e2e-model", http_bases=[base],
|
|
27
|
+
description="e2e", tags=["test"])
|
|
28
|
+
write_entry(index_dir, entry)
|
|
29
|
+
index_json = index_dir / "index.json"
|
|
30
|
+
assert index_json.exists()
|
|
31
|
+
|
|
32
|
+
# client: install into an isolated HF cache
|
|
33
|
+
hf_home = tmp_path / "hf"
|
|
34
|
+
monkeypatch.setenv("HF_HOME", str(hf_home))
|
|
35
|
+
rc = main(["get", "unmuzzle/e2e-model", "--index", str(index_json), "--jobs", "4"])
|
|
36
|
+
assert rc == 0
|
|
37
|
+
srv.shutdown()
|
|
38
|
+
|
|
39
|
+
out = capsys.readouterr().out
|
|
40
|
+
assert "installed into HF cache" in out
|
|
41
|
+
|
|
42
|
+
from unmuzzle.index import validate_entry
|
|
43
|
+
rev = validate_entry(entry).revision()
|
|
44
|
+
snapshot = hfcache.model_cache_dir("unmuzzle/e2e-model") / "snapshots" / rev
|
|
45
|
+
assert (snapshot / "config.json").read_text() == (model_dir / "config.json").read_text()
|
|
46
|
+
assert (snapshot / "weights.bin").read_bytes() == (model_dir / "weights.bin").read_bytes()
|
|
47
|
+
assert not (snapshot / ".DS_Store").exists()
|
|
48
|
+
|
|
49
|
+
# verify command passes
|
|
50
|
+
rc = main(["verify", "unmuzzle/e2e-model", "--index", str(index_json)])
|
|
51
|
+
assert rc == 0
|
|
52
|
+
|
|
53
|
+
# refs/main points at the revision, so transformers can resolve it offline
|
|
54
|
+
refs = hfcache.model_cache_dir("unmuzzle/e2e-model") / "refs" / "main"
|
|
55
|
+
assert refs.read_text() == rev
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def test_get_to_plain_dest(tmp_path, capsys):
|
|
59
|
+
model_dir = make_model(tmp_path)
|
|
60
|
+
srv, base = serve(model_dir)
|
|
61
|
+
index_dir = tmp_path / "index"
|
|
62
|
+
write_entry(index_dir, build_entry(model_dir, "unmuzzle/plain", http_bases=[base]))
|
|
63
|
+
dest = tmp_path / "dest"
|
|
64
|
+
rc = main(["get", "unmuzzle/plain", "--dest", str(dest),
|
|
65
|
+
"--index", str(index_dir / "index.json")])
|
|
66
|
+
srv.shutdown()
|
|
67
|
+
assert rc == 0
|
|
68
|
+
assert (dest / "weights.bin").read_bytes() == (model_dir / "weights.bin").read_bytes()
|
|
69
|
+
assert "done. Files in" in capsys.readouterr().out
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def test_publish_rejects_empty_dir(tmp_path):
|
|
73
|
+
empty = tmp_path / "empty"
|
|
74
|
+
empty.mkdir()
|
|
75
|
+
try:
|
|
76
|
+
build_entry(empty, "unmuzzle/x", http_bases=["https://x"])
|
|
77
|
+
assert False, "should have raised"
|
|
78
|
+
except ValueError:
|
|
79
|
+
pass
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
3
|
+
from unmuzzle.hfcache import install, model_cache_dir
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def test_install_layout(tmp_path, monkeypatch):
|
|
7
|
+
monkeypatch.setenv("HF_HOME", str(tmp_path / "hf"))
|
|
8
|
+
staging = tmp_path / "staging"
|
|
9
|
+
staging.mkdir()
|
|
10
|
+
(staging / "config.json").write_text("{}")
|
|
11
|
+
sub = staging / "subdir"
|
|
12
|
+
sub.mkdir()
|
|
13
|
+
(sub / "weights.bin").write_bytes(os.urandom(1000))
|
|
14
|
+
|
|
15
|
+
files = [
|
|
16
|
+
{"path": "config.json", "size": 2, "sha256": "a" * 64},
|
|
17
|
+
{"path": "subdir/weights.bin", "size": 1000, "sha256": "b" * 64},
|
|
18
|
+
]
|
|
19
|
+
snapshot = install("unmuzzle/test", "rev123", staging, files)
|
|
20
|
+
|
|
21
|
+
cache = model_cache_dir("unmuzzle/test")
|
|
22
|
+
assert (cache / "blobs" / ("a" * 64)).exists()
|
|
23
|
+
assert (cache / "blobs" / ("b" * 64)).exists()
|
|
24
|
+
link = snapshot / "subdir" / "weights.bin"
|
|
25
|
+
assert link.read_bytes() == (sub / "weights.bin").read_bytes()
|
|
26
|
+
assert (cache / "refs" / "main").read_text() == "rev123"
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import json
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from unmuzzle.index import IndexError, find_model, load_index, validate_entry
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def entry(**kw):
|
|
9
|
+
base = {
|
|
10
|
+
"name": "unmuzzle/test-model",
|
|
11
|
+
"files": [
|
|
12
|
+
{"path": "config.json", "size": 10, "sha256": "a" * 64},
|
|
13
|
+
{"path": "model.safetensors", "size": 100, "sha256": "b" * 64},
|
|
14
|
+
],
|
|
15
|
+
"mirrors": {"http": ["https://example.com/m"], "magnet": None},
|
|
16
|
+
}
|
|
17
|
+
base.update(kw)
|
|
18
|
+
return base
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def test_validate_ok():
|
|
22
|
+
e = validate_entry(entry())
|
|
23
|
+
assert e.name == "unmuzzle/test-model"
|
|
24
|
+
assert e.total_size == 110
|
|
25
|
+
assert e.revision() and len(e.revision()) == 16
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def test_manifest_is_deterministic_and_sorted():
|
|
29
|
+
e = validate_entry(entry())
|
|
30
|
+
assert e.canonical_manifest() == f"{'a' * 64} config.json\n{'b' * 64} model.safetensors\n"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def test_rejects_traversal():
|
|
34
|
+
with pytest.raises(IndexError, match="unsafe path"):
|
|
35
|
+
validate_entry(entry(files=[{"path": "../x", "size": 1, "sha256": "a" * 64}]))
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def test_rejects_missing_mirrors():
|
|
39
|
+
with pytest.raises(IndexError, match="no mirrors"):
|
|
40
|
+
validate_entry(entry(mirrors={"http": [], "magnet": None}))
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def test_rejects_bad_name():
|
|
44
|
+
with pytest.raises(IndexError):
|
|
45
|
+
validate_entry(entry(name="no-org"))
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def test_load_and_find(tmp_path):
|
|
49
|
+
idx = tmp_path / "index.json"
|
|
50
|
+
idx.write_text(json.dumps({"version": 1, "models": [entry()]}))
|
|
51
|
+
entries = load_index(str(idx))
|
|
52
|
+
assert find_model(entries, "unmuzzle/test-model").name == "unmuzzle/test-model"
|
|
53
|
+
assert find_model(entries, "test-model").name == "unmuzzle/test-model" # shorthand
|
|
54
|
+
assert find_model(entries, "nope") is None
|