pirateface 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pirateface/__init__.py +1 -0
- pirateface/_boot.py +47 -0
- pirateface/cli.py +16 -0
- pirateface/client/__init__.py +0 -0
- pirateface/client/api.py +63 -0
- pirateface/client/torrent.py +206 -0
- pirateface/config.py +12 -0
- pirateface/huggingface.py +111 -0
- pirateface/ui.py +112 -0
- pirateface-0.1.0.dist-info/METADATA +89 -0
- pirateface-0.1.0.dist-info/RECORD +15 -0
- pirateface-0.1.0.dist-info/WHEEL +4 -0
- pirateface-0.1.0.dist-info/entry_points.txt +2 -0
- pirateface-0.1.0.dist-info/licenses/LICENSE +21 -0
- pirateface.pth +1 -0
pirateface/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
pirateface/_boot.py
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""Runs at every Python startup via pirateface.pth, so it must be cheap and never raise.
|
|
2
|
+
Patches huggingface_hub lazily, when something imports it."""
|
|
3
|
+
|
|
4
|
+
import sys
|
|
5
|
+
|
|
6
|
+
from pirateface.config import flag
|
|
7
|
+
|
|
8
|
+
TARGETS = {"huggingface_hub.file_download"}
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _install(name, module):
|
|
12
|
+
try:
|
|
13
|
+
from pirateface import huggingface
|
|
14
|
+
|
|
15
|
+
huggingface.install(module)
|
|
16
|
+
except Exception as e:
|
|
17
|
+
print(f"[pirateface] hook disabled: {e}", file=sys.stderr)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class _PatchOnImport:
|
|
21
|
+
def find_spec(self, name, path=None, target=None):
|
|
22
|
+
if name not in TARGETS:
|
|
23
|
+
return None
|
|
24
|
+
sys.meta_path.remove(self)
|
|
25
|
+
try:
|
|
26
|
+
import importlib.util
|
|
27
|
+
|
|
28
|
+
spec = importlib.util.find_spec(name)
|
|
29
|
+
finally:
|
|
30
|
+
sys.meta_path.insert(0, self)
|
|
31
|
+
if spec is None or spec.loader is None:
|
|
32
|
+
return None
|
|
33
|
+
exec_module = spec.loader.exec_module
|
|
34
|
+
|
|
35
|
+
def exec_and_patch(module):
|
|
36
|
+
exec_module(module)
|
|
37
|
+
_install(name, module)
|
|
38
|
+
|
|
39
|
+
spec.loader.exec_module = exec_and_patch
|
|
40
|
+
return spec
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
if not flag("PF_DISABLE"):
|
|
44
|
+
sys.meta_path.insert(0, _PatchOnImport())
|
|
45
|
+
for name in TARGETS:
|
|
46
|
+
if name in sys.modules:
|
|
47
|
+
_install(name, sys.modules[name])
|
pirateface/cli.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
|
|
3
|
+
import pirateface._boot # noqa: F401 # uv tools skip .pth files
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def main():
|
|
7
|
+
p = argparse.ArgumentParser(prog="pf")
|
|
8
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
9
|
+
d = sub.add_parser("download", help="download a model into the Hugging Face cache")
|
|
10
|
+
d.add_argument("repo_id")
|
|
11
|
+
d.add_argument("--revision")
|
|
12
|
+
args = p.parse_args()
|
|
13
|
+
|
|
14
|
+
from huggingface_hub import snapshot_download
|
|
15
|
+
|
|
16
|
+
print(snapshot_download(args.repo_id, revision=args.revision, max_workers=1))
|
|
File without changes
|
pirateface/client/api.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Everything that talks to pirateface.co."""
|
|
2
|
+
|
|
3
|
+
import atexit
|
|
4
|
+
import contextlib
|
|
5
|
+
import functools
|
|
6
|
+
import json
|
|
7
|
+
import threading
|
|
8
|
+
import urllib.error
|
|
9
|
+
import urllib.parse
|
|
10
|
+
import urllib.request
|
|
11
|
+
|
|
12
|
+
from pirateface.config import API, flag
|
|
13
|
+
|
|
14
|
+
NOT_LISTED = object()
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@functools.cache
|
|
18
|
+
def _lookup(repo_id):
|
|
19
|
+
"""Pirate Face's entry for the repo, NOT_LISTED, or None if unknown (network error)."""
|
|
20
|
+
url = f"{API}/api/torrents?" + urllib.parse.urlencode({"repo": repo_id})
|
|
21
|
+
try:
|
|
22
|
+
with urllib.request.urlopen(url, timeout=10) as r:
|
|
23
|
+
return json.load(r)
|
|
24
|
+
except urllib.error.HTTPError as e:
|
|
25
|
+
return NOT_LISTED if e.code == 404 else None
|
|
26
|
+
except Exception:
|
|
27
|
+
return None
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def lookup_magnet(repo_id):
|
|
31
|
+
"""The magnet, NOT_LISTED, or None if unknown (network error, or license terms to accept first)."""
|
|
32
|
+
entry = _lookup(repo_id)
|
|
33
|
+
return entry.get("magnet") if isinstance(entry, dict) else entry
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
_rescues = []
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _send_rescue(repo_id):
|
|
40
|
+
req = urllib.request.Request(
|
|
41
|
+
f"{API}/api/rescue",
|
|
42
|
+
data=json.dumps({"repoId": repo_id}).encode(),
|
|
43
|
+
headers={"Content-Type": "application/json"},
|
|
44
|
+
method="POST",
|
|
45
|
+
)
|
|
46
|
+
with contextlib.suppress(Exception):
|
|
47
|
+
urllib.request.urlopen(req, timeout=10).close()
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def request_rescue(repo_id):
|
|
51
|
+
"""Ask Pirate Face to add a model it doesn't have yet, in the background."""
|
|
52
|
+
if flag("PF_NO_RESCUE"):
|
|
53
|
+
return False
|
|
54
|
+
t = threading.Thread(target=_send_rescue, args=(repo_id,), daemon=True)
|
|
55
|
+
t.start()
|
|
56
|
+
_rescues.append(t)
|
|
57
|
+
return True
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@atexit.register
|
|
61
|
+
def _wait_for_rescues():
|
|
62
|
+
for t in _rescues:
|
|
63
|
+
t.join(timeout=2)
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
import atexit
|
|
2
|
+
import os
|
|
3
|
+
import shutil
|
|
4
|
+
import sys
|
|
5
|
+
import threading
|
|
6
|
+
import time
|
|
7
|
+
from importlib.metadata import version
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
import libtorrent as lt
|
|
11
|
+
|
|
12
|
+
from pirateface import config, ui
|
|
13
|
+
from pirateface.client import api
|
|
14
|
+
|
|
15
|
+
CACHE_DIR = Path(config.CACHE_DIR)
|
|
16
|
+
_lock = threading.Lock()
|
|
17
|
+
_session = None
|
|
18
|
+
_handles = {} # repo_id -> handle, or None if not on Pirate Face
|
|
19
|
+
_wanted = {}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class Unavailable(Exception):
|
|
23
|
+
pass
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _get_session():
|
|
27
|
+
global _session
|
|
28
|
+
if _session is None:
|
|
29
|
+
debug = config.flag("PF_DEBUG")
|
|
30
|
+
_session = lt.session(
|
|
31
|
+
{
|
|
32
|
+
"listen_interfaces": config.LISTEN,
|
|
33
|
+
"user_agent": f"pirateface/{version('pirateface')}",
|
|
34
|
+
"alert_mask": lt.alert.category_t.all_categories if debug else lt.alert.category_t.storage_notification,
|
|
35
|
+
# The torrent goes "finished" between files and drops the web seed; reconnect in 2s, not 30s.
|
|
36
|
+
# Keep close_redundant_connections on: idle web seeds otherwise never request the next file.
|
|
37
|
+
"urlseed_wait_retry": 2,
|
|
38
|
+
}
|
|
39
|
+
)
|
|
40
|
+
threading.Thread(target=_pump_alerts, args=(_session, debug), daemon=True).start()
|
|
41
|
+
return _session
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
# libtorrent reports pieces done before they're written, so files can read as zeros until flushed.
|
|
45
|
+
# cache_flushed_alert doesn't say which flush it answers, so only one runs at a time.
|
|
46
|
+
_flush = threading.Condition()
|
|
47
|
+
_flush_one_at_a_time = threading.Lock()
|
|
48
|
+
_flush_done = {}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _pump_alerts(session, debug):
|
|
52
|
+
"""The only reader of libtorrent's alerts."""
|
|
53
|
+
noisy = ("PIECE", "HAVE", "REQUEST", "picker", "ALLOWED_FAST", "dht", "portmap", "stats", "block")
|
|
54
|
+
while True:
|
|
55
|
+
session.wait_for_alert(1000)
|
|
56
|
+
for a in session.pop_alerts():
|
|
57
|
+
if isinstance(a, lt.cache_flushed_alert):
|
|
58
|
+
with _flush:
|
|
59
|
+
key = str(a.handle.info_hashes().v1)
|
|
60
|
+
_flush_done[key] = _flush_done.get(key, 0) + 1
|
|
61
|
+
_flush.notify_all()
|
|
62
|
+
if debug:
|
|
63
|
+
msg = a.message()
|
|
64
|
+
if not any(k in msg or k in type(a).__name__ for k in noisy):
|
|
65
|
+
print(f"[lt] {msg[:220]}", file=sys.stderr, flush=True)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@atexit.register
|
|
69
|
+
def _cleanup():
|
|
70
|
+
for h in _handles.values():
|
|
71
|
+
if h is None:
|
|
72
|
+
continue
|
|
73
|
+
path = Path(h.status().save_path) / h.torrent_file().name()
|
|
74
|
+
_session.remove_torrent(h)
|
|
75
|
+
if path.is_dir():
|
|
76
|
+
shutil.rmtree(path, ignore_errors=True)
|
|
77
|
+
else:
|
|
78
|
+
path.unlink(missing_ok=True)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _flush_to_disk(h, timeout=30):
|
|
82
|
+
key = str(h.info_hashes().v1)
|
|
83
|
+
with _flush_one_at_a_time, _flush:
|
|
84
|
+
before = _flush_done.get(key, 0)
|
|
85
|
+
h.flush_cache()
|
|
86
|
+
if not _flush.wait_for(lambda: _flush_done.get(key, 0) > before, timeout):
|
|
87
|
+
raise TimeoutError("libtorrent didn't confirm the file was written to disk")
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _handle(repo_id):
|
|
91
|
+
with _lock:
|
|
92
|
+
if repo_id in _handles:
|
|
93
|
+
return _handles[repo_id]
|
|
94
|
+
magnet = api.lookup_magnet(repo_id)
|
|
95
|
+
if magnet is api.NOT_LISTED:
|
|
96
|
+
ui.not_on_pirateface(repo_id, api.request_rescue(repo_id))
|
|
97
|
+
_handles[repo_id] = None
|
|
98
|
+
return None
|
|
99
|
+
if not magnet:
|
|
100
|
+
ui.not_on_pirateface(repo_id)
|
|
101
|
+
_handles[repo_id] = None
|
|
102
|
+
return None
|
|
103
|
+
atp = lt.parse_magnet_uri(magnet)
|
|
104
|
+
atp.save_path = str(CACHE_DIR)
|
|
105
|
+
atp.flags |= lt.torrent_flags.default_dont_download
|
|
106
|
+
# Cache the file list: a magnet alone needs a live peer to get it.
|
|
107
|
+
saved = CACHE_DIR / "torrents" / f"{atp.info_hashes.v1}.torrent"
|
|
108
|
+
if saved.exists():
|
|
109
|
+
atp.ti = lt.torrent_info(str(saved))
|
|
110
|
+
h = _get_session().add_torrent(atp)
|
|
111
|
+
deadline = time.monotonic() + config.METADATA_TIMEOUT
|
|
112
|
+
while not h.status().has_metadata:
|
|
113
|
+
if time.monotonic() > deadline:
|
|
114
|
+
_get_session().remove_torrent(h)
|
|
115
|
+
_handles[repo_id] = None
|
|
116
|
+
raise TimeoutError("no peer sent the file list in time")
|
|
117
|
+
time.sleep(0.2)
|
|
118
|
+
if not saved.exists():
|
|
119
|
+
saved.parent.mkdir(parents=True, exist_ok=True)
|
|
120
|
+
saved.write_bytes(lt.bencode({"info": lt.bdecode(h.torrent_file().info_section())}))
|
|
121
|
+
_handles[repo_id] = h
|
|
122
|
+
threading.Thread(target=_show_status, args=(repo_id, h), daemon=True).start()
|
|
123
|
+
fs = h.torrent_file().files()
|
|
124
|
+
real = [i for i in range(fs.num_files()) if not fs.file_flags(i) & fs.flag_pad_file]
|
|
125
|
+
ui.banner(repo_id, len(real), sum(fs.file_size(i) for i in real))
|
|
126
|
+
return h
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _show_status(repo_id, h, every=0.5, recent=3.0):
|
|
130
|
+
"""Sources = who sent bytes recently, not who is connected: the web seed reconnects per file and flickers."""
|
|
131
|
+
fs = h.torrent_file().files()
|
|
132
|
+
sent = {}
|
|
133
|
+
last_seen = {}
|
|
134
|
+
bar = None
|
|
135
|
+
while True:
|
|
136
|
+
time.sleep(every)
|
|
137
|
+
with _lock:
|
|
138
|
+
wanted = list(_wanted.get(repo_id, ()))
|
|
139
|
+
progress = h.file_progress(lt.torrent_handle.piece_granularity)
|
|
140
|
+
total = sum(fs.file_size(i) for i in wanted)
|
|
141
|
+
done = sum(min(progress[i], fs.file_size(i)) for i in wanted)
|
|
142
|
+
files_done = sum(progress[i] >= fs.file_size(i) for i in wanted)
|
|
143
|
+
if not wanted or files_done == len(wanted):
|
|
144
|
+
if bar is not None:
|
|
145
|
+
bar.close()
|
|
146
|
+
bar = None
|
|
147
|
+
continue
|
|
148
|
+
now = time.monotonic()
|
|
149
|
+
for p in h.get_peer_info():
|
|
150
|
+
conn = tuple(p.ip)
|
|
151
|
+
if p.total_download > sent.get(conn, 0):
|
|
152
|
+
last_seen[(conn[0], p.connection_type != lt.peer_info.standard_bittorrent)] = now
|
|
153
|
+
sent[conn] = p.total_download
|
|
154
|
+
active = [web for (_, web), t in last_seen.items() if now - t < recent]
|
|
155
|
+
label = ui.sources_label(any(active), active.count(False))
|
|
156
|
+
if bar is None:
|
|
157
|
+
bar = ui.repo_progress(repo_id)
|
|
158
|
+
bar.total, bar.n = total, done
|
|
159
|
+
ui.set_status(
|
|
160
|
+
bar,
|
|
161
|
+
f"{files_done}/{len(wanted)} files · {done / 1e6:.1f}/{total / 1e6:.1f} MB · {label} · {h.status().download_rate / 1e6:.1f} MB/s",
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _file_index(ti, filename):
|
|
166
|
+
fs = ti.files()
|
|
167
|
+
for i in range(fs.num_files()):
|
|
168
|
+
path = fs.file_path(i).replace(os.sep, "/")
|
|
169
|
+
rel = path.split("/", 1)[1] if fs.num_files() > 1 and "/" in path else path
|
|
170
|
+
if rel == filename:
|
|
171
|
+
return i
|
|
172
|
+
return None
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def fetch_file(repo_id, filename):
|
|
176
|
+
h = _handle(repo_id)
|
|
177
|
+
if h is None:
|
|
178
|
+
raise Unavailable()
|
|
179
|
+
ti = h.torrent_file()
|
|
180
|
+
i = _file_index(ti, filename)
|
|
181
|
+
if i is None:
|
|
182
|
+
raise Unavailable("not in the torrent")
|
|
183
|
+
size = ti.files().file_size(i)
|
|
184
|
+
path = Path(h.status().save_path) / ti.files().file_path(i)
|
|
185
|
+
# Set all priorities in one call (all-skipped for a moment drops the web seed), from our own set:
|
|
186
|
+
# libtorrent applies them lazily, so reading them back across threads loses updates.
|
|
187
|
+
with _lock:
|
|
188
|
+
wanted = _wanted.setdefault(repo_id, set())
|
|
189
|
+
wanted.add(i)
|
|
190
|
+
h.prioritize_files([4 if f in wanted else 0 for f in range(ti.num_files())])
|
|
191
|
+
|
|
192
|
+
last_done, last_change = -1, time.monotonic()
|
|
193
|
+
while True:
|
|
194
|
+
done = h.file_progress(lt.torrent_handle.piece_granularity)[i]
|
|
195
|
+
if done >= size:
|
|
196
|
+
break
|
|
197
|
+
st = h.status()
|
|
198
|
+
if st.errc.value():
|
|
199
|
+
raise RuntimeError(f"torrent error: {st.errc.message()}")
|
|
200
|
+
if done != last_done:
|
|
201
|
+
last_done, last_change = done, time.monotonic()
|
|
202
|
+
elif time.monotonic() - last_change > config.STALL_TIMEOUT:
|
|
203
|
+
raise TimeoutError(f"stalled for {config.STALL_TIMEOUT:.0f}s")
|
|
204
|
+
time.sleep(0.25)
|
|
205
|
+
_flush_to_disk(h)
|
|
206
|
+
return path
|
pirateface/config.py
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def flag(name):
|
|
5
|
+
return os.environ.get(name, "") not in ("", "0")
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
API = os.environ.get("PF_API", "https://pirateface.co")
|
|
9
|
+
CACHE_DIR = os.environ.get("PF_CACHE") or os.path.expanduser("~/.cache/pirateface") # noqa: PTH111 # pathlib slows every Python startup
|
|
10
|
+
LISTEN = os.environ.get("PF_LISTEN", "0.0.0.0:0,[::]:0")
|
|
11
|
+
METADATA_TIMEOUT = float(os.environ.get("PF_METADATA_TIMEOUT", "30"))
|
|
12
|
+
STALL_TIMEOUT = float(os.environ.get("PF_STALL_TIMEOUT", "60"))
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
"""Swap huggingface_hub's file download for a torrent download, checked against Hugging Face's fingerprint."""
|
|
2
|
+
|
|
3
|
+
import contextvars
|
|
4
|
+
import functools
|
|
5
|
+
import hashlib
|
|
6
|
+
import inspect
|
|
7
|
+
import shutil
|
|
8
|
+
import sys
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from pirateface import ui
|
|
12
|
+
from pirateface.client import torrent
|
|
13
|
+
|
|
14
|
+
FUNC = "_download_to_tmp_and_move"
|
|
15
|
+
# FUNC isn't passed the repo id, so the outer functions record it here.
|
|
16
|
+
OUTER = ("_hf_hub_download_to_cache_dir", "_hf_hub_download_to_local_dir")
|
|
17
|
+
_repo = contextvars.ContextVar("pirateface_repo", default=None)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def log(msg):
|
|
21
|
+
print(f"[pirateface] {msg}", file=sys.stderr, flush=True)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def install(file_download):
|
|
25
|
+
original = getattr(file_download, FUNC, None)
|
|
26
|
+
if original is None:
|
|
27
|
+
log(f"huggingface_hub has no {FUNC}; torrent downloads disabled")
|
|
28
|
+
return
|
|
29
|
+
|
|
30
|
+
signature = inspect.signature(original)
|
|
31
|
+
|
|
32
|
+
@functools.wraps(original)
|
|
33
|
+
def patched(*args, **kwargs):
|
|
34
|
+
bound = signature.bind(*args, **kwargs).arguments
|
|
35
|
+
dest = Path(bound["destination_path"])
|
|
36
|
+
if dest.exists() and not bound.get("force_download"):
|
|
37
|
+
return original(*args, **kwargs)
|
|
38
|
+
name = bound["filename"]
|
|
39
|
+
try:
|
|
40
|
+
if _from_torrent(name, bound["etag"], dest):
|
|
41
|
+
return None
|
|
42
|
+
except torrent.Unavailable as e:
|
|
43
|
+
ui.from_hf(name, str(e) or None)
|
|
44
|
+
except Exception as e:
|
|
45
|
+
ui.from_hf(name, str(e) or type(e).__name__)
|
|
46
|
+
return original(*args, **kwargs)
|
|
47
|
+
|
|
48
|
+
setattr(file_download, FUNC, patched)
|
|
49
|
+
_hide_token_nag()
|
|
50
|
+
for name in OUTER:
|
|
51
|
+
setattr(file_download, name, _remember_repo(getattr(file_download, name)))
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _hide_token_nag():
|
|
55
|
+
"""Silence the "set a HF_TOKEN" server warning only; other server warnings still show."""
|
|
56
|
+
try:
|
|
57
|
+
from huggingface_hub.utils import _http
|
|
58
|
+
|
|
59
|
+
_http._WARNED_TOPICS.add("unauthenticated")
|
|
60
|
+
except (ImportError, AttributeError):
|
|
61
|
+
pass
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _remember_repo(outer):
|
|
65
|
+
@functools.wraps(outer)
|
|
66
|
+
def wrapper(*args, **kwargs):
|
|
67
|
+
token = _repo.set((kwargs.get("repo_type"), kwargs.get("repo_id")))
|
|
68
|
+
try:
|
|
69
|
+
return outer(*args, **kwargs)
|
|
70
|
+
finally:
|
|
71
|
+
_repo.reset(token)
|
|
72
|
+
|
|
73
|
+
return wrapper
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _from_torrent(filename, etag, dest):
|
|
77
|
+
repo_type, repo_id = _repo.get() or (None, None)
|
|
78
|
+
if repo_id is None or repo_type not in (None, "model"):
|
|
79
|
+
return False
|
|
80
|
+
src = torrent.fetch_file(repo_id, filename)
|
|
81
|
+
if not _matches(src, etag):
|
|
82
|
+
raise ValueError("torrent copy didn't match Hugging Face's fingerprint")
|
|
83
|
+
_place(src, dest)
|
|
84
|
+
ui.from_swarm(filename, src.stat().st_size)
|
|
85
|
+
return True
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _matches(path, etag):
|
|
89
|
+
"""HF etags are SHA-256 for LFS files, git blob SHA-1 otherwise."""
|
|
90
|
+
etag = (etag or "").removeprefix("W/").strip('"')
|
|
91
|
+
if len(etag) == 64:
|
|
92
|
+
h = hashlib.sha256()
|
|
93
|
+
elif len(etag) == 40:
|
|
94
|
+
h = hashlib.sha1(b"blob %d\0" % path.stat().st_size)
|
|
95
|
+
else:
|
|
96
|
+
return False
|
|
97
|
+
with path.open("rb") as f:
|
|
98
|
+
while chunk := f.read(1 << 20):
|
|
99
|
+
h.update(chunk)
|
|
100
|
+
return h.hexdigest() == etag
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _place(src, dest):
|
|
104
|
+
"""Copy, not hard-link: HF makes its blobs read-only, which would lock libtorrent out of a shared inode."""
|
|
105
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
106
|
+
tmp = dest.with_name(dest.name + ".pf.incomplete")
|
|
107
|
+
try:
|
|
108
|
+
shutil.copyfile(src, tmp)
|
|
109
|
+
tmp.replace(dest)
|
|
110
|
+
finally:
|
|
111
|
+
tmp.unlink(missing_ok=True)
|
pirateface/ui.py
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import sys
|
|
3
|
+
import threading
|
|
4
|
+
|
|
5
|
+
from tqdm.auto import tqdm
|
|
6
|
+
|
|
7
|
+
from pirateface.config import flag
|
|
8
|
+
|
|
9
|
+
QUIET = flag("PF_QUIET")
|
|
10
|
+
_color = sys.stderr.isatty() and "NO_COLOR" not in os.environ
|
|
11
|
+
_lock = threading.Lock()
|
|
12
|
+
_announced = set()
|
|
13
|
+
|
|
14
|
+
SPLASH = r'''
|
|
15
|
+
_.-""""-._
|
|
16
|
+
__.-' (xx) '-.__
|
|
17
|
+
'-.________________.-'
|
|
18
|
+
/ \
|
|
19
|
+
| ^ (###)===|
|
|
20
|
+
\ \______/ /
|
|
21
|
+
'-.__________.-'
|
|
22
|
+
'''[1:].splitlines()
|
|
23
|
+
TAGLINE = ("pirateface", " Open-weight model downloads, from the swarm")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _c(code, text):
|
|
27
|
+
return f"\033[{code}m{text}\033[0m" if _color else text
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _print(line):
|
|
31
|
+
if not QUIET:
|
|
32
|
+
tqdm.write(line, file=sys.stderr)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _bar():
|
|
36
|
+
return _c("35", "┃")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def mb(n):
|
|
40
|
+
return f"{n / 1e6:.1f} MB" if n >= 1e5 else f"{n / 1e3:.1f} kB"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def splash():
|
|
44
|
+
if _announced or not sys.stderr.isatty():
|
|
45
|
+
return
|
|
46
|
+
pad = " " * ((len("".join(TAGLINE)) - max(map(len, SPLASH))) // 2)
|
|
47
|
+
art = [pad + (_c("1", line) if i < 3 else _c("33", line)) for i, line in enumerate(SPLASH)]
|
|
48
|
+
_print("\n".join(art) + "\n" + _c("1;35", TAGLINE[0]) + _c("2", TAGLINE[1]) + "\n")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def banner(repo_id, files, size):
|
|
52
|
+
with _lock:
|
|
53
|
+
if repo_id in _announced:
|
|
54
|
+
return
|
|
55
|
+
splash()
|
|
56
|
+
_announced.add(repo_id)
|
|
57
|
+
_print(
|
|
58
|
+
_bar()
|
|
59
|
+
+ " "
|
|
60
|
+
+ _c("1;35", "🏴☠️ pirateface")
|
|
61
|
+
+ f" {repo_id} is on Pirate Face ({files} files, {mb(size)}), downloading from the swarm"
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def not_on_pirateface(repo_id, rescue_requested=False):
|
|
66
|
+
with _lock:
|
|
67
|
+
if repo_id in _announced:
|
|
68
|
+
return
|
|
69
|
+
_announced.add(repo_id)
|
|
70
|
+
line = _bar() + " " + _c("1;35", "🏴☠️ pirateface") + f" {repo_id} isn't on Pirate Face yet, using Hugging Face"
|
|
71
|
+
if rescue_requested:
|
|
72
|
+
line += _c("2", " · requested a rescue (turn off: PF_NO_RESCUE=1)")
|
|
73
|
+
_print(line)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def repo_progress(repo_id):
|
|
77
|
+
return tqdm(
|
|
78
|
+
total=1,
|
|
79
|
+
leave=False,
|
|
80
|
+
file=sys.stderr,
|
|
81
|
+
disable=QUIET or not sys.stderr.isatty(),
|
|
82
|
+
desc=f"{_bar()} 🏴☠️ {repo_id}",
|
|
83
|
+
bar_format="{desc} {bar:20}",
|
|
84
|
+
colour="magenta",
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def set_status(bar, text):
|
|
89
|
+
# Not tqdm's postfix: it always prepends ", ".
|
|
90
|
+
bar.bar_format = "{desc} {bar:20} " + text.replace("{", "{{").replace("}", "}}")
|
|
91
|
+
bar.refresh()
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def sources_label(webseed, peers):
|
|
95
|
+
if webseed and peers:
|
|
96
|
+
return f"web seed + {peers} peer{'s' * (peers > 1)}"
|
|
97
|
+
if webseed:
|
|
98
|
+
return "web seed"
|
|
99
|
+
if peers:
|
|
100
|
+
return f"{peers} peer{'s' * (peers > 1)}"
|
|
101
|
+
return "finding sources…"
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def from_swarm(filename, size):
|
|
105
|
+
_print(
|
|
106
|
+
f"{_bar()} {_c('32', '✔')} {filename} {_c('2', mb(size) + ' from the swarm, fingerprint matches Hugging Face')}"
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def from_hf(filename, reason=None):
|
|
111
|
+
if reason:
|
|
112
|
+
_print(f"{_bar()} {_c('33', '↩')} {filename} {_c('2', 'from Hugging Face (' + reason + ')')}")
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: pirateface
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Download Hugging Face models over BitTorrent from Pirate Face seeders. No code changes.
|
|
5
|
+
Project-URL: Homepage, https://pirateface.co
|
|
6
|
+
Project-URL: Source, https://github.com/pirateface/pirateface_hub
|
|
7
|
+
Project-URL: Issues, https://github.com/pirateface/pirateface_hub/issues
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: bittorrent,download,huggingface,libtorrent,models
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
15
|
+
Requires-Python: >=3.12
|
|
16
|
+
Requires-Dist: huggingface-hub>=2.0
|
|
17
|
+
Requires-Dist: libtorrent>=2.0
|
|
18
|
+
Requires-Dist: tqdm>=4.42
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
|
|
21
|
+
# Pirate Face Hub
|
|
22
|
+
|
|
23
|
+
**Plug and play torrent downloads for Hugging Face models.** Add one package. Change nothing else.
|
|
24
|
+
|
|
25
|
+
```sh
|
|
26
|
+
uv run --with pirateface python your_script.py
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
or install it once and forget about it:
|
|
30
|
+
|
|
31
|
+
```sh
|
|
32
|
+
pip install pirateface
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
That's the whole setup. No code changes, no config, no new API. Your script still calls transformers or
|
|
36
|
+
`huggingface_hub` like it always did, and the files now come from [Pirate Face](https://pirateface.co)
|
|
37
|
+
seeders over BitTorrent:
|
|
38
|
+
|
|
39
|
+
```python
|
|
40
|
+
from transformers import AutoModelForCausalLM
|
|
41
|
+
|
|
42
|
+
model = AutoModelForCausalLM.from_pretrained("org/model") # unchanged, now from the swarm
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
The same goes for anything that downloads through `huggingface_hub`: diffusers, sentence-transformers,
|
|
46
|
+
`hf_hub_download`, `snapshot_download` and the `hf download` CLI.
|
|
47
|
+
|
|
48
|
+
Every file is checked against Hugging Face's own fingerprint (SHA-256 for large files, git SHA-1 for the
|
|
49
|
+
rest) before it goes into the normal Hugging Face cache. If Pirate Face doesn't have the model, a peer is
|
|
50
|
+
missing, or a check fails, that file comes from Hugging Face as usual. Uninstall it and your code
|
|
51
|
+
runs exactly as before.
|
|
52
|
+
|
|
53
|
+
There's also a small CLI:
|
|
54
|
+
|
|
55
|
+
```sh
|
|
56
|
+
pf download org/model [--revision REV]
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## How it hooks in
|
|
60
|
+
|
|
61
|
+
The package installs `pirateface.pth`, which Python runs at startup. It does nothing until something
|
|
62
|
+
imports `huggingface_hub`, then fetches each file's bytes from the swarm. Hugging Face still answers
|
|
63
|
+
which commit and fingerprint to expect, and every file from the swarm is checked against it. If Pirate
|
|
64
|
+
Face has no torrent, or no peer sends the file list within 30 s, the file comes from Hugging Face as usual.
|
|
65
|
+
Cache layout, symlinks and locks stay with `huggingface_hub`.
|
|
66
|
+
|
|
67
|
+
## Settings
|
|
68
|
+
|
|
69
|
+
| Variable | Default | |
|
|
70
|
+
|---|---|---|
|
|
71
|
+
| `PF_DISABLE` | off | Turn the hook off |
|
|
72
|
+
| `PF_QUIET` | off | Print nothing |
|
|
73
|
+
| `PF_NO_RESCUE` | off | Don't ask Pirate Face to add models it doesn't have yet |
|
|
74
|
+
| `PF_DEBUG` | off | Print libtorrent's log |
|
|
75
|
+
| `PF_CACHE` | `~/.cache/pirateface` | Where torrent data and file lists are kept |
|
|
76
|
+
| `PF_API` | `https://pirateface.co` | Pirate Face server |
|
|
77
|
+
| `PF_LISTEN` | any free port | libtorrent listen interfaces |
|
|
78
|
+
| `PF_METADATA_TIMEOUT` | `30` | Seconds to wait for a peer to send the file list |
|
|
79
|
+
| `PF_STALL_TIMEOUT` | `60` | Seconds without progress before falling back to Hugging Face |
|
|
80
|
+
|
|
81
|
+
Flags are on for any value except empty or `0`.
|
|
82
|
+
|
|
83
|
+
Needs Python 3.12 or 3.13 (libtorrent has no wheels for 3.14 yet).
|
|
84
|
+
|
|
85
|
+
## Development
|
|
86
|
+
|
|
87
|
+
```sh
|
|
88
|
+
uv run --group dev ruff check
|
|
89
|
+
```
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
pirateface/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
2
|
+
pirateface/_boot.py,sha256=KE2q3VjrPA5Fr06cyol-08GcbbH_HNoxQg_T8lUm5Yg,1254
|
|
3
|
+
pirateface/cli.py,sha256=3pNChsiZ10UcOa_NBLBEG0-THLxG_qjewcVp0D7ASyw,505
|
|
4
|
+
pirateface/config.py,sha256=CO518BToBT4_zh2O8FLrPNFyxYNruLsUpw4vgzy6-_o,468
|
|
5
|
+
pirateface/huggingface.py,sha256=ZXOwqJSDWO7TLTxyavdWfZP-bdX_QvBtaf25VXpVg-U,3468
|
|
6
|
+
pirateface/ui.py,sha256=PumXsIDNDEC66truEZOW7FJeOYDNuwPtRNKZpeiL0xk,2954
|
|
7
|
+
pirateface/client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
pirateface/client/api.py,sha256=uBbsIPrgeoxrbKq7h9Lg8k70y8plA1FqN7X00RUKfXQ,1655
|
|
9
|
+
pirateface/client/torrent.py,sha256=lJuSrEqUy1HgRVB3ibbwdcL9EDK1i8DoYpqrxjQtlZo,7773
|
|
10
|
+
pirateface.pth,sha256=td-k_oJyYi0LmTbM21u12-JJznkRxyw-KyKYUxKSEWk,24
|
|
11
|
+
pirateface-0.1.0.dist-info/METADATA,sha256=vFA8j7KOlKeRrtoZxe4Aex_dBp2ohFzXxW46V5iE1ik,3340
|
|
12
|
+
pirateface-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
13
|
+
pirateface-0.1.0.dist-info/entry_points.txt,sha256=QZGSjlOv0Ei3LVt2XIUddorqA5iyzSsRjCUMdRfzqaI,43
|
|
14
|
+
pirateface-0.1.0.dist-info/licenses/LICENSE,sha256=C5oOi3ki7RFP562DUN0iOTYaV7XHDr6fBao5NMM5djw,1068
|
|
15
|
+
pirateface-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Pirate Face
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
pirateface.pth
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import pirateface._boot
|