weed-cli 1.8.2__tar.gz → 1.8.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {weed_cli-1.8.2/weed_cli.egg-info → weed_cli-1.8.4}/PKG-INFO +2 -1
- {weed_cli-1.8.2 → weed_cli-1.8.4}/pyproject.toml +10 -1
- weed_cli-1.8.4/tests/test_web_ui_api.py +273 -0
- {weed_cli-1.8.2 → weed_cli-1.8.4}/tests/testutil.py +14 -0
- {weed_cli-1.8.2 → weed_cli-1.8.4}/web_ui.py +163 -0
- {weed_cli-1.8.2 → weed_cli-1.8.4/weed_cli.egg-info}/PKG-INFO +2 -1
- {weed_cli-1.8.2 → weed_cli-1.8.4}/weed_cli.egg-info/requires.txt +1 -0
- weed_cli-1.8.2/tests/test_web_ui_api.py +0 -127
- {weed_cli-1.8.2 → weed_cli-1.8.4}/LICENSE +0 -0
- {weed_cli-1.8.2 → weed_cli-1.8.4}/README.md +0 -0
- {weed_cli-1.8.2 → weed_cli-1.8.4}/dht.py +0 -0
- {weed_cli-1.8.2 → weed_cli-1.8.4}/discovery_relay.py +0 -0
- {weed_cli-1.8.2 → weed_cli-1.8.4}/lightning_settle.py +0 -0
- {weed_cli-1.8.2 → weed_cli-1.8.4}/node.py +0 -0
- {weed_cli-1.8.2 → weed_cli-1.8.4}/poc_reputation.py +0 -0
- {weed_cli-1.8.2 → weed_cli-1.8.4}/setup.cfg +0 -0
- {weed_cli-1.8.2 → weed_cli-1.8.4}/shell.py +0 -0
- {weed_cli-1.8.2 → weed_cli-1.8.4}/tests/test_dht.py +0 -0
- {weed_cli-1.8.2 → weed_cli-1.8.4}/tests/test_discovery_relay.py +0 -0
- {weed_cli-1.8.2 → weed_cli-1.8.4}/tests/test_node_manifest.py +0 -0
- {weed_cli-1.8.2 → weed_cli-1.8.4}/tunnel_relay.py +0 -0
- {weed_cli-1.8.2 → weed_cli-1.8.4}/weed.py +0 -0
- {weed_cli-1.8.2 → weed_cli-1.8.4}/weed_cli.egg-info/SOURCES.txt +0 -0
- {weed_cli-1.8.2 → weed_cli-1.8.4}/weed_cli.egg-info/dependency_links.txt +0 -0
- {weed_cli-1.8.2 → weed_cli-1.8.4}/weed_cli.egg-info/entry_points.txt +0 -0
- {weed_cli-1.8.2 → weed_cli-1.8.4}/weed_cli.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: weed-cli
|
|
3
|
-
Version: 1.8.
|
|
3
|
+
Version: 1.8.4
|
|
4
4
|
Summary: Censorship-resistant video PoC — discovery/hosting/download over signed relay events, a real Kademlia DHT, or a TLS-capable NAT-traversal tunnel
|
|
5
5
|
License: MIT
|
|
6
6
|
Keywords: p2p,video,censorship-resistant,discovery,dht,kademlia,nat-traversal
|
|
@@ -21,6 +21,7 @@ Requires-Dist: pytest>=8.0; extra == "dev"
|
|
|
21
21
|
Requires-Dist: pytest-cov>=5.0; extra == "dev"
|
|
22
22
|
Requires-Dist: ruff>=0.5; extra == "dev"
|
|
23
23
|
Requires-Dist: pytest-playwright>=0.5; extra == "dev"
|
|
24
|
+
Requires-Dist: weed-cli[dht]; extra == "dev"
|
|
24
25
|
Provides-Extra: dht
|
|
25
26
|
Requires-Dist: kademlia>=2.2; extra == "dht"
|
|
26
27
|
Provides-Extra: qr
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "weed-cli"
|
|
7
|
-
version = "1.8.
|
|
7
|
+
version = "1.8.4"
|
|
8
8
|
description = "Censorship-resistant video PoC — discovery/hosting/download over signed relay events, a real Kademlia DHT, or a TLS-capable NAT-traversal tunnel"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = { text = "MIT" }
|
|
@@ -45,6 +45,15 @@ dev = [
|
|
|
45
45
|
# too -- `playwright install chromium`, or point WEED_TEST_CHROMIUM at
|
|
46
46
|
# an existing one (see tests/e2e/conftest.py).
|
|
47
47
|
"pytest-playwright>=0.5",
|
|
48
|
+
# tests/test_dht.py imports dht.py, which imports kademlia at module
|
|
49
|
+
# level -- self-referencing the dht extra (not duplicating its version
|
|
50
|
+
# constraint here) means bumping kademlia's pin in one place stays
|
|
51
|
+
# enough. Missing this is exactly what broke CI: `pip install -e .[dev]`
|
|
52
|
+
# alone left kademlia (an otherwise-optional runtime dependency, real
|
|
53
|
+
# users of plain relay/DHT-less discovery never need it) uninstalled,
|
|
54
|
+
# so importing dht.py at test collection time raised ModuleNotFoundError
|
|
55
|
+
# before a single test even ran.
|
|
56
|
+
"weed-cli[dht]",
|
|
48
57
|
]
|
|
49
58
|
dht = [
|
|
50
59
|
"kademlia>=2.2",
|
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
"""
|
|
2
|
+
web_ui.py's REST API against a real, isolated WebUIServer instance (see
|
|
3
|
+
conftest.web_server) -- real HTTP requests via the stdlib, same as the
|
|
4
|
+
rest of this codebase's own "no new dependency" convention. Every
|
|
5
|
+
identity/library/hosts file this touches is redirected into tmp_path by
|
|
6
|
+
the isolated_paths fixture web_server depends on; nothing here can ever
|
|
7
|
+
read or write a real ~/.weed_* file.
|
|
8
|
+
"""
|
|
9
|
+
import json
|
|
10
|
+
import os
|
|
11
|
+
import threading
|
|
12
|
+
import urllib.parse
|
|
13
|
+
|
|
14
|
+
import node
|
|
15
|
+
from testutil import http_get_json, http_post_json, http_post_raw
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def test_whoami_and_empty_library(web_server):
|
|
19
|
+
who = http_get_json(f'{web_server}/api/whoami')
|
|
20
|
+
assert len(who['pubkey']) == 64 # 32-byte Ed25519 pubkey, hex-encoded
|
|
21
|
+
|
|
22
|
+
lib = http_get_json(f'{web_server}/api/library')
|
|
23
|
+
assert lib == {'downloads': [], 'likes': [], 'subscriptions': [], 'playlists': [], 'history': []}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def test_like_is_idempotent(web_server):
|
|
27
|
+
status, resp = http_post_json(f'{web_server}/api/like', {'content_hash': 'c' * 64})
|
|
28
|
+
assert status == 200
|
|
29
|
+
for _ in range(3):
|
|
30
|
+
http_post_json(f'{web_server}/api/like', {'content_hash': 'c' * 64})
|
|
31
|
+
lib = http_get_json(f'{web_server}/api/library')
|
|
32
|
+
assert lib['likes'] == ['c' * 64]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def test_like_requires_content_hash(web_server):
|
|
36
|
+
status, resp = http_post_json(f'{web_server}/api/like', {})
|
|
37
|
+
assert status == 400
|
|
38
|
+
assert 'content_hash' in resp['error']
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def test_playlist_create_add_reorder_remove_delete(web_server):
|
|
42
|
+
status, resp = http_post_json(f'{web_server}/api/playlists/create', {'name': 'My List'})
|
|
43
|
+
assert status == 200
|
|
44
|
+
playlist_id = resp['playlist']['id']
|
|
45
|
+
|
|
46
|
+
for h in ('a' * 64, 'b' * 64):
|
|
47
|
+
status, resp = http_post_json(f'{web_server}/api/playlists/add', {
|
|
48
|
+
'playlist_id': playlist_id, 'content_hash': h, 'title': h[:8],
|
|
49
|
+
})
|
|
50
|
+
assert status == 200
|
|
51
|
+
|
|
52
|
+
lib = http_get_json(f'{web_server}/api/library')
|
|
53
|
+
pl = next(p for p in lib['playlists'] if p['id'] == playlist_id)
|
|
54
|
+
assert [it['content_hash'] for it in pl['items']] == ['a' * 64, 'b' * 64]
|
|
55
|
+
|
|
56
|
+
status, _ = http_post_json(f'{web_server}/api/playlists/reorder', {
|
|
57
|
+
'playlist_id': playlist_id, 'order': ['b' * 64, 'a' * 64],
|
|
58
|
+
})
|
|
59
|
+
assert status == 200
|
|
60
|
+
lib = http_get_json(f'{web_server}/api/library')
|
|
61
|
+
pl = next(p for p in lib['playlists'] if p['id'] == playlist_id)
|
|
62
|
+
assert [it['content_hash'] for it in pl['items']] == ['b' * 64, 'a' * 64]
|
|
63
|
+
|
|
64
|
+
status, _ = http_post_json(f'{web_server}/api/playlists/remove', {
|
|
65
|
+
'playlist_id': playlist_id, 'content_hash': 'a' * 64,
|
|
66
|
+
})
|
|
67
|
+
assert status == 200
|
|
68
|
+
lib = http_get_json(f'{web_server}/api/library')
|
|
69
|
+
pl = next(p for p in lib['playlists'] if p['id'] == playlist_id)
|
|
70
|
+
assert [it['content_hash'] for it in pl['items']] == ['b' * 64]
|
|
71
|
+
|
|
72
|
+
status, _ = http_post_json(f'{web_server}/api/playlists/delete', {'playlist_id': playlist_id})
|
|
73
|
+
assert status == 200
|
|
74
|
+
lib = http_get_json(f'{web_server}/api/library')
|
|
75
|
+
assert lib['playlists'] == []
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def test_play_requires_content_hash(web_server):
|
|
79
|
+
status, resp = http_post_json(f'{web_server}/api/play', {})
|
|
80
|
+
assert status == 400
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def test_play_bumps_count_and_history_for_a_known_download(web_server):
|
|
84
|
+
import web_ui
|
|
85
|
+
web_ui._library['downloads']['c' * 64] = {
|
|
86
|
+
'content_hash': 'c' * 64, 'job_id': 'job1', 'path': '/tmp/x.mp4',
|
|
87
|
+
'title': 'X', 'downloaded_at': 0, 'size': 1, 'bps': 1, 'signer_pubkey': None,
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
status, resp = http_post_json(f'{web_server}/api/play', {'content_hash': 'c' * 64, 'title': 'X'})
|
|
91
|
+
assert status == 200
|
|
92
|
+
assert resp['play_count'] == 1
|
|
93
|
+
assert resp['last_played'] is not None
|
|
94
|
+
|
|
95
|
+
status, resp = http_post_json(f'{web_server}/api/play', {'content_hash': 'c' * 64, 'title': 'X'})
|
|
96
|
+
assert resp['play_count'] == 2
|
|
97
|
+
|
|
98
|
+
lib = http_get_json(f'{web_server}/api/library')
|
|
99
|
+
rec = next(d for d in lib['downloads'] if d['content_hash'] == 'c' * 64)
|
|
100
|
+
assert rec['play_count'] == 2
|
|
101
|
+
assert len(lib['history']) == 2
|
|
102
|
+
assert lib['history'][0]['content_hash'] == 'c' * 64 # newest first
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def test_play_for_unknown_content_hash_still_logs_history_but_no_count(web_server):
|
|
106
|
+
"""A play_count only exists on a downloads record -- playing something
|
|
107
|
+
that isn't (yet, or ever) in _library['downloads'] shouldn't 500, it
|
|
108
|
+
just can't report a play_count."""
|
|
109
|
+
status, resp = http_post_json(f'{web_server}/api/play', {'content_hash': 'z' * 64, 'title': 'Ghost'})
|
|
110
|
+
assert status == 200
|
|
111
|
+
assert resp['play_count'] is None
|
|
112
|
+
|
|
113
|
+
lib = http_get_json(f'{web_server}/api/library')
|
|
114
|
+
assert len(lib['history']) == 1
|
|
115
|
+
assert lib['history'][0]['title'] == 'Ghost'
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _upload_url(web_server, name, archive_dir):
|
|
119
|
+
qs = urllib.parse.urlencode({'name': name, 'archive_dir': archive_dir})
|
|
120
|
+
return f'{web_server}/api/upload?{qs}'
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def test_upload_video_produces_a_real_hostable_archive(web_server, tmp_path):
|
|
124
|
+
"""End to end: upload real bytes, then confirm node.py's own
|
|
125
|
+
manifest/chunk readers (the actual code `host` runs) can load the
|
|
126
|
+
result back correctly -- not just that the endpoint returned 200."""
|
|
127
|
+
archive_dir = str(tmp_path / 'archive')
|
|
128
|
+
data = os.urandom(200_000)
|
|
129
|
+
status, resp = http_post_raw(_upload_url(web_server, 'clip.mp4', archive_dir), data)
|
|
130
|
+
|
|
131
|
+
assert status == 200
|
|
132
|
+
assert resp['ok'] is True
|
|
133
|
+
assert resp['name'] == 'clip.mp4'
|
|
134
|
+
assert len(resp['content_hash']) == 64
|
|
135
|
+
assert resp['n_chunks'] >= 1
|
|
136
|
+
|
|
137
|
+
dest = os.path.join(archive_dir, 'clip.mp4')
|
|
138
|
+
assert os.path.isfile(dest)
|
|
139
|
+
assert os.path.getsize(dest) == len(data)
|
|
140
|
+
assert not os.path.exists(dest + '.uploading') # tmp file cleaned up
|
|
141
|
+
|
|
142
|
+
entries = node.load_manifest_entries(archive_dir)
|
|
143
|
+
assert len(entries) == 1
|
|
144
|
+
assert entries[0]['sha256'] == resp['content_hash']
|
|
145
|
+
leaves = node.load_leaves(archive_dir, resp['content_hash'])
|
|
146
|
+
assert len(leaves) == resp['n_chunks']
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def test_upload_appends_correctly_when_existing_manifest_has_no_trailing_newline(web_server, tmp_path):
|
|
150
|
+
"""Real incident, not a hypothetical: an existing manifest.jsonl that
|
|
151
|
+
doesn't end in a newline (this archive's did not) used to get a new
|
|
152
|
+
entry appended directly onto the end of the last line via a bare
|
|
153
|
+
open(path, 'a') -- merging two JSON objects into one unparseable
|
|
154
|
+
line. node.load_manifest_entries has no per-line error handling, so
|
|
155
|
+
that one bad line broke reading the *entire* manifest, not just the
|
|
156
|
+
new upload -- every pre-existing file in the archive became
|
|
157
|
+
unloadable ("files could not be found") until the manifest was
|
|
158
|
+
rebuilt from scratch."""
|
|
159
|
+
archive_dir = tmp_path / 'archive'
|
|
160
|
+
ott_dir = archive_dir / '.ott'
|
|
161
|
+
ott_dir.mkdir(parents=True)
|
|
162
|
+
pre_existing = {'sha256': 'b' * 64, 'name': 'old.mp4', 'orig_path': 'old.mp4',
|
|
163
|
+
'last_path': str(archive_dir / 'old.mp4'), 'size': 1,
|
|
164
|
+
'added': '2020-01-01T00:00:00Z', 'type': 'video', 'n_chunks': 1, 'chunk_size': 262144}
|
|
165
|
+
# deliberately no trailing newline -- this is the exact condition that broke it
|
|
166
|
+
(ott_dir / 'manifest.jsonl').write_text(json.dumps(pre_existing))
|
|
167
|
+
|
|
168
|
+
status, resp = http_post_raw(_upload_url(web_server, 'new.mp4', str(archive_dir)), os.urandom(50_000))
|
|
169
|
+
assert status == 200
|
|
170
|
+
|
|
171
|
+
# the real assertion: BOTH entries must still be independently
|
|
172
|
+
# loadable afterward, old and new alike
|
|
173
|
+
entries = node.load_manifest_entries(str(archive_dir))
|
|
174
|
+
names = {e['name'] for e in entries}
|
|
175
|
+
assert names == {'old.mp4', 'new.mp4'}
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def test_concurrent_uploads_to_the_same_archive_dont_lose_an_entry(web_server, tmp_path):
|
|
179
|
+
"""Dropping several files at once in the browser fires one upload
|
|
180
|
+
request per file, concurrently -- all racing to read-modify-write the
|
|
181
|
+
same manifest.jsonl. Without a lock around that, two requests can
|
|
182
|
+
both read the same "before" state and whichever writes last wins,
|
|
183
|
+
silently dropping the other's entry."""
|
|
184
|
+
archive_dir = str(tmp_path / 'archive')
|
|
185
|
+
threads = [
|
|
186
|
+
threading.Thread(target=http_post_raw,
|
|
187
|
+
args=(_upload_url(web_server, f'concurrent{i}.mp4', archive_dir), os.urandom(20_000)))
|
|
188
|
+
for i in range(8)
|
|
189
|
+
]
|
|
190
|
+
for t in threads:
|
|
191
|
+
t.start()
|
|
192
|
+
for t in threads:
|
|
193
|
+
t.join()
|
|
194
|
+
|
|
195
|
+
entries = node.load_manifest_entries(archive_dir)
|
|
196
|
+
assert {e['name'] for e in entries} == {f'concurrent{i}.mp4' for i in range(8)}
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def test_upload_rejects_non_video_extension(web_server, tmp_path):
|
|
200
|
+
archive_dir = str(tmp_path / 'archive')
|
|
201
|
+
status, resp = http_post_raw(_upload_url(web_server, 'notes.txt', archive_dir), b'hello')
|
|
202
|
+
assert status == 400
|
|
203
|
+
assert 'not a recognized video extension' in resp['error']
|
|
204
|
+
assert not os.path.exists(archive_dir) # never even created
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def test_upload_sanitizes_path_traversal_in_filename(web_server, tmp_path):
|
|
208
|
+
archive_dir = str(tmp_path / 'archive')
|
|
209
|
+
outside_target = tmp_path / 'evil.mp4'
|
|
210
|
+
status, resp = http_post_raw(
|
|
211
|
+
_upload_url(web_server, '../evil.mp4', archive_dir), os.urandom(1000))
|
|
212
|
+
assert status == 200 # basename strips the traversal, so this is just "evil.mp4" inside archive_dir
|
|
213
|
+
assert resp['name'] == 'evil.mp4'
|
|
214
|
+
assert not outside_target.exists() # never escaped archive_dir
|
|
215
|
+
assert (tmp_path / 'archive' / 'evil.mp4').exists()
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def test_upload_requires_name_param(web_server, tmp_path):
|
|
219
|
+
conn_status, resp = http_post_raw(
|
|
220
|
+
f'{web_server}/api/upload?archive_dir=' + urllib.parse.quote(str(tmp_path)), b'data')
|
|
221
|
+
assert conn_status == 400
|
|
222
|
+
assert 'name' in resp['error']
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def test_upload_defaults_archive_dir_when_omitted(web_server, tmp_path, monkeypatch):
|
|
226
|
+
"""No archive_dir query param at all, and no /share directory present
|
|
227
|
+
(the bare-metal / dev case) -- falls back to './share', resolved
|
|
228
|
+
relative to the server process's cwd, which is also this test process
|
|
229
|
+
(web_server runs in a background thread, not a subprocess) --
|
|
230
|
+
monkeypatch.chdir into tmp_path first so that resolves somewhere
|
|
231
|
+
throwaway instead of this repo's own real ./share (which has real,
|
|
232
|
+
non-test content -- see the repo root). It's extremely unlikely this
|
|
233
|
+
test machine has a real /share directory, but see
|
|
234
|
+
test_default_upload_archive_dir_unit below for the Docker-mount-
|
|
235
|
+
present branch, tested in isolation instead of through a real
|
|
236
|
+
filesystem write to a faked-out /share."""
|
|
237
|
+
monkeypatch.chdir(tmp_path)
|
|
238
|
+
status, resp = http_post_raw(f'{web_server}/api/upload?name=clip.mp4', os.urandom(1000))
|
|
239
|
+
assert status == 200
|
|
240
|
+
assert resp['archive_dir'] == './share'
|
|
241
|
+
assert (tmp_path / 'share' / 'clip.mp4').exists()
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def test_default_upload_archive_dir_unit(monkeypatch):
|
|
245
|
+
"""web_ui._default_upload_archive_dir() in isolation, covering both
|
|
246
|
+
branches without ever touching a real filesystem path -- see its own
|
|
247
|
+
docstring for the real incident (a file archived into /app/share
|
|
248
|
+
inside the container while the user's /share went on looking empty,
|
|
249
|
+
fixable only by restarting) that this default exists to prevent."""
|
|
250
|
+
import web_ui
|
|
251
|
+
monkeypatch.setattr(os.path, 'isdir', lambda p: p == '/share')
|
|
252
|
+
assert web_ui._default_upload_archive_dir() == '/share'
|
|
253
|
+
|
|
254
|
+
monkeypatch.setattr(os.path, 'isdir', lambda p: False)
|
|
255
|
+
assert web_ui._default_upload_archive_dir() == './share'
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def test_cross_origin_post_rejected(web_server):
|
|
259
|
+
"""No auth at all by design (see web_ui.py's module docstring) --
|
|
260
|
+
Origin-checking is the only thing standing between this and any other
|
|
261
|
+
open tab silently POSTing here. A request claiming a different Origin
|
|
262
|
+
must be refused."""
|
|
263
|
+
status, resp = http_post_json(f'{web_server}/api/like', {'content_hash': 'c' * 64},
|
|
264
|
+
headers={'Origin': 'http://evil.example'})
|
|
265
|
+
assert status == 403
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def test_request_with_no_origin_header_is_allowed(web_server):
|
|
269
|
+
"""curl / server-to-server / direct API use never sets Origin -- only
|
|
270
|
+
a real cross-site browser request does, so a missing header is let
|
|
271
|
+
through (see web_ui.py's _check_origin docstring)."""
|
|
272
|
+
status, resp = http_post_json(f'{web_server}/api/like', {'content_hash': 'd' * 64})
|
|
273
|
+
assert status == 200
|
|
@@ -66,6 +66,20 @@ def http_post_json(url, body, headers=None):
|
|
|
66
66
|
conn.close()
|
|
67
67
|
|
|
68
68
|
|
|
69
|
+
def http_post_raw(url, data, headers=None):
|
|
70
|
+
"""For /api/upload -- the body is the raw file bytes, not JSON."""
|
|
71
|
+
conn_host, conn_port, path = _split_url(url)
|
|
72
|
+
conn = http.client.HTTPConnection(conn_host, conn_port, timeout=15)
|
|
73
|
+
try:
|
|
74
|
+
h = {'Content-Type': 'application/octet-stream'}
|
|
75
|
+
h.update(headers or {})
|
|
76
|
+
conn.request('POST', path, body=data, headers=h)
|
|
77
|
+
resp = conn.getresponse()
|
|
78
|
+
return resp.status, json.loads(resp.read().decode())
|
|
79
|
+
finally:
|
|
80
|
+
conn.close()
|
|
81
|
+
|
|
82
|
+
|
|
69
83
|
def _split_url(url):
|
|
70
84
|
# http.client wants (host, port) and a bare path, not a full URL
|
|
71
85
|
assert url.startswith('http://')
|
|
@@ -14,6 +14,7 @@ already apply elsewhere in this repo). Pass --bind to expose it on a LAN
|
|
|
14
14
|
at your own risk.
|
|
15
15
|
"""
|
|
16
16
|
import contextlib
|
|
17
|
+
import hashlib
|
|
17
18
|
import io
|
|
18
19
|
import json
|
|
19
20
|
import mimetypes
|
|
@@ -184,6 +185,23 @@ def _save_persisted_hosts():
|
|
|
184
185
|
os.replace(tmp, HOSTS_PATH)
|
|
185
186
|
|
|
186
187
|
|
|
188
|
+
def _default_upload_archive_dir():
|
|
189
|
+
"""/share only exists as a real directory when running inside the
|
|
190
|
+
Docker image built from Dockerfile.node -- docker-compose.node.yml
|
|
191
|
+
bind-mounts the host's archive there, and its own comments tell the
|
|
192
|
+
user to type /share into this exact form field. Defaulting an
|
|
193
|
+
omitted archive_dir to the relative './share' instead would land an
|
|
194
|
+
upload in this process's cwd (/app inside that container), a
|
|
195
|
+
directory nobody else is looking at: the file would archive fine,
|
|
196
|
+
but every subsequent /api/host call against the /share the user
|
|
197
|
+
actually typed would report "no archived file found", with no
|
|
198
|
+
restart able to fix it since the file was never in /share to begin
|
|
199
|
+
with. Outside Docker, /share won't exist and this falls back to
|
|
200
|
+
'./share', matching docker-compose.node.yml's own default bind-mount
|
|
201
|
+
source on the host side."""
|
|
202
|
+
return '/share' if os.path.isdir('/share') else './share'
|
|
203
|
+
|
|
204
|
+
|
|
187
205
|
def _remember_host(archive_dir, file_name, port, price, relay_urls, advertise_host, tunnel, ln_node):
|
|
188
206
|
key = f'{archive_dir}|{file_name}|{port}'
|
|
189
207
|
_persisted_hosts[key] = {
|
|
@@ -578,6 +596,21 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
578
596
|
if not self._check_origin():
|
|
579
597
|
return self._json({'error': 'rejected: request Origin does not match this server — '
|
|
580
598
|
'looks like a cross-site request, not this UI'}, status=403)
|
|
599
|
+
|
|
600
|
+
# Not a JSON-body endpoint like everything else here -- the body
|
|
601
|
+
# *is* the raw file being uploaded (see _handle_upload's own
|
|
602
|
+
# docstring for why: no multipart/form-data parser exists in the
|
|
603
|
+
# stdlib, and this repo's own "no new dependency" rule already
|
|
604
|
+
# ruled one out elsewhere). Handled before _read_json_body ever
|
|
605
|
+
# runs, since that would try to json.loads() raw video bytes and
|
|
606
|
+
# fail every single upload with a confusing "bad JSON body" error
|
|
607
|
+
# before this endpoint's own code ever ran.
|
|
608
|
+
if path == '/api/upload':
|
|
609
|
+
try:
|
|
610
|
+
return self._handle_upload()
|
|
611
|
+
except Exception as e:
|
|
612
|
+
return self._json({'error': f'{type(e).__name__}: {e}'}, status=400)
|
|
613
|
+
|
|
581
614
|
try:
|
|
582
615
|
body = self._read_json_body()
|
|
583
616
|
except Exception as e:
|
|
@@ -602,6 +635,136 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
602
635
|
except Exception as e:
|
|
603
636
|
self._json({'error': f'{type(e).__name__}: {e}'}, status=400)
|
|
604
637
|
|
|
638
|
+
def _handle_upload(self):
|
|
639
|
+
"""POST /api/upload?name=<file>&archive_dir=<dir> -- the file's raw
|
|
640
|
+
bytes as the whole request body (application/octet-stream, not
|
|
641
|
+
multipart/form-data: there's no multipart parser in the stdlib,
|
|
642
|
+
and pulling in a dependency just for this is exactly the kind of
|
|
643
|
+
thing this file's own module docstring already rules out
|
|
644
|
+
elsewhere). Streams straight to disk in fixed-size chunks rather
|
|
645
|
+
than reading the whole body into memory first -- fine for a
|
|
646
|
+
JSON API's few-KB bodies, not for a multi-GB video.
|
|
647
|
+
|
|
648
|
+
Archives it immediately (chunks + a real manifest.jsonl entry,
|
|
649
|
+
the exact on-disk shape node.load_manifest_entries/load_leaves
|
|
650
|
+
already read) so it's hostable the moment the upload finishes,
|
|
651
|
+
with no separate `ott add` step -- same reasoning as the rest of
|
|
652
|
+
this UI existing at all: don't make someone learn a second tool
|
|
653
|
+
just to do the thing this one already knows how to do.
|
|
654
|
+
Video-only, matching ott's own is_video() -- a non-video upload
|
|
655
|
+
would just be a manifest entry that can never actually be
|
|
656
|
+
hosted (see load_manifest_entries' own video-only filter, added
|
|
657
|
+
after exactly that silently broke `host` for everything else in
|
|
658
|
+
the same archive_dir), so it's rejected up front instead."""
|
|
659
|
+
from ott import is_video, chunk_hashes, merkle_root, OttStore
|
|
660
|
+
|
|
661
|
+
qs = parse_qs(urlparse(self.path).query)
|
|
662
|
+
raw_name = (qs.get('name') or [''])[0]
|
|
663
|
+
archive_dir = (qs.get('archive_dir') or [None])[0] or _default_upload_archive_dir()
|
|
664
|
+
if not raw_name:
|
|
665
|
+
return self._json({'error': 'name query param required'}, status=400)
|
|
666
|
+
|
|
667
|
+
# basename only -- '..' or an absolute path in the filename
|
|
668
|
+
# can't escape archive_dir this way
|
|
669
|
+
safe_name = os.path.basename(raw_name)
|
|
670
|
+
if not safe_name or safe_name in ('.', '..'):
|
|
671
|
+
return self._json({'error': f'invalid file name: {raw_name!r}'}, status=400)
|
|
672
|
+
|
|
673
|
+
if not is_video(safe_name):
|
|
674
|
+
return self._json(
|
|
675
|
+
{'error': f'{safe_name}: not a recognized video extension -- only video '
|
|
676
|
+
'files can be hosted (see ott.is_video)'}, status=400)
|
|
677
|
+
|
|
678
|
+
archive_dir = os.path.expanduser(archive_dir)
|
|
679
|
+
os.makedirs(archive_dir, exist_ok=True)
|
|
680
|
+
dest_path = os.path.join(archive_dir, safe_name)
|
|
681
|
+
|
|
682
|
+
length = int(self.headers.get('Content-Length', 0))
|
|
683
|
+
if length <= 0:
|
|
684
|
+
return self._json({'error': 'empty upload'}, status=400)
|
|
685
|
+
|
|
686
|
+
# write to a temp name and os.replace at the end, same reasoning
|
|
687
|
+
# as _save_library's own tmp-file-then-replace: a client
|
|
688
|
+
# disconnecting mid-upload (closed laptop lid, flaky wifi) must
|
|
689
|
+
# not leave a truncated file sitting at the real destination
|
|
690
|
+
# name, silently masquerading as a complete one later.
|
|
691
|
+
tmp_path = dest_path + '.uploading'
|
|
692
|
+
written = 0
|
|
693
|
+
try:
|
|
694
|
+
with open(tmp_path, 'wb') as f:
|
|
695
|
+
remaining = length
|
|
696
|
+
while remaining > 0:
|
|
697
|
+
chunk = self.rfile.read(min(1024 * 1024, remaining))
|
|
698
|
+
if not chunk:
|
|
699
|
+
break
|
|
700
|
+
f.write(chunk)
|
|
701
|
+
written += len(chunk)
|
|
702
|
+
remaining -= len(chunk)
|
|
703
|
+
except Exception:
|
|
704
|
+
if os.path.exists(tmp_path):
|
|
705
|
+
os.remove(tmp_path)
|
|
706
|
+
raise
|
|
707
|
+
if written != length:
|
|
708
|
+
os.remove(tmp_path)
|
|
709
|
+
return self._json(
|
|
710
|
+
{'error': f'incomplete upload ({written:,}/{length:,} bytes) -- connection dropped?'},
|
|
711
|
+
status=400)
|
|
712
|
+
os.replace(tmp_path, dest_path)
|
|
713
|
+
|
|
714
|
+
ott_dir = os.path.join(archive_dir, '.ott')
|
|
715
|
+
os.makedirs(os.path.join(ott_dir, 'chunks'), exist_ok=True)
|
|
716
|
+
chunk_size = OttStore(ott_dir).chunk_size
|
|
717
|
+
chunks = chunk_hashes(dest_path, chunk_size)
|
|
718
|
+
digest = merkle_root(chunks) if chunks else hashlib.sha256(b'').hexdigest()
|
|
719
|
+
entry = {
|
|
720
|
+
'sha256': digest, 'name': safe_name, 'orig_path': safe_name, 'last_path': dest_path,
|
|
721
|
+
'size': written, 'added': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
|
|
722
|
+
'type': 'video', 'n_chunks': len(chunks), 'chunk_size': chunk_size,
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
# _lock (not just for _jobs/_hosts/_library, see its own comment
|
|
726
|
+
# up top -- this is the same "one file, must not be torn by two
|
|
727
|
+
# threads at once" concern) since dropping several files at once
|
|
728
|
+
# in the browser fires one of these per file, concurrently, and
|
|
729
|
+
# every one of them touches this *same* manifest.jsonl.
|
|
730
|
+
#
|
|
731
|
+
# tmp-file-then-os.replace, not a bare open(path, 'a') -- a real
|
|
732
|
+
# incident: appending blindly assumes the file already ends with
|
|
733
|
+
# a newline, and it didn't. That merged this entry onto the end
|
|
734
|
+
# of the previous line into one unparseable JSON blob, and
|
|
735
|
+
# load_manifest_entries' `[json.loads(line) for line in f]` has
|
|
736
|
+
# no per-line error handling -- one bad line raises and the
|
|
737
|
+
# *entire* manifest fails to load, which is exactly "files could
|
|
738
|
+
# not be found" for everything in the archive, not just the new
|
|
739
|
+
# upload. Reading the whole file, normalizing a missing trailing
|
|
740
|
+
# newline, and writing the result to a temp file before
|
|
741
|
+
# replacing the original atomically (same pattern _save_library
|
|
742
|
+
# already uses) can't leave a half-written or malformed file on
|
|
743
|
+
# disk no matter when a crash or a second concurrent request
|
|
744
|
+
# lands, and fixes the missing-newline case outright instead of
|
|
745
|
+
# just avoiding making it worse.
|
|
746
|
+
chunks_path = os.path.join(ott_dir, 'chunks', f'{digest}.json')
|
|
747
|
+
manifest_path = os.path.join(ott_dir, 'manifest.jsonl')
|
|
748
|
+
with _lock:
|
|
749
|
+
chunks_tmp = chunks_path + '.tmp'
|
|
750
|
+
with open(chunks_tmp, 'w') as f:
|
|
751
|
+
json.dump(chunks, f)
|
|
752
|
+
os.replace(chunks_tmp, chunks_path)
|
|
753
|
+
|
|
754
|
+
existing = ''
|
|
755
|
+
if os.path.exists(manifest_path):
|
|
756
|
+
with open(manifest_path) as f:
|
|
757
|
+
existing = f.read()
|
|
758
|
+
if existing and not existing.endswith('\n'):
|
|
759
|
+
existing += '\n'
|
|
760
|
+
manifest_tmp = manifest_path + '.tmp'
|
|
761
|
+
with open(manifest_tmp, 'w') as f:
|
|
762
|
+
f.write(existing + json.dumps(entry) + '\n')
|
|
763
|
+
os.replace(manifest_tmp, manifest_path)
|
|
764
|
+
|
|
765
|
+
self._json({'ok': True, 'name': safe_name, 'content_hash': digest,
|
|
766
|
+
'size': written, 'n_chunks': len(chunks), 'archive_dir': archive_dir})
|
|
767
|
+
|
|
605
768
|
def _handle_host(self, body):
|
|
606
769
|
archive_dir = body.get('archive_dir')
|
|
607
770
|
if not archive_dir:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: weed-cli
|
|
3
|
-
Version: 1.8.
|
|
3
|
+
Version: 1.8.4
|
|
4
4
|
Summary: Censorship-resistant video PoC — discovery/hosting/download over signed relay events, a real Kademlia DHT, or a TLS-capable NAT-traversal tunnel
|
|
5
5
|
License: MIT
|
|
6
6
|
Keywords: p2p,video,censorship-resistant,discovery,dht,kademlia,nat-traversal
|
|
@@ -21,6 +21,7 @@ Requires-Dist: pytest>=8.0; extra == "dev"
|
|
|
21
21
|
Requires-Dist: pytest-cov>=5.0; extra == "dev"
|
|
22
22
|
Requires-Dist: ruff>=0.5; extra == "dev"
|
|
23
23
|
Requires-Dist: pytest-playwright>=0.5; extra == "dev"
|
|
24
|
+
Requires-Dist: weed-cli[dht]; extra == "dev"
|
|
24
25
|
Provides-Extra: dht
|
|
25
26
|
Requires-Dist: kademlia>=2.2; extra == "dht"
|
|
26
27
|
Provides-Extra: qr
|
|
@@ -1,127 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
web_ui.py's REST API against a real, isolated WebUIServer instance (see
|
|
3
|
-
conftest.web_server) -- real HTTP requests via the stdlib, same as the
|
|
4
|
-
rest of this codebase's own "no new dependency" convention. Every
|
|
5
|
-
identity/library/hosts file this touches is redirected into tmp_path by
|
|
6
|
-
the isolated_paths fixture web_server depends on; nothing here can ever
|
|
7
|
-
read or write a real ~/.weed_* file.
|
|
8
|
-
"""
|
|
9
|
-
from testutil import http_get_json, http_post_json
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
def test_whoami_and_empty_library(web_server):
|
|
13
|
-
who = http_get_json(f'{web_server}/api/whoami')
|
|
14
|
-
assert len(who['pubkey']) == 64 # 32-byte Ed25519 pubkey, hex-encoded
|
|
15
|
-
|
|
16
|
-
lib = http_get_json(f'{web_server}/api/library')
|
|
17
|
-
assert lib == {'downloads': [], 'likes': [], 'subscriptions': [], 'playlists': [], 'history': []}
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
def test_like_is_idempotent(web_server):
|
|
21
|
-
status, resp = http_post_json(f'{web_server}/api/like', {'content_hash': 'c' * 64})
|
|
22
|
-
assert status == 200
|
|
23
|
-
for _ in range(3):
|
|
24
|
-
http_post_json(f'{web_server}/api/like', {'content_hash': 'c' * 64})
|
|
25
|
-
lib = http_get_json(f'{web_server}/api/library')
|
|
26
|
-
assert lib['likes'] == ['c' * 64]
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
def test_like_requires_content_hash(web_server):
|
|
30
|
-
status, resp = http_post_json(f'{web_server}/api/like', {})
|
|
31
|
-
assert status == 400
|
|
32
|
-
assert 'content_hash' in resp['error']
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
def test_playlist_create_add_reorder_remove_delete(web_server):
|
|
36
|
-
status, resp = http_post_json(f'{web_server}/api/playlists/create', {'name': 'My List'})
|
|
37
|
-
assert status == 200
|
|
38
|
-
playlist_id = resp['playlist']['id']
|
|
39
|
-
|
|
40
|
-
for h in ('a' * 64, 'b' * 64):
|
|
41
|
-
status, resp = http_post_json(f'{web_server}/api/playlists/add', {
|
|
42
|
-
'playlist_id': playlist_id, 'content_hash': h, 'title': h[:8],
|
|
43
|
-
})
|
|
44
|
-
assert status == 200
|
|
45
|
-
|
|
46
|
-
lib = http_get_json(f'{web_server}/api/library')
|
|
47
|
-
pl = next(p for p in lib['playlists'] if p['id'] == playlist_id)
|
|
48
|
-
assert [it['content_hash'] for it in pl['items']] == ['a' * 64, 'b' * 64]
|
|
49
|
-
|
|
50
|
-
status, _ = http_post_json(f'{web_server}/api/playlists/reorder', {
|
|
51
|
-
'playlist_id': playlist_id, 'order': ['b' * 64, 'a' * 64],
|
|
52
|
-
})
|
|
53
|
-
assert status == 200
|
|
54
|
-
lib = http_get_json(f'{web_server}/api/library')
|
|
55
|
-
pl = next(p for p in lib['playlists'] if p['id'] == playlist_id)
|
|
56
|
-
assert [it['content_hash'] for it in pl['items']] == ['b' * 64, 'a' * 64]
|
|
57
|
-
|
|
58
|
-
status, _ = http_post_json(f'{web_server}/api/playlists/remove', {
|
|
59
|
-
'playlist_id': playlist_id, 'content_hash': 'a' * 64,
|
|
60
|
-
})
|
|
61
|
-
assert status == 200
|
|
62
|
-
lib = http_get_json(f'{web_server}/api/library')
|
|
63
|
-
pl = next(p for p in lib['playlists'] if p['id'] == playlist_id)
|
|
64
|
-
assert [it['content_hash'] for it in pl['items']] == ['b' * 64]
|
|
65
|
-
|
|
66
|
-
status, _ = http_post_json(f'{web_server}/api/playlists/delete', {'playlist_id': playlist_id})
|
|
67
|
-
assert status == 200
|
|
68
|
-
lib = http_get_json(f'{web_server}/api/library')
|
|
69
|
-
assert lib['playlists'] == []
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
def test_play_requires_content_hash(web_server):
|
|
73
|
-
status, resp = http_post_json(f'{web_server}/api/play', {})
|
|
74
|
-
assert status == 400
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
def test_play_bumps_count_and_history_for_a_known_download(web_server):
|
|
78
|
-
import web_ui
|
|
79
|
-
web_ui._library['downloads']['c' * 64] = {
|
|
80
|
-
'content_hash': 'c' * 64, 'job_id': 'job1', 'path': '/tmp/x.mp4',
|
|
81
|
-
'title': 'X', 'downloaded_at': 0, 'size': 1, 'bps': 1, 'signer_pubkey': None,
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
status, resp = http_post_json(f'{web_server}/api/play', {'content_hash': 'c' * 64, 'title': 'X'})
|
|
85
|
-
assert status == 200
|
|
86
|
-
assert resp['play_count'] == 1
|
|
87
|
-
assert resp['last_played'] is not None
|
|
88
|
-
|
|
89
|
-
status, resp = http_post_json(f'{web_server}/api/play', {'content_hash': 'c' * 64, 'title': 'X'})
|
|
90
|
-
assert resp['play_count'] == 2
|
|
91
|
-
|
|
92
|
-
lib = http_get_json(f'{web_server}/api/library')
|
|
93
|
-
rec = next(d for d in lib['downloads'] if d['content_hash'] == 'c' * 64)
|
|
94
|
-
assert rec['play_count'] == 2
|
|
95
|
-
assert len(lib['history']) == 2
|
|
96
|
-
assert lib['history'][0]['content_hash'] == 'c' * 64 # newest first
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
def test_play_for_unknown_content_hash_still_logs_history_but_no_count(web_server):
|
|
100
|
-
"""A play_count only exists on a downloads record -- playing something
|
|
101
|
-
that isn't (yet, or ever) in _library['downloads'] shouldn't 500, it
|
|
102
|
-
just can't report a play_count."""
|
|
103
|
-
status, resp = http_post_json(f'{web_server}/api/play', {'content_hash': 'z' * 64, 'title': 'Ghost'})
|
|
104
|
-
assert status == 200
|
|
105
|
-
assert resp['play_count'] is None
|
|
106
|
-
|
|
107
|
-
lib = http_get_json(f'{web_server}/api/library')
|
|
108
|
-
assert len(lib['history']) == 1
|
|
109
|
-
assert lib['history'][0]['title'] == 'Ghost'
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
def test_cross_origin_post_rejected(web_server):
|
|
113
|
-
"""No auth at all by design (see web_ui.py's module docstring) --
|
|
114
|
-
Origin-checking is the only thing standing between this and any other
|
|
115
|
-
open tab silently POSTing here. A request claiming a different Origin
|
|
116
|
-
must be refused."""
|
|
117
|
-
status, resp = http_post_json(f'{web_server}/api/like', {'content_hash': 'c' * 64},
|
|
118
|
-
headers={'Origin': 'http://evil.example'})
|
|
119
|
-
assert status == 403
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
def test_request_with_no_origin_header_is_allowed(web_server):
|
|
123
|
-
"""curl / server-to-server / direct API use never sets Origin -- only
|
|
124
|
-
a real cross-site browser request does, so a missing header is let
|
|
125
|
-
through (see web_ui.py's _check_origin docstring)."""
|
|
126
|
-
status, resp = http_post_json(f'{web_server}/api/like', {'content_hash': 'd' * 64})
|
|
127
|
-
assert status == 200
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|