hashserver 1.0.2__tar.gz → 1.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {hashserver-1.0.2/hashserver.egg-info → hashserver-1.1.0}/PKG-INFO +21 -4
- {hashserver-1.0.2 → hashserver-1.1.0}/README.md +18 -2
- hashserver-1.1.0/compression_utils.py +51 -0
- {hashserver-1.0.2 → hashserver-1.1.0}/hash_file_response.py +133 -19
- {hashserver-1.0.2 → hashserver-1.1.0/hashserver.egg-info}/PKG-INFO +21 -4
- {hashserver-1.0.2 → hashserver-1.1.0}/hashserver.egg-info/SOURCES.txt +3 -0
- {hashserver-1.0.2 → hashserver-1.1.0}/hashserver.egg-info/requires.txt +1 -0
- {hashserver-1.0.2 → hashserver-1.1.0}/hashserver.egg-info/top_level.txt +1 -0
- {hashserver-1.0.2 → hashserver-1.1.0}/hashserver.py +170 -8
- {hashserver-1.0.2 → hashserver-1.1.0}/pyproject.toml +5 -4
- hashserver-1.1.0/tests/test_compression.py +259 -0
- hashserver-1.1.0/tests/test_compression_utils.py +21 -0
- {hashserver-1.0.2 → hashserver-1.1.0}/LICENSE.txt +0 -0
- {hashserver-1.0.2 → hashserver-1.1.0}/hashserver.egg-info/dependency_links.txt +0 -0
- {hashserver-1.0.2 → hashserver-1.1.0}/hashserver.egg-info/entry_points.txt +0 -0
- {hashserver-1.0.2 → hashserver-1.1.0}/setup.cfg +0 -0
- {hashserver-1.0.2 → hashserver-1.1.0}/tests/test_basic.py +0 -0
- {hashserver-1.0.2 → hashserver-1.1.0}/tests/test_basic_uvicorn.py +0 -0
- {hashserver-1.0.2 → hashserver-1.1.0}/tests/test_global_lock.py +0 -0
- {hashserver-1.0.2 → hashserver-1.1.0}/tests/test_has_buffers.py +0 -0
- {hashserver-1.0.2 → hashserver-1.1.0}/tests/test_lock.py +0 -0
- {hashserver-1.0.2 → hashserver-1.1.0}/tests/test_lock_uvicorn.py +0 -0
- {hashserver-1.0.2 → hashserver-1.1.0}/tests/test_port_range.py +0 -0
- {hashserver-1.0.2 → hashserver-1.1.0}/tests/test_promises.py +0 -0
- {hashserver-1.0.2 → hashserver-1.1.0}/tests/test_put_read.py +0 -0
- {hashserver-1.0.2 → hashserver-1.1.0}/tests/test_put_read_big.py +0 -0
- {hashserver-1.0.2 → hashserver-1.1.0}/tests/test_simple_put.py +0 -0
- {hashserver-1.0.2 → hashserver-1.1.0}/tests/test_timeout.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: hashserver
|
|
3
|
-
Version: 1.0
|
|
3
|
+
Version: 1.1.0
|
|
4
4
|
Summary: Simple FastAPI-based hash server
|
|
5
5
|
Author: Sjoerd de Vries
|
|
6
6
|
License-Expression: MIT
|
|
@@ -15,12 +15,13 @@ Classifier: Programming Language :: Python :: 3 :: Only
|
|
|
15
15
|
Classifier: Operating System :: OS Independent
|
|
16
16
|
Classifier: Topic :: Scientific/Engineering
|
|
17
17
|
Classifier: Topic :: Software Development :: Libraries
|
|
18
|
-
Requires-Python: >=3.
|
|
18
|
+
Requires-Python: >=3.10
|
|
19
19
|
Description-Content-Type: text/markdown
|
|
20
20
|
License-File: LICENSE.txt
|
|
21
21
|
Requires-Dist: fastapi
|
|
22
22
|
Requires-Dist: uvicorn[standard]
|
|
23
23
|
Requires-Dist: typing-extensions
|
|
24
|
+
Requires-Dist: zstandard
|
|
24
25
|
Provides-Extra: dev
|
|
25
26
|
Dynamic: license-file
|
|
26
27
|
|
|
@@ -48,7 +49,7 @@ However, **hashserver has no dependency on Seamless** and no knowledge of it. It
|
|
|
48
49
|
|
|
49
50
|
- **Content-addressed**: buffers are stored and retrieved by their cryptographic checksum.
|
|
50
51
|
- **Configurable hash algorithm**: SHA-256 (default) or SHA3-256, selected at startup.
|
|
51
|
-
- **Integrity-verified reads**: every buffer is re-checksummed on GET to detect corruption.
|
|
52
|
+
- **Integrity-verified reads**: every buffer (if not compressed) is re-checksummed on GET to detect corruption.
|
|
52
53
|
- **Prefix directory layout**: by default, buffers are stored under a two-character prefix subdirectory (e.g. `ab/ab3f7c...`) to avoid filesystem performance problems with large flat directories. A flat layout is also supported.
|
|
53
54
|
- **Extra read-only directories**: additional buffer directories can be mounted as fallback read sources.
|
|
54
55
|
- **Promises**: a client can announce that a buffer will be uploaded soon via `PUT /promise/{checksum}`. Other clients reading that checksum will wait for the upload rather than getting a 404.
|
|
@@ -193,6 +194,22 @@ In **flat** layout, the same buffer is stored as `<directory>/ab3f7c...`.
|
|
|
193
194
|
|
|
194
195
|
Extra directories auto-detect their layout by checking for the `.HASHSERVER_PREFIX` sentinel.
|
|
195
196
|
|
|
197
|
+
## Compression
|
|
198
|
+
|
|
199
|
+
Hashserver supports transparent compression for stored buffers. Supported formats: Zstandard (`.zst`) and gzip (`.gz`).
|
|
200
|
+
|
|
201
|
+
**Core invariant:** The URL checksum is always the canonical checksum of the *decompressed* bytes. A compressed upload of the same content produces the same URL as an uncompressed upload.
|
|
202
|
+
|
|
203
|
+
**Uploading compressed buffers:** Send `Content-Encoding: zstd` (or `gzip`) on PUT. The server streams the compressed body to disk while simultaneously decompressing to verify the canonical checksum. The compressed bytes are stored at `{dir}/{cs}.zst` (or `.gz`).
|
|
204
|
+
|
|
205
|
+
**Downloading:** GET always uses the canonical checksum URL. If the stored form is compressed, the response includes `Content-Encoding: zstd` (or `gzip`). The client must check this header and decompress if present. The server tries to honor `Accept-Encoding` when a matching on-disk form exists; otherwise it falls back to the uncompressed file or the server's normal on-disk preference order.
|
|
206
|
+
|
|
207
|
+
**Coexistence:** Compressed and uncompressed forms of the same buffer coexist safely: `{cs}`, `{cs}.zst`, `{cs}.gz` can all be present.
|
|
208
|
+
|
|
209
|
+
**`/has` and `/has-now`:** Any stored form satisfies the existence check.
|
|
210
|
+
|
|
211
|
+
**`/buffer-length`:** Always returns the *uncompressed* byte count. When only a compressed form exists, this is read from a `.BUFFERLENGTH` sidecar file written during the compressed PUT.
|
|
212
|
+
|
|
196
213
|
## Running tests
|
|
197
214
|
|
|
198
215
|
```bash
|
|
@@ -202,4 +219,4 @@ pytest tests/
|
|
|
202
219
|
|
|
203
220
|
## License
|
|
204
221
|
|
|
205
|
-
See [LICENSE.txt](LICENSE.txt).
|
|
222
|
+
MIT. See [LICENSE.txt](LICENSE.txt).
|
|
@@ -22,7 +22,7 @@ However, **hashserver has no dependency on Seamless** and no knowledge of it. It
|
|
|
22
22
|
|
|
23
23
|
- **Content-addressed**: buffers are stored and retrieved by their cryptographic checksum.
|
|
24
24
|
- **Configurable hash algorithm**: SHA-256 (default) or SHA3-256, selected at startup.
|
|
25
|
-
- **Integrity-verified reads**: every buffer is re-checksummed on GET to detect corruption.
|
|
25
|
+
- **Integrity-verified reads**: every buffer (if not compressed) is re-checksummed on GET to detect corruption.
|
|
26
26
|
- **Prefix directory layout**: by default, buffers are stored under a two-character prefix subdirectory (e.g. `ab/ab3f7c...`) to avoid filesystem performance problems with large flat directories. A flat layout is also supported.
|
|
27
27
|
- **Extra read-only directories**: additional buffer directories can be mounted as fallback read sources.
|
|
28
28
|
- **Promises**: a client can announce that a buffer will be uploaded soon via `PUT /promise/{checksum}`. Other clients reading that checksum will wait for the upload rather than getting a 404.
|
|
@@ -167,6 +167,22 @@ In **flat** layout, the same buffer is stored as `<directory>/ab3f7c...`.
|
|
|
167
167
|
|
|
168
168
|
Extra directories auto-detect their layout by checking for the `.HASHSERVER_PREFIX` sentinel.
|
|
169
169
|
|
|
170
|
+
## Compression
|
|
171
|
+
|
|
172
|
+
Hashserver supports transparent compression for stored buffers. Supported formats: Zstandard (`.zst`) and gzip (`.gz`).
|
|
173
|
+
|
|
174
|
+
**Core invariant:** The URL checksum is always the canonical checksum of the *decompressed* bytes. A compressed upload of the same content produces the same URL as an uncompressed upload.
|
|
175
|
+
|
|
176
|
+
**Uploading compressed buffers:** Send `Content-Encoding: zstd` (or `gzip`) on PUT. The server streams the compressed body to disk while simultaneously decompressing to verify the canonical checksum. The compressed bytes are stored at `{dir}/{cs}.zst` (or `.gz`).
|
|
177
|
+
|
|
178
|
+
**Downloading:** GET always uses the canonical checksum URL. If the stored form is compressed, the response includes `Content-Encoding: zstd` (or `gzip`). The client must check this header and decompress if present. The server tries to honor `Accept-Encoding` when a matching on-disk form exists; otherwise it falls back to the uncompressed file or the server's normal on-disk preference order.
|
|
179
|
+
|
|
180
|
+
**Coexistence:** Compressed and uncompressed forms of the same buffer coexist safely: `{cs}`, `{cs}.zst`, `{cs}.gz` can all be present.
|
|
181
|
+
|
|
182
|
+
**`/has` and `/has-now`:** Any stored form satisfies the existence check.
|
|
183
|
+
|
|
184
|
+
**`/buffer-length`:** Always returns the *uncompressed* byte count. When only a compressed form exists, this is read from a `.BUFFERLENGTH` sidecar file written during the compressed PUT.
|
|
185
|
+
|
|
170
186
|
## Running tests
|
|
171
187
|
|
|
172
188
|
```bash
|
|
@@ -176,4 +192,4 @@ pytest tests/
|
|
|
176
192
|
|
|
177
193
|
## License
|
|
178
194
|
|
|
179
|
-
See [LICENSE.txt](LICENSE.txt).
|
|
195
|
+
MIT. See [LICENSE.txt](LICENSE.txt).
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import gzip
|
|
4
|
+
from typing import Optional, Tuple
|
|
5
|
+
|
|
6
|
+
import zstandard
|
|
7
|
+
|
|
8
|
+
COMPRESSION_SUFFIXES = (".zst", ".gz")
|
|
9
|
+
COMPRESSION_PREFERENCE = (".zst", ".gz")
|
|
10
|
+
CONTENT_ENCODING_TO_SUFFIX = {
|
|
11
|
+
"zstd": ".zst",
|
|
12
|
+
"gzip": ".gz",
|
|
13
|
+
}
|
|
14
|
+
SUFFIX_TO_CONTENT_ENCODING = {
|
|
15
|
+
suffix: encoding for encoding, suffix in CONTENT_ENCODING_TO_SUFFIX.items()
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def strip_compression_suffix(name: str) -> Tuple[str, Optional[str]]:
|
|
20
|
+
for suffix in COMPRESSION_SUFFIXES:
|
|
21
|
+
if name.endswith(suffix):
|
|
22
|
+
return name[: -len(suffix)], suffix
|
|
23
|
+
return name, None
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def decompress_bytes(data: bytes, suffix: str) -> bytes:
|
|
27
|
+
if suffix == ".zst":
|
|
28
|
+
return zstandard.ZstdDecompressor().decompress(data)
|
|
29
|
+
if suffix == ".gz":
|
|
30
|
+
return gzip.decompress(data)
|
|
31
|
+
raise ValueError(suffix)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def compress_bytes(data: bytes, suffix: str) -> bytes:
|
|
35
|
+
if suffix == ".zst":
|
|
36
|
+
return zstandard.ZstdCompressor().compress(data)
|
|
37
|
+
if suffix == ".gz":
|
|
38
|
+
return gzip.compress(data)
|
|
39
|
+
raise ValueError(suffix)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def content_encoding_to_suffix(value: Optional[str]) -> Optional[str]:
|
|
43
|
+
if value is None:
|
|
44
|
+
return None
|
|
45
|
+
return CONTENT_ENCODING_TO_SUFFIX.get(value.strip().lower())
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def suffix_to_content_encoding(suffix: Optional[str]) -> Optional[str]:
|
|
49
|
+
if suffix is None:
|
|
50
|
+
return None
|
|
51
|
+
return SUFFIX_TO_CONTENT_ENCODING.get(suffix)
|
|
@@ -2,14 +2,21 @@ import os
|
|
|
2
2
|
import stat
|
|
3
3
|
import time
|
|
4
4
|
import typing
|
|
5
|
+
import zlib
|
|
5
6
|
from hashlib import sha3_256, sha256
|
|
6
7
|
|
|
7
8
|
import anyio
|
|
9
|
+
import zstandard
|
|
8
10
|
|
|
9
11
|
from starlette.background import BackgroundTask
|
|
10
12
|
from starlette.types import Receive, Scope, Send
|
|
11
13
|
from starlette.responses import FileResponse
|
|
12
14
|
|
|
15
|
+
from compression_utils import (
|
|
16
|
+
COMPRESSION_PREFERENCE,
|
|
17
|
+
suffix_to_content_encoding,
|
|
18
|
+
)
|
|
19
|
+
|
|
13
20
|
HASH_ALGORITHMS = {
|
|
14
21
|
"sha3-256": sha3_256,
|
|
15
22
|
"sha-256": sha256,
|
|
@@ -64,6 +71,87 @@ class HashFileResponse(FileResponse):
|
|
|
64
71
|
lock_timeout = 120
|
|
65
72
|
chunk_size = 640 * 1024
|
|
66
73
|
|
|
74
|
+
@staticmethod
|
|
75
|
+
def _parse_accept_encoding(accept_encoding: str | None) -> list[str | None]:
|
|
76
|
+
if not accept_encoding:
|
|
77
|
+
return [None] + list(COMPRESSION_PREFERENCE)
|
|
78
|
+
|
|
79
|
+
parsed: list[tuple[float, int, str | None]] = []
|
|
80
|
+
for idx, item in enumerate(accept_encoding.split(",")):
|
|
81
|
+
item = item.strip()
|
|
82
|
+
if not item:
|
|
83
|
+
continue
|
|
84
|
+
parts = [part.strip() for part in item.split(";")]
|
|
85
|
+
encoding = parts[0].lower()
|
|
86
|
+
q = 1.0
|
|
87
|
+
for param in parts[1:]:
|
|
88
|
+
if param.startswith("q="):
|
|
89
|
+
try:
|
|
90
|
+
q = float(param[2:])
|
|
91
|
+
except ValueError:
|
|
92
|
+
q = 0.0
|
|
93
|
+
if q <= 0:
|
|
94
|
+
continue
|
|
95
|
+
if encoding == "gzip":
|
|
96
|
+
parsed.append((q, idx, ".gz"))
|
|
97
|
+
elif encoding in ("zstd", "x-zstd"):
|
|
98
|
+
parsed.append((q, idx, ".zst"))
|
|
99
|
+
elif encoding == "identity":
|
|
100
|
+
parsed.append((q, idx, None))
|
|
101
|
+
elif encoding == "*":
|
|
102
|
+
parsed.append((q, idx, "*"))
|
|
103
|
+
|
|
104
|
+
parsed.sort(key=lambda item: (-item[0], item[1]))
|
|
105
|
+
ordered: list[str | None] = []
|
|
106
|
+
seen: set[str | None] = set()
|
|
107
|
+
wildcard = False
|
|
108
|
+
for _q, _idx, suffix in parsed:
|
|
109
|
+
if suffix == "*":
|
|
110
|
+
wildcard = True
|
|
111
|
+
continue
|
|
112
|
+
if suffix not in seen:
|
|
113
|
+
seen.add(suffix)
|
|
114
|
+
ordered.append(suffix)
|
|
115
|
+
|
|
116
|
+
if wildcard:
|
|
117
|
+
for suffix in [None, *COMPRESSION_PREFERENCE]:
|
|
118
|
+
if suffix not in seen:
|
|
119
|
+
seen.add(suffix)
|
|
120
|
+
ordered.append(suffix)
|
|
121
|
+
|
|
122
|
+
if None not in seen:
|
|
123
|
+
ordered.append(None)
|
|
124
|
+
for suffix in COMPRESSION_PREFERENCE:
|
|
125
|
+
if suffix not in seen:
|
|
126
|
+
ordered.append(suffix)
|
|
127
|
+
return ordered
|
|
128
|
+
|
|
129
|
+
def _compression_candidates(self, path: str) -> list[tuple[str, str | None]]:
|
|
130
|
+
candidates = [(path, None)] + [
|
|
131
|
+
(path + suffix, suffix) for suffix in COMPRESSION_PREFERENCE
|
|
132
|
+
]
|
|
133
|
+
preferred_suffixes = self._parse_accept_encoding(self._accept_encoding)
|
|
134
|
+
ordered_candidates: list[tuple[str, str | None]] = []
|
|
135
|
+
seen: set[tuple[str, str | None]] = set()
|
|
136
|
+
for suffix in preferred_suffixes:
|
|
137
|
+
candidate = (path if suffix is None else path + suffix, suffix)
|
|
138
|
+
if candidate not in seen:
|
|
139
|
+
seen.add(candidate)
|
|
140
|
+
ordered_candidates.append(candidate)
|
|
141
|
+
for candidate in candidates:
|
|
142
|
+
if candidate not in seen:
|
|
143
|
+
seen.add(candidate)
|
|
144
|
+
ordered_candidates.append(candidate)
|
|
145
|
+
return ordered_candidates
|
|
146
|
+
|
|
147
|
+
@staticmethod
|
|
148
|
+
def _get_decompressor(suffix: str):
|
|
149
|
+
if suffix == ".zst":
|
|
150
|
+
return zstandard.ZstdDecompressor().decompressobj()
|
|
151
|
+
if suffix == ".gz":
|
|
152
|
+
return zlib.decompressobj(wbits=31)
|
|
153
|
+
raise ValueError(suffix)
|
|
154
|
+
|
|
67
155
|
def __init__(
|
|
68
156
|
self,
|
|
69
157
|
checksum: str,
|
|
@@ -73,9 +161,9 @@ class HashFileResponse(FileResponse):
|
|
|
73
161
|
media_type: typing.Optional[str] = None,
|
|
74
162
|
background: typing.Optional[BackgroundTask] = None,
|
|
75
163
|
stat_result: typing.Optional[os.stat_result] = None,
|
|
76
|
-
method: typing.Optional[str] = None,
|
|
77
164
|
content_disposition_type: str = "attachment",
|
|
78
165
|
extra_dirs: typing.Optional[typing.List[str]] = None,
|
|
166
|
+
accept_encoding: typing.Optional[str] = None,
|
|
79
167
|
) -> None:
|
|
80
168
|
filename = parse_checksum(checksum)
|
|
81
169
|
self.prefix = filename[:2]
|
|
@@ -84,6 +172,9 @@ class HashFileResponse(FileResponse):
|
|
|
84
172
|
path = os.path.join(directory, self.prefix, filename)
|
|
85
173
|
else:
|
|
86
174
|
path = os.path.join(directory, filename)
|
|
175
|
+
self._primary_path = path
|
|
176
|
+
self._compression_suffix = None
|
|
177
|
+
self._accept_encoding = accept_encoding
|
|
87
178
|
super().__init__(
|
|
88
179
|
path=path,
|
|
89
180
|
status_code=status_code,
|
|
@@ -92,7 +183,6 @@ class HashFileResponse(FileResponse):
|
|
|
92
183
|
background=background,
|
|
93
184
|
filename=filename,
|
|
94
185
|
stat_result=stat_result,
|
|
95
|
-
method=method,
|
|
96
186
|
content_disposition_type=content_disposition_type,
|
|
97
187
|
)
|
|
98
188
|
self.directory = directory
|
|
@@ -107,16 +197,34 @@ class HashFileResponse(FileResponse):
|
|
|
107
197
|
extra_dirs_layout[extra_dir] = layout
|
|
108
198
|
self.extra_dirs_layout = extra_dirs_layout
|
|
109
199
|
|
|
200
|
+
async def _resolve_existing_path(
|
|
201
|
+
self, base_path: str
|
|
202
|
+
) -> tuple[str | None, str | None]:
|
|
203
|
+
for candidate_path, suffix in self._compression_candidates(base_path):
|
|
204
|
+
if await anyio.Path(candidate_path).exists():
|
|
205
|
+
return candidate_path, suffix
|
|
206
|
+
return None, None
|
|
207
|
+
|
|
110
208
|
async def refresh_stat_headers(self):
|
|
111
|
-
|
|
209
|
+
resolved_path, compression_suffix = await self._resolve_existing_path(
|
|
210
|
+
self._primary_path
|
|
211
|
+
)
|
|
212
|
+
if resolved_path is not None:
|
|
213
|
+
self.path = resolved_path
|
|
214
|
+
self._compression_suffix = compression_suffix
|
|
215
|
+
elif self.extra_dirs:
|
|
112
216
|
for extra_dir in self.extra_dirs:
|
|
113
217
|
layout = self.extra_dirs_layout[extra_dir]
|
|
114
218
|
if layout == "prefix":
|
|
115
|
-
|
|
219
|
+
base_path = os.path.join(extra_dir, self.prefix, self.filename)
|
|
116
220
|
else:
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
221
|
+
base_path = os.path.join(extra_dir, self.filename)
|
|
222
|
+
resolved_path, compression_suffix = await self._resolve_existing_path(
|
|
223
|
+
base_path
|
|
224
|
+
)
|
|
225
|
+
if resolved_path is not None:
|
|
226
|
+
self.path = resolved_path
|
|
227
|
+
self._compression_suffix = compression_suffix
|
|
120
228
|
break
|
|
121
229
|
|
|
122
230
|
try:
|
|
@@ -176,16 +284,22 @@ class HashFileResponse(FileResponse):
|
|
|
176
284
|
stat_result = await self.refresh_stat_headers()
|
|
177
285
|
self.stat_result = stat_result
|
|
178
286
|
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
)
|
|
287
|
+
if self._compression_suffix is not None:
|
|
288
|
+
content_encoding = suffix_to_content_encoding(self._compression_suffix)
|
|
289
|
+
if content_encoding is not None:
|
|
290
|
+
self.headers["content-encoding"] = content_encoding
|
|
291
|
+
|
|
292
|
+
if self._compression_suffix is None:
|
|
293
|
+
checksum = await self.calculate_checksum()
|
|
294
|
+
if checksum != self.filename:
|
|
295
|
+
await self.until_no_lock()
|
|
296
|
+
stat_result = await self.refresh_stat_headers()
|
|
297
|
+
self.stat_result = stat_result
|
|
298
|
+
checksum2 = await self.calculate_checksum()
|
|
299
|
+
if checksum2 != self.filename:
|
|
300
|
+
raise RuntimeError(
|
|
301
|
+
f"File corruption: file at path {self.path} does not have the correct {_current_hash_algorithm} checksum."
|
|
302
|
+
)
|
|
189
303
|
|
|
190
304
|
await super().__call__(scope=scope, receive=receive, send=send)
|
|
191
305
|
|
|
@@ -209,9 +323,9 @@ class PrefixHashFileResponse(HashFileResponse):
|
|
|
209
323
|
media_type: typing.Optional[str] = None,
|
|
210
324
|
background: typing.Optional[BackgroundTask] = None,
|
|
211
325
|
stat_result: typing.Optional[os.stat_result] = None,
|
|
212
|
-
method: typing.Optional[str] = None,
|
|
213
326
|
content_disposition_type: str = "attachment",
|
|
214
327
|
extra_dirs: typing.Optional[typing.List[str]] = None,
|
|
328
|
+
accept_encoding: typing.Optional[str] = None,
|
|
215
329
|
) -> None:
|
|
216
330
|
|
|
217
331
|
super().__init__(
|
|
@@ -222,9 +336,9 @@ class PrefixHashFileResponse(HashFileResponse):
|
|
|
222
336
|
media_type=media_type,
|
|
223
337
|
background=background,
|
|
224
338
|
stat_result=stat_result,
|
|
225
|
-
method=method,
|
|
226
339
|
content_disposition_type=content_disposition_type,
|
|
227
340
|
extra_dirs=extra_dirs,
|
|
341
|
+
accept_encoding=accept_encoding,
|
|
228
342
|
)
|
|
229
343
|
prefix_file = os.path.join(directory, ".HASHSERVER_PREFIX")
|
|
230
344
|
with open(prefix_file, mode="wb") as f:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: hashserver
|
|
3
|
-
Version: 1.0
|
|
3
|
+
Version: 1.1.0
|
|
4
4
|
Summary: Simple FastAPI-based hash server
|
|
5
5
|
Author: Sjoerd de Vries
|
|
6
6
|
License-Expression: MIT
|
|
@@ -15,12 +15,13 @@ Classifier: Programming Language :: Python :: 3 :: Only
|
|
|
15
15
|
Classifier: Operating System :: OS Independent
|
|
16
16
|
Classifier: Topic :: Scientific/Engineering
|
|
17
17
|
Classifier: Topic :: Software Development :: Libraries
|
|
18
|
-
Requires-Python: >=3.
|
|
18
|
+
Requires-Python: >=3.10
|
|
19
19
|
Description-Content-Type: text/markdown
|
|
20
20
|
License-File: LICENSE.txt
|
|
21
21
|
Requires-Dist: fastapi
|
|
22
22
|
Requires-Dist: uvicorn[standard]
|
|
23
23
|
Requires-Dist: typing-extensions
|
|
24
|
+
Requires-Dist: zstandard
|
|
24
25
|
Provides-Extra: dev
|
|
25
26
|
Dynamic: license-file
|
|
26
27
|
|
|
@@ -48,7 +49,7 @@ However, **hashserver has no dependency on Seamless** and no knowledge of it. It
|
|
|
48
49
|
|
|
49
50
|
- **Content-addressed**: buffers are stored and retrieved by their cryptographic checksum.
|
|
50
51
|
- **Configurable hash algorithm**: SHA-256 (default) or SHA3-256, selected at startup.
|
|
51
|
-
- **Integrity-verified reads**: every buffer is re-checksummed on GET to detect corruption.
|
|
52
|
+
- **Integrity-verified reads**: every buffer (if not compressed) is re-checksummed on GET to detect corruption.
|
|
52
53
|
- **Prefix directory layout**: by default, buffers are stored under a two-character prefix subdirectory (e.g. `ab/ab3f7c...`) to avoid filesystem performance problems with large flat directories. A flat layout is also supported.
|
|
53
54
|
- **Extra read-only directories**: additional buffer directories can be mounted as fallback read sources.
|
|
54
55
|
- **Promises**: a client can announce that a buffer will be uploaded soon via `PUT /promise/{checksum}`. Other clients reading that checksum will wait for the upload rather than getting a 404.
|
|
@@ -193,6 +194,22 @@ In **flat** layout, the same buffer is stored as `<directory>/ab3f7c...`.
|
|
|
193
194
|
|
|
194
195
|
Extra directories auto-detect their layout by checking for the `.HASHSERVER_PREFIX` sentinel.
|
|
195
196
|
|
|
197
|
+
## Compression
|
|
198
|
+
|
|
199
|
+
Hashserver supports transparent compression for stored buffers. Supported formats: Zstandard (`.zst`) and gzip (`.gz`).
|
|
200
|
+
|
|
201
|
+
**Core invariant:** The URL checksum is always the canonical checksum of the *decompressed* bytes. A compressed upload of the same content produces the same URL as an uncompressed upload.
|
|
202
|
+
|
|
203
|
+
**Uploading compressed buffers:** Send `Content-Encoding: zstd` (or `gzip`) on PUT. The server streams the compressed body to disk while simultaneously decompressing to verify the canonical checksum. The compressed bytes are stored at `{dir}/{cs}.zst` (or `.gz`).
|
|
204
|
+
|
|
205
|
+
**Downloading:** GET always uses the canonical checksum URL. If the stored form is compressed, the response includes `Content-Encoding: zstd` (or `gzip`). The client must check this header and decompress if present. The server tries to honor `Accept-Encoding` when a matching on-disk form exists; otherwise it falls back to the uncompressed file or the server's normal on-disk preference order.
|
|
206
|
+
|
|
207
|
+
**Coexistence:** Compressed and uncompressed forms of the same buffer coexist safely: `{cs}`, `{cs}.zst`, `{cs}.gz` can all be present.
|
|
208
|
+
|
|
209
|
+
**`/has` and `/has-now`:** Any stored form satisfies the existence check.
|
|
210
|
+
|
|
211
|
+
**`/buffer-length`:** Always returns the *uncompressed* byte count. When only a compressed form exists, this is read from a `.BUFFERLENGTH` sidecar file written during the compressed PUT.
|
|
212
|
+
|
|
196
213
|
## Running tests
|
|
197
214
|
|
|
198
215
|
```bash
|
|
@@ -202,4 +219,4 @@ pytest tests/
|
|
|
202
219
|
|
|
203
220
|
## License
|
|
204
221
|
|
|
205
|
-
See [LICENSE.txt](LICENSE.txt).
|
|
222
|
+
MIT. See [LICENSE.txt](LICENSE.txt).
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
LICENSE.txt
|
|
2
2
|
README.md
|
|
3
|
+
compression_utils.py
|
|
3
4
|
hash_file_response.py
|
|
4
5
|
hashserver.py
|
|
5
6
|
pyproject.toml
|
|
@@ -11,6 +12,8 @@ hashserver.egg-info/requires.txt
|
|
|
11
12
|
hashserver.egg-info/top_level.txt
|
|
12
13
|
tests/test_basic.py
|
|
13
14
|
tests/test_basic_uvicorn.py
|
|
15
|
+
tests/test_compression.py
|
|
16
|
+
tests/test_compression_utils.py
|
|
14
17
|
tests/test_global_lock.py
|
|
15
18
|
tests/test_has_buffers.py
|
|
16
19
|
tests/test_lock.py
|
|
@@ -11,6 +11,7 @@ import aiofiles.os
|
|
|
11
11
|
import aiofiles.tempfile
|
|
12
12
|
import contextlib
|
|
13
13
|
import copy
|
|
14
|
+
import zlib
|
|
14
15
|
from dataclasses import dataclass
|
|
15
16
|
from typing import Iterable, List, Optional, Set, Union
|
|
16
17
|
from fastapi import FastAPI, Path, Body, Request
|
|
@@ -37,6 +38,12 @@ from pydantic.functional_validators import BeforeValidator
|
|
|
37
38
|
import anyio
|
|
38
39
|
import pathlib
|
|
39
40
|
import time
|
|
41
|
+
import zstandard
|
|
42
|
+
|
|
43
|
+
from compression_utils import (
|
|
44
|
+
COMPRESSION_PREFERENCE,
|
|
45
|
+
content_encoding_to_suffix,
|
|
46
|
+
)
|
|
40
47
|
|
|
41
48
|
Checksum = Annotated[str, BeforeValidator(parse_checksum)]
|
|
42
49
|
|
|
@@ -67,6 +74,54 @@ def calculate_checksum_stream():
|
|
|
67
74
|
return checksum_constructor()
|
|
68
75
|
|
|
69
76
|
|
|
77
|
+
def _compression_candidates(path: str) -> list[tuple[str, str | None]]:
|
|
78
|
+
return [(path, None)] + [
|
|
79
|
+
(path + suffix, suffix) for suffix in COMPRESSION_PREFERENCE
|
|
80
|
+
]
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _get_streaming_decompressor(suffix: str):
|
|
84
|
+
if suffix == ".zst":
|
|
85
|
+
return zstandard.ZstdDecompressor().decompressobj()
|
|
86
|
+
if suffix == ".gz":
|
|
87
|
+
return zlib.decompressobj(wbits=31)
|
|
88
|
+
raise ValueError(suffix)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
async def _read_buffer_length_sidecar(path: str) -> int | None:
|
|
92
|
+
try:
|
|
93
|
+
async with await anyio.open_file(path + ".BUFFERLENGTH", mode="r") as sidecar:
|
|
94
|
+
data = await sidecar.read()
|
|
95
|
+
except FileNotFoundError:
|
|
96
|
+
return None
|
|
97
|
+
return int(data.strip())
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
async def _write_buffer_length_sidecar(path: str, buffer_length: int) -> None:
|
|
101
|
+
async with await anyio.open_file(path + ".BUFFERLENGTH", mode="w") as sidecar:
|
|
102
|
+
await sidecar.write(str(buffer_length))
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
async def _remove_buffer_length_sidecar(path: str) -> None:
|
|
106
|
+
try:
|
|
107
|
+
await aiofiles.os.unlink(path + ".BUFFERLENGTH")
|
|
108
|
+
except FileNotFoundError:
|
|
109
|
+
pass
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
async def _stat_canonical_buffer_length(path: str, suffix: str) -> int:
|
|
113
|
+
decompressor = _get_streaming_decompressor(suffix)
|
|
114
|
+
total_length = 0
|
|
115
|
+
async with await anyio.open_file(path, mode="rb") as file:
|
|
116
|
+
while True:
|
|
117
|
+
chunk = await file.read(640 * 1024)
|
|
118
|
+
if not chunk:
|
|
119
|
+
break
|
|
120
|
+
total_length += len(decompressor.decompress(chunk))
|
|
121
|
+
total_length += len(decompressor.flush())
|
|
122
|
+
return total_length
|
|
123
|
+
|
|
124
|
+
|
|
70
125
|
def wait_for_status_file(path: str, timeout: float = STATUS_FILE_WAIT_TIMEOUT):
|
|
71
126
|
deadline = time.monotonic() + timeout
|
|
72
127
|
while True:
|
|
@@ -476,6 +531,65 @@ async def buffer_length(checksums: Annotated[List[Checksum], Body()]) -> JSONRes
|
|
|
476
531
|
continue
|
|
477
532
|
curr_results[nr] = stat.st_size
|
|
478
533
|
|
|
534
|
+
async def read_sidecars(paths):
|
|
535
|
+
futures = []
|
|
536
|
+
for _, path in paths:
|
|
537
|
+
futures.append(_read_buffer_length_sidecar(path))
|
|
538
|
+
result0 = await asyncio.gather(*futures, return_exceptions=True)
|
|
539
|
+
for (nr, _path), value in zip(paths, result0):
|
|
540
|
+
if isinstance(value, Exception) or value is None:
|
|
541
|
+
continue
|
|
542
|
+
curr_results[nr] = value
|
|
543
|
+
|
|
544
|
+
async def stat_compressed(paths):
|
|
545
|
+
futures = []
|
|
546
|
+
metadata = []
|
|
547
|
+
for nr, path in paths:
|
|
548
|
+
for candidate_path, suffix in _compression_candidates(path)[1:]:
|
|
549
|
+
metadata.append((nr, candidate_path, suffix))
|
|
550
|
+
futures.append(anyio.Path(candidate_path).exists())
|
|
551
|
+
result0 = await asyncio.gather(*futures, return_exceptions=True)
|
|
552
|
+
fallback = []
|
|
553
|
+
for (nr, candidate_path, suffix), exists in zip(metadata, result0):
|
|
554
|
+
if curr_results[nr]:
|
|
555
|
+
continue
|
|
556
|
+
if isinstance(exists, Exception) or not exists:
|
|
557
|
+
continue
|
|
558
|
+
fallback.append((nr, candidate_path, suffix))
|
|
559
|
+
if not fallback:
|
|
560
|
+
return
|
|
561
|
+
sidecar_reads = []
|
|
562
|
+
for _nr, candidate_path, suffix in fallback:
|
|
563
|
+
sidecar_reads.append(
|
|
564
|
+
_read_buffer_length_sidecar(candidate_path[: -len(suffix)])
|
|
565
|
+
)
|
|
566
|
+
sidecar_values = await asyncio.gather(*sidecar_reads, return_exceptions=True)
|
|
567
|
+
expensive = []
|
|
568
|
+
for (nr, candidate_path, suffix), sidecar_value in zip(
|
|
569
|
+
fallback, sidecar_values
|
|
570
|
+
):
|
|
571
|
+
if curr_results[nr]:
|
|
572
|
+
continue
|
|
573
|
+
if isinstance(sidecar_value, Exception):
|
|
574
|
+
expensive.append((nr, candidate_path, suffix))
|
|
575
|
+
continue
|
|
576
|
+
if sidecar_value is None:
|
|
577
|
+
expensive.append((nr, candidate_path, suffix))
|
|
578
|
+
continue
|
|
579
|
+
curr_results[nr] = sidecar_value
|
|
580
|
+
if expensive:
|
|
581
|
+
lengths = await asyncio.gather(
|
|
582
|
+
*[
|
|
583
|
+
_stat_canonical_buffer_length(candidate_path, suffix)
|
|
584
|
+
for _, candidate_path, suffix in expensive
|
|
585
|
+
],
|
|
586
|
+
return_exceptions=True,
|
|
587
|
+
)
|
|
588
|
+
for (nr, _candidate_path, _suffix), length in zip(expensive, lengths):
|
|
589
|
+
if isinstance(length, Exception):
|
|
590
|
+
continue
|
|
591
|
+
curr_results[nr] = length
|
|
592
|
+
|
|
479
593
|
paths = []
|
|
480
594
|
for nr, checksum in enumerate(checksums2):
|
|
481
595
|
assert isinstance(checksum, str)
|
|
@@ -487,8 +601,15 @@ async def buffer_length(checksums: Annotated[List[Checksum], Body()]) -> JSONRes
|
|
|
487
601
|
paths.append((nr, path))
|
|
488
602
|
|
|
489
603
|
await stat_all(paths)
|
|
604
|
+
unresolved_paths = [item for item in paths if not curr_results[item[0]]]
|
|
605
|
+
if unresolved_paths:
|
|
606
|
+
await read_sidecars(unresolved_paths)
|
|
607
|
+
unresolved_paths = [item for item in paths if not curr_results[item[0]]]
|
|
608
|
+
if unresolved_paths:
|
|
609
|
+
await stat_compressed(unresolved_paths)
|
|
490
610
|
|
|
491
611
|
for extra_dir in extra_dirs:
|
|
612
|
+
paths = []
|
|
492
613
|
for nr, checksum in enumerate(checksums2):
|
|
493
614
|
if curr_results[nr]:
|
|
494
615
|
continue
|
|
@@ -497,6 +618,12 @@ async def buffer_length(checksums: Annotated[List[Checksum], Body()]) -> JSONRes
|
|
|
497
618
|
if not len(paths):
|
|
498
619
|
break
|
|
499
620
|
await stat_all(paths)
|
|
621
|
+
unresolved_paths = [item for item in paths if not curr_results[item[0]]]
|
|
622
|
+
if unresolved_paths:
|
|
623
|
+
await read_sidecars(unresolved_paths)
|
|
624
|
+
unresolved_paths = [item for item in paths if not curr_results[item[0]]]
|
|
625
|
+
if unresolved_paths:
|
|
626
|
+
await stat_compressed(unresolved_paths)
|
|
500
627
|
|
|
501
628
|
promised = await _promise_registry.promised_indices(checksums2)
|
|
502
629
|
for idx in promised:
|
|
@@ -516,11 +643,13 @@ async def _has(checksums: List[Checksum], include_promises: bool) -> List[bool]:
|
|
|
516
643
|
|
|
517
644
|
async def exists_all(paths):
|
|
518
645
|
futures = []
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
646
|
+
metadata = []
|
|
647
|
+
for nr, path in paths:
|
|
648
|
+
for candidate_path, _suffix in _compression_candidates(path):
|
|
649
|
+
metadata.append((nr, candidate_path))
|
|
650
|
+
futures.append(anyio.Path(candidate_path).exists())
|
|
522
651
|
result0 = await asyncio.gather(*futures, return_exceptions=True)
|
|
523
|
-
for (nr,
|
|
652
|
+
for (nr, _path), exists in zip(metadata, result0):
|
|
524
653
|
if isinstance(exists, Exception):
|
|
525
654
|
continue
|
|
526
655
|
if exists:
|
|
@@ -608,13 +737,18 @@ async def healthcheck() -> Response:
|
|
|
608
737
|
|
|
609
738
|
|
|
610
739
|
@app.get("/{checksum}")
|
|
611
|
-
async def get_file(
|
|
740
|
+
async def get_file(
|
|
741
|
+
checksum: Annotated[Checksum, Path()], rq: Request
|
|
742
|
+
) -> HashFileResponse:
|
|
612
743
|
checksum2 = parse_checksum(checksum)
|
|
613
744
|
LOGGER.info("GET %s", checksum2)
|
|
614
745
|
await _wait_for_current_put_requests((checksum2,))
|
|
615
746
|
ResponseClass = _response_classes_get_file[layout]
|
|
616
747
|
response = ResponseClass(
|
|
617
|
-
directory=directory,
|
|
748
|
+
directory=directory,
|
|
749
|
+
checksum=checksum2,
|
|
750
|
+
extra_dirs=extra_dirs,
|
|
751
|
+
accept_encoding=rq.headers.get("accept-encoding"),
|
|
618
752
|
)
|
|
619
753
|
return response
|
|
620
754
|
|
|
@@ -728,7 +862,9 @@ async def put_file(checksum: Annotated[Checksum, Path()], rq: Request) -> Respon
|
|
|
728
862
|
target_dir = os.path.join(directory, prefix)
|
|
729
863
|
else:
|
|
730
864
|
target_dir = directory
|
|
731
|
-
|
|
865
|
+
base_path = os.path.join(target_dir, checksum_str)
|
|
866
|
+
compression_suffix = content_encoding_to_suffix(rq.headers.get("content-encoding"))
|
|
867
|
+
path = base_path if compression_suffix is None else base_path + compression_suffix
|
|
732
868
|
|
|
733
869
|
if layout == "prefix":
|
|
734
870
|
target_directory = anyio.Path(target_dir)
|
|
@@ -746,6 +882,8 @@ async def put_file(checksum: Annotated[Checksum, Path()], rq: Request) -> Respon
|
|
|
746
882
|
cs_stream = calculate_checksum_stream()
|
|
747
883
|
temp_path = None
|
|
748
884
|
buffer_checksum = None
|
|
885
|
+
decompressed_length = 0
|
|
886
|
+
decompressor = None
|
|
749
887
|
try:
|
|
750
888
|
async with _current_put_condition:
|
|
751
889
|
if checksum_str in _current_put_requests:
|
|
@@ -753,14 +891,28 @@ async def put_file(checksum: Annotated[Checksum, Path()], rq: Request) -> Respon
|
|
|
753
891
|
return Response(status_code=202)
|
|
754
892
|
_current_put_requests.add(checksum_str)
|
|
755
893
|
added_to_put_requests = True
|
|
894
|
+
if compression_suffix is not None:
|
|
895
|
+
decompressor = _get_streaming_decompressor(compression_suffix)
|
|
756
896
|
async with aiofiles.tempfile.NamedTemporaryFile(
|
|
757
897
|
dir=target_dir,
|
|
758
898
|
prefix=checksum_str + "-",
|
|
759
899
|
delete=False,
|
|
760
900
|
) as file:
|
|
761
901
|
async for chunk in rq.stream():
|
|
762
|
-
|
|
902
|
+
if not chunk:
|
|
903
|
+
continue
|
|
763
904
|
await file.write(chunk)
|
|
905
|
+
if decompressor is None:
|
|
906
|
+
cs_stream.update(chunk)
|
|
907
|
+
decompressed_length += len(chunk)
|
|
908
|
+
else:
|
|
909
|
+
decompressed = decompressor.decompress(chunk)
|
|
910
|
+
cs_stream.update(decompressed)
|
|
911
|
+
decompressed_length += len(decompressed)
|
|
912
|
+
if decompressor is not None:
|
|
913
|
+
flushed = decompressor.flush()
|
|
914
|
+
cs_stream.update(flushed)
|
|
915
|
+
decompressed_length += len(flushed)
|
|
764
916
|
buffer_checksum = cs_stream.hexdigest()
|
|
765
917
|
temp_path = file.name
|
|
766
918
|
if buffer_checksum != checksum_str:
|
|
@@ -772,6 +924,10 @@ async def put_file(checksum: Annotated[Checksum, Path()], rq: Request) -> Respon
|
|
|
772
924
|
except Exception:
|
|
773
925
|
if not await aiofiles.ospath.exists(path):
|
|
774
926
|
raise
|
|
927
|
+
if compression_suffix is None:
|
|
928
|
+
await _remove_buffer_length_sidecar(base_path)
|
|
929
|
+
else:
|
|
930
|
+
await _write_buffer_length_sidecar(base_path, decompressed_length)
|
|
775
931
|
ok = True
|
|
776
932
|
try:
|
|
777
933
|
await aiofiles_chmod(path, 0o444)
|
|
@@ -781,6 +937,9 @@ async def put_file(checksum: Annotated[Checksum, Path()], rq: Request) -> Respon
|
|
|
781
937
|
except ClientDisconnect:
|
|
782
938
|
LOGGER.warning("PUT %s client disconnected", checksum_str)
|
|
783
939
|
return Response(status_code=400)
|
|
940
|
+
except (zlib.error, zstandard.ZstdError, ValueError):
|
|
941
|
+
LOGGER.warning("PUT %s invalid compressed payload", checksum_str)
|
|
942
|
+
return Response(status_code=400, content="Invalid compressed payload")
|
|
784
943
|
|
|
785
944
|
finally:
|
|
786
945
|
if added_to_put_requests:
|
|
@@ -865,6 +1024,9 @@ if as_commandline_tool:
|
|
|
865
1024
|
print("OK")
|
|
866
1025
|
try:
|
|
867
1026
|
server.run()
|
|
1027
|
+
except KeyboardInterrupt:
|
|
1028
|
+
# Normal interactive shutdown path (CTRL+C / forwarded SIGINT).
|
|
1029
|
+
pass
|
|
868
1030
|
except BaseException:
|
|
869
1031
|
if status_tracker and not status_tracker.running_written:
|
|
870
1032
|
status_tracker.write_failed()
|
|
@@ -4,16 +4,17 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "hashserver"
|
|
7
|
-
version = "1.0
|
|
7
|
+
version = "1.1.0"
|
|
8
8
|
description = "Simple FastAPI-based hash server"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
11
11
|
authors = [{name = "Sjoerd de Vries"}]
|
|
12
|
-
requires-python = ">=3.
|
|
12
|
+
requires-python = ">=3.10"
|
|
13
13
|
dependencies = [
|
|
14
14
|
"fastapi",
|
|
15
15
|
"uvicorn[standard]",
|
|
16
|
-
"typing-extensions"
|
|
16
|
+
"typing-extensions",
|
|
17
|
+
"zstandard",
|
|
17
18
|
]
|
|
18
19
|
keywords = ["seamless", "content-addressed", "file server", "HTTP"]
|
|
19
20
|
classifiers = [
|
|
@@ -35,7 +36,7 @@ Repository = "https://github.com/sjdv1982/hashserver"
|
|
|
35
36
|
dev = []
|
|
36
37
|
|
|
37
38
|
[tool.setuptools]
|
|
38
|
-
py-modules = ["hashserver", "hash_file_response"]
|
|
39
|
+
py-modules = ["hashserver", "hash_file_response", "compression_utils"]
|
|
39
40
|
|
|
40
41
|
[project.scripts]
|
|
41
42
|
hashserver = "hashserver:main"
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
import gzip
|
|
2
|
+
from hashlib import sha256
|
|
3
|
+
import sys
|
|
4
|
+
|
|
5
|
+
import requests
|
|
6
|
+
import zstandard
|
|
7
|
+
|
|
8
|
+
from .utils import start_server, wait_for_server
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
BUFFER = (b"compressed payload\n" * 128) + b"tail"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def calculate_checksum(buffer: bytes) -> str:
|
|
15
|
+
return sha256(buffer).digest().hex()
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def compress_zstd(buffer: bytes) -> bytes:
|
|
19
|
+
return zstandard.ZstdCompressor().compress(buffer)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def put_buffer(port: int, checksum: str, buffer: bytes, *, content_encoding=None):
|
|
23
|
+
headers = {}
|
|
24
|
+
if content_encoding is not None:
|
|
25
|
+
headers["Content-Encoding"] = content_encoding
|
|
26
|
+
response = requests.put(
|
|
27
|
+
f"http://127.0.0.1:{port}/{checksum}",
|
|
28
|
+
data=buffer,
|
|
29
|
+
headers=headers,
|
|
30
|
+
timeout=10,
|
|
31
|
+
)
|
|
32
|
+
return response
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def get_buffer_raw(port: int, checksum: str, *, headers=None):
|
|
36
|
+
response = requests.get(
|
|
37
|
+
f"http://127.0.0.1:{port}/{checksum}",
|
|
38
|
+
stream=True,
|
|
39
|
+
headers=headers,
|
|
40
|
+
timeout=10,
|
|
41
|
+
)
|
|
42
|
+
response.raw.decode_content = False
|
|
43
|
+
body = response.raw.read()
|
|
44
|
+
return response, body
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def has_buffers(port: int, checksums):
|
|
48
|
+
response = requests.get(f"http://127.0.0.1:{port}/has", json=checksums, timeout=10)
|
|
49
|
+
return response
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def buffer_lengths(port: int, checksums):
|
|
53
|
+
response = requests.get(
|
|
54
|
+
f"http://127.0.0.1:{port}/buffer-length", json=checksums, timeout=10
|
|
55
|
+
)
|
|
56
|
+
return response
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _server_command(write_dir, port):
|
|
60
|
+
env = {
|
|
61
|
+
"HASHSERVER_DIRECTORY": str(write_dir),
|
|
62
|
+
"HASHSERVER_PORT": str(port),
|
|
63
|
+
"HASHSERVER_WRITABLE": "1",
|
|
64
|
+
"HASHSERVER_LAYOUT": "flat",
|
|
65
|
+
}
|
|
66
|
+
command = [
|
|
67
|
+
sys.executable,
|
|
68
|
+
"-m",
|
|
69
|
+
"uvicorn",
|
|
70
|
+
"hashserver:app",
|
|
71
|
+
"--host",
|
|
72
|
+
"127.0.0.1",
|
|
73
|
+
"--port",
|
|
74
|
+
str(port),
|
|
75
|
+
"--log-level",
|
|
76
|
+
"warning",
|
|
77
|
+
]
|
|
78
|
+
return command, env
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _readonly_server_command(read_dir, port):
|
|
82
|
+
env = {
|
|
83
|
+
"HASHSERVER_DIRECTORY": str(read_dir),
|
|
84
|
+
"HASHSERVER_PORT": str(port),
|
|
85
|
+
"HASHSERVER_LAYOUT": "flat",
|
|
86
|
+
}
|
|
87
|
+
command = [
|
|
88
|
+
sys.executable,
|
|
89
|
+
"-m",
|
|
90
|
+
"uvicorn",
|
|
91
|
+
"hashserver:app",
|
|
92
|
+
"--host",
|
|
93
|
+
"127.0.0.1",
|
|
94
|
+
"--port",
|
|
95
|
+
str(port),
|
|
96
|
+
"--log-level",
|
|
97
|
+
"warning",
|
|
98
|
+
]
|
|
99
|
+
return command, env
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def test_put_zstd_stores_compressed_and_sidecar(tmp_path, available_port):
|
|
103
|
+
write_dir = tmp_path / "writedir"
|
|
104
|
+
write_dir.mkdir()
|
|
105
|
+
checksum = calculate_checksum(BUFFER)
|
|
106
|
+
compressed = compress_zstd(BUFFER)
|
|
107
|
+
|
|
108
|
+
command, env = _server_command(write_dir, available_port)
|
|
109
|
+
|
|
110
|
+
with start_server(command, env=env) as _server:
|
|
111
|
+
wait_for_server(available_port)
|
|
112
|
+
|
|
113
|
+
response = put_buffer(
|
|
114
|
+
available_port,
|
|
115
|
+
checksum,
|
|
116
|
+
compressed,
|
|
117
|
+
content_encoding="zstd",
|
|
118
|
+
)
|
|
119
|
+
assert response.status_code == 200, response.content
|
|
120
|
+
|
|
121
|
+
stored = write_dir / f"{checksum}.zst"
|
|
122
|
+
assert stored.read_bytes() == compressed
|
|
123
|
+
assert (write_dir / f"{checksum}.BUFFERLENGTH").read_text() == str(len(BUFFER))
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def test_put_compressed_checksum_mismatch_returns_400(tmp_path, available_port):
|
|
127
|
+
write_dir = tmp_path / "writedir"
|
|
128
|
+
write_dir.mkdir()
|
|
129
|
+
checksum = calculate_checksum(BUFFER)
|
|
130
|
+
compressed = compress_zstd(BUFFER + b"x")
|
|
131
|
+
|
|
132
|
+
command, env = _server_command(write_dir, available_port)
|
|
133
|
+
|
|
134
|
+
with start_server(command, env=env) as _server:
|
|
135
|
+
wait_for_server(available_port)
|
|
136
|
+
|
|
137
|
+
response = put_buffer(
|
|
138
|
+
available_port,
|
|
139
|
+
checksum,
|
|
140
|
+
compressed,
|
|
141
|
+
content_encoding="zstd",
|
|
142
|
+
)
|
|
143
|
+
assert response.status_code == 400, response.content
|
|
144
|
+
assert not (write_dir / f"{checksum}.zst").exists()
|
|
145
|
+
assert not (write_dir / f"{checksum}.BUFFERLENGTH").exists()
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def test_put_compressed_and_uncompressed_can_coexist(tmp_path, available_port):
|
|
149
|
+
write_dir = tmp_path / "writedir"
|
|
150
|
+
write_dir.mkdir()
|
|
151
|
+
checksum = calculate_checksum(BUFFER)
|
|
152
|
+
compressed = compress_zstd(BUFFER)
|
|
153
|
+
|
|
154
|
+
command, env = _server_command(write_dir, available_port)
|
|
155
|
+
|
|
156
|
+
with start_server(command, env=env) as _server:
|
|
157
|
+
wait_for_server(available_port)
|
|
158
|
+
|
|
159
|
+
response = put_buffer(
|
|
160
|
+
available_port,
|
|
161
|
+
checksum,
|
|
162
|
+
compressed,
|
|
163
|
+
content_encoding="zstd",
|
|
164
|
+
)
|
|
165
|
+
assert response.status_code == 200, response.content
|
|
166
|
+
|
|
167
|
+
response = put_buffer(available_port, checksum, BUFFER)
|
|
168
|
+
assert response.status_code == 200, response.content
|
|
169
|
+
|
|
170
|
+
assert (write_dir / checksum).read_bytes() == BUFFER
|
|
171
|
+
assert (write_dir / f"{checksum}.zst").read_bytes() == compressed
|
|
172
|
+
assert not (write_dir / f"{checksum}.BUFFERLENGTH").exists()
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def test_get_prefers_compressed_when_only_compressed_exists(tmp_path, available_port):
|
|
176
|
+
read_dir = tmp_path / "bufferdir"
|
|
177
|
+
read_dir.mkdir()
|
|
178
|
+
checksum = calculate_checksum(BUFFER)
|
|
179
|
+
compressed = compress_zstd(BUFFER)
|
|
180
|
+
(read_dir / f"{checksum}.zst").write_bytes(compressed)
|
|
181
|
+
(read_dir / f"{checksum}.BUFFERLENGTH").write_text(str(len(BUFFER)))
|
|
182
|
+
|
|
183
|
+
command, env = _readonly_server_command(read_dir, available_port)
|
|
184
|
+
|
|
185
|
+
with start_server(command, env=env) as _server:
|
|
186
|
+
wait_for_server(available_port)
|
|
187
|
+
|
|
188
|
+
response, body = get_buffer_raw(available_port, checksum)
|
|
189
|
+
assert response.status_code == 200, body
|
|
190
|
+
assert response.headers["Content-Encoding"] == "zstd"
|
|
191
|
+
assert body == compressed
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def test_get_prefers_uncompressed_when_identity_is_requested(
|
|
195
|
+
tmp_path, available_port
|
|
196
|
+
):
|
|
197
|
+
read_dir = tmp_path / "bufferdir"
|
|
198
|
+
read_dir.mkdir()
|
|
199
|
+
checksum = calculate_checksum(BUFFER)
|
|
200
|
+
compressed = gzip.compress(BUFFER)
|
|
201
|
+
(read_dir / checksum).write_bytes(BUFFER)
|
|
202
|
+
(read_dir / f"{checksum}.gz").write_bytes(compressed)
|
|
203
|
+
|
|
204
|
+
command, env = _readonly_server_command(read_dir, available_port)
|
|
205
|
+
|
|
206
|
+
with start_server(command, env=env) as _server:
|
|
207
|
+
wait_for_server(available_port)
|
|
208
|
+
|
|
209
|
+
response, body = get_buffer_raw(
|
|
210
|
+
available_port, checksum, headers={"Accept-Encoding": "identity"}
|
|
211
|
+
)
|
|
212
|
+
assert response.status_code == 200, body
|
|
213
|
+
assert "Content-Encoding" not in response.headers
|
|
214
|
+
assert body == BUFFER
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def test_get_prefers_requested_encoding_when_available(
|
|
218
|
+
tmp_path, available_port
|
|
219
|
+
):
|
|
220
|
+
read_dir = tmp_path / "bufferdir"
|
|
221
|
+
read_dir.mkdir()
|
|
222
|
+
checksum = calculate_checksum(BUFFER)
|
|
223
|
+
compressed = gzip.compress(BUFFER)
|
|
224
|
+
(read_dir / checksum).write_bytes(BUFFER)
|
|
225
|
+
(read_dir / f"{checksum}.gz").write_bytes(compressed)
|
|
226
|
+
|
|
227
|
+
command, env = _readonly_server_command(read_dir, available_port)
|
|
228
|
+
|
|
229
|
+
with start_server(command, env=env) as _server:
|
|
230
|
+
wait_for_server(available_port)
|
|
231
|
+
|
|
232
|
+
response, body = get_buffer_raw(
|
|
233
|
+
available_port, checksum, headers={"Accept-Encoding": "gzip"}
|
|
234
|
+
)
|
|
235
|
+
assert response.status_code == 200, body
|
|
236
|
+
assert response.headers["Content-Encoding"] == "gzip"
|
|
237
|
+
assert body == compressed
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def test_has_and_buffer_length_recognize_compressed_form(tmp_path, available_port):
|
|
241
|
+
read_dir = tmp_path / "bufferdir"
|
|
242
|
+
read_dir.mkdir()
|
|
243
|
+
checksum = calculate_checksum(BUFFER)
|
|
244
|
+
compressed = compress_zstd(BUFFER)
|
|
245
|
+
(read_dir / f"{checksum}.zst").write_bytes(compressed)
|
|
246
|
+
(read_dir / f"{checksum}.BUFFERLENGTH").write_text(str(len(BUFFER)))
|
|
247
|
+
|
|
248
|
+
command, env = _readonly_server_command(read_dir, available_port)
|
|
249
|
+
|
|
250
|
+
with start_server(command, env=env) as _server:
|
|
251
|
+
wait_for_server(available_port)
|
|
252
|
+
|
|
253
|
+
has_response = has_buffers(available_port, [checksum, "0" * 64])
|
|
254
|
+
assert has_response.status_code == 200, has_response.text
|
|
255
|
+
assert has_response.json() == [True, False]
|
|
256
|
+
|
|
257
|
+
length_response = buffer_lengths(available_port, [checksum])
|
|
258
|
+
assert length_response.status_code == 200, length_response.text
|
|
259
|
+
assert length_response.json() == [len(BUFFER)]
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import gzip
|
|
2
|
+
|
|
3
|
+
import zstandard
|
|
4
|
+
|
|
5
|
+
from compression_utils import compress_bytes, decompress_bytes
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def test_compression_utils_round_trip():
|
|
9
|
+
payloads = [b"", b"hello", b"0123456789" * 10000]
|
|
10
|
+
for suffix in (".zst", ".gz"):
|
|
11
|
+
for payload in payloads:
|
|
12
|
+
assert decompress_bytes(compress_bytes(payload, suffix), suffix) == payload
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def test_compression_utils_match_standard_codecs():
|
|
16
|
+
payload = b"payload" * 100
|
|
17
|
+
assert (
|
|
18
|
+
decompress_bytes(zstandard.ZstdCompressor().compress(payload), ".zst")
|
|
19
|
+
== payload
|
|
20
|
+
)
|
|
21
|
+
assert decompress_bytes(gzip.compress(payload), ".gz") == payload
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|