hashserver 1.0.2__tar.gz → 1.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. {hashserver-1.0.2/hashserver.egg-info → hashserver-1.1.0}/PKG-INFO +21 -4
  2. {hashserver-1.0.2 → hashserver-1.1.0}/README.md +18 -2
  3. hashserver-1.1.0/compression_utils.py +51 -0
  4. {hashserver-1.0.2 → hashserver-1.1.0}/hash_file_response.py +133 -19
  5. {hashserver-1.0.2 → hashserver-1.1.0/hashserver.egg-info}/PKG-INFO +21 -4
  6. {hashserver-1.0.2 → hashserver-1.1.0}/hashserver.egg-info/SOURCES.txt +3 -0
  7. {hashserver-1.0.2 → hashserver-1.1.0}/hashserver.egg-info/requires.txt +1 -0
  8. {hashserver-1.0.2 → hashserver-1.1.0}/hashserver.egg-info/top_level.txt +1 -0
  9. {hashserver-1.0.2 → hashserver-1.1.0}/hashserver.py +170 -8
  10. {hashserver-1.0.2 → hashserver-1.1.0}/pyproject.toml +5 -4
  11. hashserver-1.1.0/tests/test_compression.py +259 -0
  12. hashserver-1.1.0/tests/test_compression_utils.py +21 -0
  13. {hashserver-1.0.2 → hashserver-1.1.0}/LICENSE.txt +0 -0
  14. {hashserver-1.0.2 → hashserver-1.1.0}/hashserver.egg-info/dependency_links.txt +0 -0
  15. {hashserver-1.0.2 → hashserver-1.1.0}/hashserver.egg-info/entry_points.txt +0 -0
  16. {hashserver-1.0.2 → hashserver-1.1.0}/setup.cfg +0 -0
  17. {hashserver-1.0.2 → hashserver-1.1.0}/tests/test_basic.py +0 -0
  18. {hashserver-1.0.2 → hashserver-1.1.0}/tests/test_basic_uvicorn.py +0 -0
  19. {hashserver-1.0.2 → hashserver-1.1.0}/tests/test_global_lock.py +0 -0
  20. {hashserver-1.0.2 → hashserver-1.1.0}/tests/test_has_buffers.py +0 -0
  21. {hashserver-1.0.2 → hashserver-1.1.0}/tests/test_lock.py +0 -0
  22. {hashserver-1.0.2 → hashserver-1.1.0}/tests/test_lock_uvicorn.py +0 -0
  23. {hashserver-1.0.2 → hashserver-1.1.0}/tests/test_port_range.py +0 -0
  24. {hashserver-1.0.2 → hashserver-1.1.0}/tests/test_promises.py +0 -0
  25. {hashserver-1.0.2 → hashserver-1.1.0}/tests/test_put_read.py +0 -0
  26. {hashserver-1.0.2 → hashserver-1.1.0}/tests/test_put_read_big.py +0 -0
  27. {hashserver-1.0.2 → hashserver-1.1.0}/tests/test_simple_put.py +0 -0
  28. {hashserver-1.0.2 → hashserver-1.1.0}/tests/test_timeout.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: hashserver
3
- Version: 1.0.2
3
+ Version: 1.1.0
4
4
  Summary: Simple FastAPI-based hash server
5
5
  Author: Sjoerd de Vries
6
6
  License-Expression: MIT
@@ -15,12 +15,13 @@ Classifier: Programming Language :: Python :: 3 :: Only
15
15
  Classifier: Operating System :: OS Independent
16
16
  Classifier: Topic :: Scientific/Engineering
17
17
  Classifier: Topic :: Software Development :: Libraries
18
- Requires-Python: >=3.9
18
+ Requires-Python: >=3.10
19
19
  Description-Content-Type: text/markdown
20
20
  License-File: LICENSE.txt
21
21
  Requires-Dist: fastapi
22
22
  Requires-Dist: uvicorn[standard]
23
23
  Requires-Dist: typing-extensions
24
+ Requires-Dist: zstandard
24
25
  Provides-Extra: dev
25
26
  Dynamic: license-file
26
27
 
@@ -48,7 +49,7 @@ However, **hashserver has no dependency on Seamless** and no knowledge of it. It
48
49
 
49
50
  - **Content-addressed**: buffers are stored and retrieved by their cryptographic checksum.
50
51
  - **Configurable hash algorithm**: SHA-256 (default) or SHA3-256, selected at startup.
51
- - **Integrity-verified reads**: every buffer is re-checksummed on GET to detect corruption.
52
+ - **Integrity-verified reads**: every buffer (if not compressed) is re-checksummed on GET to detect corruption.
52
53
  - **Prefix directory layout**: by default, buffers are stored under a two-character prefix subdirectory (e.g. `ab/ab3f7c...`) to avoid filesystem performance problems with large flat directories. A flat layout is also supported.
53
54
  - **Extra read-only directories**: additional buffer directories can be mounted as fallback read sources.
54
55
  - **Promises**: a client can announce that a buffer will be uploaded soon via `PUT /promise/{checksum}`. Other clients reading that checksum will wait for the upload rather than getting a 404.
@@ -193,6 +194,22 @@ In **flat** layout, the same buffer is stored as `<directory>/ab3f7c...`.
193
194
 
194
195
  Extra directories auto-detect their layout by checking for the `.HASHSERVER_PREFIX` sentinel.
195
196
 
197
+ ## Compression
198
+
199
+ Hashserver supports transparent compression for stored buffers. Supported formats: Zstandard (`.zst`) and gzip (`.gz`).
200
+
201
+ **Core invariant:** The URL checksum is always the canonical checksum of the *decompressed* bytes. A compressed upload of the same content produces the same URL as an uncompressed upload.
202
+
203
+ **Uploading compressed buffers:** Send `Content-Encoding: zstd` (or `gzip`) on PUT. The server streams the compressed body to disk while simultaneously decompressing to verify the canonical checksum. The compressed bytes are stored at `{dir}/{cs}.zst` (or `.gz`).
204
+
205
+ **Downloading:** GET always uses the canonical checksum URL. If the stored form is compressed, the response includes `Content-Encoding: zstd` (or `gzip`). The client must check this header and decompress if present. The server tries to honor `Accept-Encoding` when a matching on-disk form exists; otherwise it falls back to the uncompressed file or the server's normal on-disk preference order.
206
+
207
+ **Coexistence:** Compressed and uncompressed forms of the same buffer coexist safely: `{cs}`, `{cs}.zst`, `{cs}.gz` can all be present.
208
+
209
+ **`/has` and `/has-now`:** Any stored form satisfies the existence check.
210
+
211
+ **`/buffer-length`:** Always returns the *uncompressed* byte count. When only a compressed form exists, this is read from a `.BUFFERLENGTH` sidecar file written during the compressed PUT.
212
+
196
213
  ## Running tests
197
214
 
198
215
  ```bash
@@ -202,4 +219,4 @@ pytest tests/
202
219
 
203
220
  ## License
204
221
 
205
- See [LICENSE.txt](LICENSE.txt).
222
+ MIT. See [LICENSE.txt](LICENSE.txt).
@@ -22,7 +22,7 @@ However, **hashserver has no dependency on Seamless** and no knowledge of it. It
22
22
 
23
23
  - **Content-addressed**: buffers are stored and retrieved by their cryptographic checksum.
24
24
  - **Configurable hash algorithm**: SHA-256 (default) or SHA3-256, selected at startup.
25
- - **Integrity-verified reads**: every buffer is re-checksummed on GET to detect corruption.
25
+ - **Integrity-verified reads**: every buffer (if not compressed) is re-checksummed on GET to detect corruption.
26
26
  - **Prefix directory layout**: by default, buffers are stored under a two-character prefix subdirectory (e.g. `ab/ab3f7c...`) to avoid filesystem performance problems with large flat directories. A flat layout is also supported.
27
27
  - **Extra read-only directories**: additional buffer directories can be mounted as fallback read sources.
28
28
  - **Promises**: a client can announce that a buffer will be uploaded soon via `PUT /promise/{checksum}`. Other clients reading that checksum will wait for the upload rather than getting a 404.
@@ -167,6 +167,22 @@ In **flat** layout, the same buffer is stored as `<directory>/ab3f7c...`.
167
167
 
168
168
  Extra directories auto-detect their layout by checking for the `.HASHSERVER_PREFIX` sentinel.
169
169
 
170
+ ## Compression
171
+
172
+ Hashserver supports transparent compression for stored buffers. Supported formats: Zstandard (`.zst`) and gzip (`.gz`).
173
+
174
+ **Core invariant:** The URL checksum is always the canonical checksum of the *decompressed* bytes. A compressed upload of the same content produces the same URL as an uncompressed upload.
175
+
176
+ **Uploading compressed buffers:** Send `Content-Encoding: zstd` (or `gzip`) on PUT. The server streams the compressed body to disk while simultaneously decompressing to verify the canonical checksum. The compressed bytes are stored at `{dir}/{cs}.zst` (or `.gz`).
177
+
178
+ **Downloading:** GET always uses the canonical checksum URL. If the stored form is compressed, the response includes `Content-Encoding: zstd` (or `gzip`). The client must check this header and decompress if present. The server tries to honor `Accept-Encoding` when a matching on-disk form exists; otherwise it falls back to the uncompressed file or the server's normal on-disk preference order.
179
+
180
+ **Coexistence:** Compressed and uncompressed forms of the same buffer coexist safely: `{cs}`, `{cs}.zst`, `{cs}.gz` can all be present.
181
+
182
+ **`/has` and `/has-now`:** Any stored form satisfies the existence check.
183
+
184
+ **`/buffer-length`:** Always returns the *uncompressed* byte count. When only a compressed form exists, this is read from a `.BUFFERLENGTH` sidecar file written during the compressed PUT.
185
+
170
186
  ## Running tests
171
187
 
172
188
  ```bash
@@ -176,4 +192,4 @@ pytest tests/
176
192
 
177
193
  ## License
178
194
 
179
- See [LICENSE.txt](LICENSE.txt).
195
+ MIT. See [LICENSE.txt](LICENSE.txt).
@@ -0,0 +1,51 @@
1
+ from __future__ import annotations
2
+
3
+ import gzip
4
+ from typing import Optional, Tuple
5
+
6
+ import zstandard
7
+
8
+ COMPRESSION_SUFFIXES = (".zst", ".gz")
9
+ COMPRESSION_PREFERENCE = (".zst", ".gz")
10
+ CONTENT_ENCODING_TO_SUFFIX = {
11
+ "zstd": ".zst",
12
+ "gzip": ".gz",
13
+ }
14
+ SUFFIX_TO_CONTENT_ENCODING = {
15
+ suffix: encoding for encoding, suffix in CONTENT_ENCODING_TO_SUFFIX.items()
16
+ }
17
+
18
+
19
+ def strip_compression_suffix(name: str) -> Tuple[str, Optional[str]]:
20
+ for suffix in COMPRESSION_SUFFIXES:
21
+ if name.endswith(suffix):
22
+ return name[: -len(suffix)], suffix
23
+ return name, None
24
+
25
+
26
+ def decompress_bytes(data: bytes, suffix: str) -> bytes:
27
+ if suffix == ".zst":
28
+ return zstandard.ZstdDecompressor().decompress(data)
29
+ if suffix == ".gz":
30
+ return gzip.decompress(data)
31
+ raise ValueError(suffix)
32
+
33
+
34
+ def compress_bytes(data: bytes, suffix: str) -> bytes:
35
+ if suffix == ".zst":
36
+ return zstandard.ZstdCompressor().compress(data)
37
+ if suffix == ".gz":
38
+ return gzip.compress(data)
39
+ raise ValueError(suffix)
40
+
41
+
42
+ def content_encoding_to_suffix(value: Optional[str]) -> Optional[str]:
43
+ if value is None:
44
+ return None
45
+ return CONTENT_ENCODING_TO_SUFFIX.get(value.strip().lower())
46
+
47
+
48
+ def suffix_to_content_encoding(suffix: Optional[str]) -> Optional[str]:
49
+ if suffix is None:
50
+ return None
51
+ return SUFFIX_TO_CONTENT_ENCODING.get(suffix)
@@ -2,14 +2,21 @@ import os
2
2
  import stat
3
3
  import time
4
4
  import typing
5
+ import zlib
5
6
  from hashlib import sha3_256, sha256
6
7
 
7
8
  import anyio
9
+ import zstandard
8
10
 
9
11
  from starlette.background import BackgroundTask
10
12
  from starlette.types import Receive, Scope, Send
11
13
  from starlette.responses import FileResponse
12
14
 
15
+ from compression_utils import (
16
+ COMPRESSION_PREFERENCE,
17
+ suffix_to_content_encoding,
18
+ )
19
+
13
20
  HASH_ALGORITHMS = {
14
21
  "sha3-256": sha3_256,
15
22
  "sha-256": sha256,
@@ -64,6 +71,87 @@ class HashFileResponse(FileResponse):
64
71
  lock_timeout = 120
65
72
  chunk_size = 640 * 1024
66
73
 
74
+ @staticmethod
75
+ def _parse_accept_encoding(accept_encoding: str | None) -> list[str | None]:
76
+ if not accept_encoding:
77
+ return [None] + list(COMPRESSION_PREFERENCE)
78
+
79
+ parsed: list[tuple[float, int, str | None]] = []
80
+ for idx, item in enumerate(accept_encoding.split(",")):
81
+ item = item.strip()
82
+ if not item:
83
+ continue
84
+ parts = [part.strip() for part in item.split(";")]
85
+ encoding = parts[0].lower()
86
+ q = 1.0
87
+ for param in parts[1:]:
88
+ if param.startswith("q="):
89
+ try:
90
+ q = float(param[2:])
91
+ except ValueError:
92
+ q = 0.0
93
+ if q <= 0:
94
+ continue
95
+ if encoding == "gzip":
96
+ parsed.append((q, idx, ".gz"))
97
+ elif encoding in ("zstd", "x-zstd"):
98
+ parsed.append((q, idx, ".zst"))
99
+ elif encoding == "identity":
100
+ parsed.append((q, idx, None))
101
+ elif encoding == "*":
102
+ parsed.append((q, idx, "*"))
103
+
104
+ parsed.sort(key=lambda item: (-item[0], item[1]))
105
+ ordered: list[str | None] = []
106
+ seen: set[str | None] = set()
107
+ wildcard = False
108
+ for _q, _idx, suffix in parsed:
109
+ if suffix == "*":
110
+ wildcard = True
111
+ continue
112
+ if suffix not in seen:
113
+ seen.add(suffix)
114
+ ordered.append(suffix)
115
+
116
+ if wildcard:
117
+ for suffix in [None, *COMPRESSION_PREFERENCE]:
118
+ if suffix not in seen:
119
+ seen.add(suffix)
120
+ ordered.append(suffix)
121
+
122
+ if None not in seen:
123
+ ordered.append(None)
124
+ for suffix in COMPRESSION_PREFERENCE:
125
+ if suffix not in seen:
126
+ ordered.append(suffix)
127
+ return ordered
128
+
129
+ def _compression_candidates(self, path: str) -> list[tuple[str, str | None]]:
130
+ candidates = [(path, None)] + [
131
+ (path + suffix, suffix) for suffix in COMPRESSION_PREFERENCE
132
+ ]
133
+ preferred_suffixes = self._parse_accept_encoding(self._accept_encoding)
134
+ ordered_candidates: list[tuple[str, str | None]] = []
135
+ seen: set[tuple[str, str | None]] = set()
136
+ for suffix in preferred_suffixes:
137
+ candidate = (path if suffix is None else path + suffix, suffix)
138
+ if candidate not in seen:
139
+ seen.add(candidate)
140
+ ordered_candidates.append(candidate)
141
+ for candidate in candidates:
142
+ if candidate not in seen:
143
+ seen.add(candidate)
144
+ ordered_candidates.append(candidate)
145
+ return ordered_candidates
146
+
147
+ @staticmethod
148
+ def _get_decompressor(suffix: str):
149
+ if suffix == ".zst":
150
+ return zstandard.ZstdDecompressor().decompressobj()
151
+ if suffix == ".gz":
152
+ return zlib.decompressobj(wbits=31)
153
+ raise ValueError(suffix)
154
+
67
155
  def __init__(
68
156
  self,
69
157
  checksum: str,
@@ -73,9 +161,9 @@ class HashFileResponse(FileResponse):
73
161
  media_type: typing.Optional[str] = None,
74
162
  background: typing.Optional[BackgroundTask] = None,
75
163
  stat_result: typing.Optional[os.stat_result] = None,
76
- method: typing.Optional[str] = None,
77
164
  content_disposition_type: str = "attachment",
78
165
  extra_dirs: typing.Optional[typing.List[str]] = None,
166
+ accept_encoding: typing.Optional[str] = None,
79
167
  ) -> None:
80
168
  filename = parse_checksum(checksum)
81
169
  self.prefix = filename[:2]
@@ -84,6 +172,9 @@ class HashFileResponse(FileResponse):
84
172
  path = os.path.join(directory, self.prefix, filename)
85
173
  else:
86
174
  path = os.path.join(directory, filename)
175
+ self._primary_path = path
176
+ self._compression_suffix = None
177
+ self._accept_encoding = accept_encoding
87
178
  super().__init__(
88
179
  path=path,
89
180
  status_code=status_code,
@@ -92,7 +183,6 @@ class HashFileResponse(FileResponse):
92
183
  background=background,
93
184
  filename=filename,
94
185
  stat_result=stat_result,
95
- method=method,
96
186
  content_disposition_type=content_disposition_type,
97
187
  )
98
188
  self.directory = directory
@@ -107,16 +197,34 @@ class HashFileResponse(FileResponse):
107
197
  extra_dirs_layout[extra_dir] = layout
108
198
  self.extra_dirs_layout = extra_dirs_layout
109
199
 
200
+ async def _resolve_existing_path(
201
+ self, base_path: str
202
+ ) -> tuple[str | None, str | None]:
203
+ for candidate_path, suffix in self._compression_candidates(base_path):
204
+ if await anyio.Path(candidate_path).exists():
205
+ return candidate_path, suffix
206
+ return None, None
207
+
110
208
  async def refresh_stat_headers(self):
111
- if self.extra_dirs and not await anyio.Path(self.path).exists():
209
+ resolved_path, compression_suffix = await self._resolve_existing_path(
210
+ self._primary_path
211
+ )
212
+ if resolved_path is not None:
213
+ self.path = resolved_path
214
+ self._compression_suffix = compression_suffix
215
+ elif self.extra_dirs:
112
216
  for extra_dir in self.extra_dirs:
113
217
  layout = self.extra_dirs_layout[extra_dir]
114
218
  if layout == "prefix":
115
- path0 = os.path.join(extra_dir, self.prefix, self.filename)
219
+ base_path = os.path.join(extra_dir, self.prefix, self.filename)
116
220
  else:
117
- path0 = os.path.join(extra_dir, self.filename)
118
- if await anyio.Path(path0).exists():
119
- self.path = path0
221
+ base_path = os.path.join(extra_dir, self.filename)
222
+ resolved_path, compression_suffix = await self._resolve_existing_path(
223
+ base_path
224
+ )
225
+ if resolved_path is not None:
226
+ self.path = resolved_path
227
+ self._compression_suffix = compression_suffix
120
228
  break
121
229
 
122
230
  try:
@@ -176,16 +284,22 @@ class HashFileResponse(FileResponse):
176
284
  stat_result = await self.refresh_stat_headers()
177
285
  self.stat_result = stat_result
178
286
 
179
- checksum = await self.calculate_checksum()
180
- if checksum != self.filename:
181
- await self.until_no_lock()
182
- stat_result = await self.refresh_stat_headers()
183
- self.stat_result = stat_result
184
- checksum2 = await self.calculate_checksum()
185
- if checksum2 != self.filename:
186
- raise RuntimeError(
187
- f"File corruption: file at path {self.path} does not have the correct {_current_hash_algorithm} checksum."
188
- )
287
+ if self._compression_suffix is not None:
288
+ content_encoding = suffix_to_content_encoding(self._compression_suffix)
289
+ if content_encoding is not None:
290
+ self.headers["content-encoding"] = content_encoding
291
+
292
+ if self._compression_suffix is None:
293
+ checksum = await self.calculate_checksum()
294
+ if checksum != self.filename:
295
+ await self.until_no_lock()
296
+ stat_result = await self.refresh_stat_headers()
297
+ self.stat_result = stat_result
298
+ checksum2 = await self.calculate_checksum()
299
+ if checksum2 != self.filename:
300
+ raise RuntimeError(
301
+ f"File corruption: file at path {self.path} does not have the correct {_current_hash_algorithm} checksum."
302
+ )
189
303
 
190
304
  await super().__call__(scope=scope, receive=receive, send=send)
191
305
 
@@ -209,9 +323,9 @@ class PrefixHashFileResponse(HashFileResponse):
209
323
  media_type: typing.Optional[str] = None,
210
324
  background: typing.Optional[BackgroundTask] = None,
211
325
  stat_result: typing.Optional[os.stat_result] = None,
212
- method: typing.Optional[str] = None,
213
326
  content_disposition_type: str = "attachment",
214
327
  extra_dirs: typing.Optional[typing.List[str]] = None,
328
+ accept_encoding: typing.Optional[str] = None,
215
329
  ) -> None:
216
330
 
217
331
  super().__init__(
@@ -222,9 +336,9 @@ class PrefixHashFileResponse(HashFileResponse):
222
336
  media_type=media_type,
223
337
  background=background,
224
338
  stat_result=stat_result,
225
- method=method,
226
339
  content_disposition_type=content_disposition_type,
227
340
  extra_dirs=extra_dirs,
341
+ accept_encoding=accept_encoding,
228
342
  )
229
343
  prefix_file = os.path.join(directory, ".HASHSERVER_PREFIX")
230
344
  with open(prefix_file, mode="wb") as f:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: hashserver
3
- Version: 1.0.2
3
+ Version: 1.1.0
4
4
  Summary: Simple FastAPI-based hash server
5
5
  Author: Sjoerd de Vries
6
6
  License-Expression: MIT
@@ -15,12 +15,13 @@ Classifier: Programming Language :: Python :: 3 :: Only
15
15
  Classifier: Operating System :: OS Independent
16
16
  Classifier: Topic :: Scientific/Engineering
17
17
  Classifier: Topic :: Software Development :: Libraries
18
- Requires-Python: >=3.9
18
+ Requires-Python: >=3.10
19
19
  Description-Content-Type: text/markdown
20
20
  License-File: LICENSE.txt
21
21
  Requires-Dist: fastapi
22
22
  Requires-Dist: uvicorn[standard]
23
23
  Requires-Dist: typing-extensions
24
+ Requires-Dist: zstandard
24
25
  Provides-Extra: dev
25
26
  Dynamic: license-file
26
27
 
@@ -48,7 +49,7 @@ However, **hashserver has no dependency on Seamless** and no knowledge of it. It
48
49
 
49
50
  - **Content-addressed**: buffers are stored and retrieved by their cryptographic checksum.
50
51
  - **Configurable hash algorithm**: SHA-256 (default) or SHA3-256, selected at startup.
51
- - **Integrity-verified reads**: every buffer is re-checksummed on GET to detect corruption.
52
+ - **Integrity-verified reads**: every buffer (if not compressed) is re-checksummed on GET to detect corruption.
52
53
  - **Prefix directory layout**: by default, buffers are stored under a two-character prefix subdirectory (e.g. `ab/ab3f7c...`) to avoid filesystem performance problems with large flat directories. A flat layout is also supported.
53
54
  - **Extra read-only directories**: additional buffer directories can be mounted as fallback read sources.
54
55
  - **Promises**: a client can announce that a buffer will be uploaded soon via `PUT /promise/{checksum}`. Other clients reading that checksum will wait for the upload rather than getting a 404.
@@ -193,6 +194,22 @@ In **flat** layout, the same buffer is stored as `<directory>/ab3f7c...`.
193
194
 
194
195
  Extra directories auto-detect their layout by checking for the `.HASHSERVER_PREFIX` sentinel.
195
196
 
197
+ ## Compression
198
+
199
+ Hashserver supports transparent compression for stored buffers. Supported formats: Zstandard (`.zst`) and gzip (`.gz`).
200
+
201
+ **Core invariant:** The URL checksum is always the canonical checksum of the *decompressed* bytes. A compressed upload of the same content produces the same URL as an uncompressed upload.
202
+
203
+ **Uploading compressed buffers:** Send `Content-Encoding: zstd` (or `gzip`) on PUT. The server streams the compressed body to disk while simultaneously decompressing to verify the canonical checksum. The compressed bytes are stored at `{dir}/{cs}.zst` (or `.gz`).
204
+
205
+ **Downloading:** GET always uses the canonical checksum URL. If the stored form is compressed, the response includes `Content-Encoding: zstd` (or `gzip`). The client must check this header and decompress if present. The server tries to honor `Accept-Encoding` when a matching on-disk form exists; otherwise it falls back to the uncompressed file or the server's normal on-disk preference order.
206
+
207
+ **Coexistence:** Compressed and uncompressed forms of the same buffer coexist safely: `{cs}`, `{cs}.zst`, `{cs}.gz` can all be present.
208
+
209
+ **`/has` and `/has-now`:** Any stored form satisfies the existence check.
210
+
211
+ **`/buffer-length`:** Always returns the *uncompressed* byte count. When only a compressed form exists, this is read from a `.BUFFERLENGTH` sidecar file written during the compressed PUT.
212
+
196
213
  ## Running tests
197
214
 
198
215
  ```bash
@@ -202,4 +219,4 @@ pytest tests/
202
219
 
203
220
  ## License
204
221
 
205
- See [LICENSE.txt](LICENSE.txt).
222
+ MIT. See [LICENSE.txt](LICENSE.txt).
@@ -1,5 +1,6 @@
1
1
  LICENSE.txt
2
2
  README.md
3
+ compression_utils.py
3
4
  hash_file_response.py
4
5
  hashserver.py
5
6
  pyproject.toml
@@ -11,6 +12,8 @@ hashserver.egg-info/requires.txt
11
12
  hashserver.egg-info/top_level.txt
12
13
  tests/test_basic.py
13
14
  tests/test_basic_uvicorn.py
15
+ tests/test_compression.py
16
+ tests/test_compression_utils.py
14
17
  tests/test_global_lock.py
15
18
  tests/test_has_buffers.py
16
19
  tests/test_lock.py
@@ -1,5 +1,6 @@
1
1
  fastapi
2
2
  uvicorn[standard]
3
3
  typing-extensions
4
+ zstandard
4
5
 
5
6
  [dev]
@@ -1,2 +1,3 @@
1
+ compression_utils
1
2
  hash_file_response
2
3
  hashserver
@@ -11,6 +11,7 @@ import aiofiles.os
11
11
  import aiofiles.tempfile
12
12
  import contextlib
13
13
  import copy
14
+ import zlib
14
15
  from dataclasses import dataclass
15
16
  from typing import Iterable, List, Optional, Set, Union
16
17
  from fastapi import FastAPI, Path, Body, Request
@@ -37,6 +38,12 @@ from pydantic.functional_validators import BeforeValidator
37
38
  import anyio
38
39
  import pathlib
39
40
  import time
41
+ import zstandard
42
+
43
+ from compression_utils import (
44
+ COMPRESSION_PREFERENCE,
45
+ content_encoding_to_suffix,
46
+ )
40
47
 
41
48
  Checksum = Annotated[str, BeforeValidator(parse_checksum)]
42
49
 
@@ -67,6 +74,54 @@ def calculate_checksum_stream():
67
74
  return checksum_constructor()
68
75
 
69
76
 
77
+ def _compression_candidates(path: str) -> list[tuple[str, str | None]]:
78
+ return [(path, None)] + [
79
+ (path + suffix, suffix) for suffix in COMPRESSION_PREFERENCE
80
+ ]
81
+
82
+
83
+ def _get_streaming_decompressor(suffix: str):
84
+ if suffix == ".zst":
85
+ return zstandard.ZstdDecompressor().decompressobj()
86
+ if suffix == ".gz":
87
+ return zlib.decompressobj(wbits=31)
88
+ raise ValueError(suffix)
89
+
90
+
91
+ async def _read_buffer_length_sidecar(path: str) -> int | None:
92
+ try:
93
+ async with await anyio.open_file(path + ".BUFFERLENGTH", mode="r") as sidecar:
94
+ data = await sidecar.read()
95
+ except FileNotFoundError:
96
+ return None
97
+ return int(data.strip())
98
+
99
+
100
+ async def _write_buffer_length_sidecar(path: str, buffer_length: int) -> None:
101
+ async with await anyio.open_file(path + ".BUFFERLENGTH", mode="w") as sidecar:
102
+ await sidecar.write(str(buffer_length))
103
+
104
+
105
+ async def _remove_buffer_length_sidecar(path: str) -> None:
106
+ try:
107
+ await aiofiles.os.unlink(path + ".BUFFERLENGTH")
108
+ except FileNotFoundError:
109
+ pass
110
+
111
+
112
+ async def _stat_canonical_buffer_length(path: str, suffix: str) -> int:
113
+ decompressor = _get_streaming_decompressor(suffix)
114
+ total_length = 0
115
+ async with await anyio.open_file(path, mode="rb") as file:
116
+ while True:
117
+ chunk = await file.read(640 * 1024)
118
+ if not chunk:
119
+ break
120
+ total_length += len(decompressor.decompress(chunk))
121
+ total_length += len(decompressor.flush())
122
+ return total_length
123
+
124
+
70
125
  def wait_for_status_file(path: str, timeout: float = STATUS_FILE_WAIT_TIMEOUT):
71
126
  deadline = time.monotonic() + timeout
72
127
  while True:
@@ -476,6 +531,65 @@ async def buffer_length(checksums: Annotated[List[Checksum], Body()]) -> JSONRes
476
531
  continue
477
532
  curr_results[nr] = stat.st_size
478
533
 
534
+ async def read_sidecars(paths):
535
+ futures = []
536
+ for _, path in paths:
537
+ futures.append(_read_buffer_length_sidecar(path))
538
+ result0 = await asyncio.gather(*futures, return_exceptions=True)
539
+ for (nr, _path), value in zip(paths, result0):
540
+ if isinstance(value, Exception) or value is None:
541
+ continue
542
+ curr_results[nr] = value
543
+
544
+ async def stat_compressed(paths):
545
+ futures = []
546
+ metadata = []
547
+ for nr, path in paths:
548
+ for candidate_path, suffix in _compression_candidates(path)[1:]:
549
+ metadata.append((nr, candidate_path, suffix))
550
+ futures.append(anyio.Path(candidate_path).exists())
551
+ result0 = await asyncio.gather(*futures, return_exceptions=True)
552
+ fallback = []
553
+ for (nr, candidate_path, suffix), exists in zip(metadata, result0):
554
+ if curr_results[nr]:
555
+ continue
556
+ if isinstance(exists, Exception) or not exists:
557
+ continue
558
+ fallback.append((nr, candidate_path, suffix))
559
+ if not fallback:
560
+ return
561
+ sidecar_reads = []
562
+ for _nr, candidate_path, suffix in fallback:
563
+ sidecar_reads.append(
564
+ _read_buffer_length_sidecar(candidate_path[: -len(suffix)])
565
+ )
566
+ sidecar_values = await asyncio.gather(*sidecar_reads, return_exceptions=True)
567
+ expensive = []
568
+ for (nr, candidate_path, suffix), sidecar_value in zip(
569
+ fallback, sidecar_values
570
+ ):
571
+ if curr_results[nr]:
572
+ continue
573
+ if isinstance(sidecar_value, Exception):
574
+ expensive.append((nr, candidate_path, suffix))
575
+ continue
576
+ if sidecar_value is None:
577
+ expensive.append((nr, candidate_path, suffix))
578
+ continue
579
+ curr_results[nr] = sidecar_value
580
+ if expensive:
581
+ lengths = await asyncio.gather(
582
+ *[
583
+ _stat_canonical_buffer_length(candidate_path, suffix)
584
+ for _, candidate_path, suffix in expensive
585
+ ],
586
+ return_exceptions=True,
587
+ )
588
+ for (nr, _candidate_path, _suffix), length in zip(expensive, lengths):
589
+ if isinstance(length, Exception):
590
+ continue
591
+ curr_results[nr] = length
592
+
479
593
  paths = []
480
594
  for nr, checksum in enumerate(checksums2):
481
595
  assert isinstance(checksum, str)
@@ -487,8 +601,15 @@ async def buffer_length(checksums: Annotated[List[Checksum], Body()]) -> JSONRes
487
601
  paths.append((nr, path))
488
602
 
489
603
  await stat_all(paths)
604
+ unresolved_paths = [item for item in paths if not curr_results[item[0]]]
605
+ if unresolved_paths:
606
+ await read_sidecars(unresolved_paths)
607
+ unresolved_paths = [item for item in paths if not curr_results[item[0]]]
608
+ if unresolved_paths:
609
+ await stat_compressed(unresolved_paths)
490
610
 
491
611
  for extra_dir in extra_dirs:
612
+ paths = []
492
613
  for nr, checksum in enumerate(checksums2):
493
614
  if curr_results[nr]:
494
615
  continue
@@ -497,6 +618,12 @@ async def buffer_length(checksums: Annotated[List[Checksum], Body()]) -> JSONRes
497
618
  if not len(paths):
498
619
  break
499
620
  await stat_all(paths)
621
+ unresolved_paths = [item for item in paths if not curr_results[item[0]]]
622
+ if unresolved_paths:
623
+ await read_sidecars(unresolved_paths)
624
+ unresolved_paths = [item for item in paths if not curr_results[item[0]]]
625
+ if unresolved_paths:
626
+ await stat_compressed(unresolved_paths)
500
627
 
501
628
  promised = await _promise_registry.promised_indices(checksums2)
502
629
  for idx in promised:
@@ -516,11 +643,13 @@ async def _has(checksums: List[Checksum], include_promises: bool) -> List[bool]:
516
643
 
517
644
  async def exists_all(paths):
518
645
  futures = []
519
- for _, path in paths:
520
- fut = anyio.Path(path).exists()
521
- futures.append(fut)
646
+ metadata = []
647
+ for nr, path in paths:
648
+ for candidate_path, _suffix in _compression_candidates(path):
649
+ metadata.append((nr, candidate_path))
650
+ futures.append(anyio.Path(candidate_path).exists())
522
651
  result0 = await asyncio.gather(*futures, return_exceptions=True)
523
- for (nr, path), exists in zip(paths, result0):
652
+ for (nr, _path), exists in zip(metadata, result0):
524
653
  if isinstance(exists, Exception):
525
654
  continue
526
655
  if exists:
@@ -608,13 +737,18 @@ async def healthcheck() -> Response:
608
737
 
609
738
 
610
739
  @app.get("/{checksum}")
611
- async def get_file(checksum: Annotated[Checksum, Path()]) -> HashFileResponse:
740
+ async def get_file(
741
+ checksum: Annotated[Checksum, Path()], rq: Request
742
+ ) -> HashFileResponse:
612
743
  checksum2 = parse_checksum(checksum)
613
744
  LOGGER.info("GET %s", checksum2)
614
745
  await _wait_for_current_put_requests((checksum2,))
615
746
  ResponseClass = _response_classes_get_file[layout]
616
747
  response = ResponseClass(
617
- directory=directory, checksum=checksum2, extra_dirs=extra_dirs
748
+ directory=directory,
749
+ checksum=checksum2,
750
+ extra_dirs=extra_dirs,
751
+ accept_encoding=rq.headers.get("accept-encoding"),
618
752
  )
619
753
  return response
620
754
 
@@ -728,7 +862,9 @@ async def put_file(checksum: Annotated[Checksum, Path()], rq: Request) -> Respon
728
862
  target_dir = os.path.join(directory, prefix)
729
863
  else:
730
864
  target_dir = directory
731
- path = os.path.join(target_dir, checksum_str)
865
+ base_path = os.path.join(target_dir, checksum_str)
866
+ compression_suffix = content_encoding_to_suffix(rq.headers.get("content-encoding"))
867
+ path = base_path if compression_suffix is None else base_path + compression_suffix
732
868
 
733
869
  if layout == "prefix":
734
870
  target_directory = anyio.Path(target_dir)
@@ -746,6 +882,8 @@ async def put_file(checksum: Annotated[Checksum, Path()], rq: Request) -> Respon
746
882
  cs_stream = calculate_checksum_stream()
747
883
  temp_path = None
748
884
  buffer_checksum = None
885
+ decompressed_length = 0
886
+ decompressor = None
749
887
  try:
750
888
  async with _current_put_condition:
751
889
  if checksum_str in _current_put_requests:
@@ -753,14 +891,28 @@ async def put_file(checksum: Annotated[Checksum, Path()], rq: Request) -> Respon
753
891
  return Response(status_code=202)
754
892
  _current_put_requests.add(checksum_str)
755
893
  added_to_put_requests = True
894
+ if compression_suffix is not None:
895
+ decompressor = _get_streaming_decompressor(compression_suffix)
756
896
  async with aiofiles.tempfile.NamedTemporaryFile(
757
897
  dir=target_dir,
758
898
  prefix=checksum_str + "-",
759
899
  delete=False,
760
900
  ) as file:
761
901
  async for chunk in rq.stream():
762
- cs_stream.update(chunk)
902
+ if not chunk:
903
+ continue
763
904
  await file.write(chunk)
905
+ if decompressor is None:
906
+ cs_stream.update(chunk)
907
+ decompressed_length += len(chunk)
908
+ else:
909
+ decompressed = decompressor.decompress(chunk)
910
+ cs_stream.update(decompressed)
911
+ decompressed_length += len(decompressed)
912
+ if decompressor is not None:
913
+ flushed = decompressor.flush()
914
+ cs_stream.update(flushed)
915
+ decompressed_length += len(flushed)
764
916
  buffer_checksum = cs_stream.hexdigest()
765
917
  temp_path = file.name
766
918
  if buffer_checksum != checksum_str:
@@ -772,6 +924,10 @@ async def put_file(checksum: Annotated[Checksum, Path()], rq: Request) -> Respon
772
924
  except Exception:
773
925
  if not await aiofiles.ospath.exists(path):
774
926
  raise
927
+ if compression_suffix is None:
928
+ await _remove_buffer_length_sidecar(base_path)
929
+ else:
930
+ await _write_buffer_length_sidecar(base_path, decompressed_length)
775
931
  ok = True
776
932
  try:
777
933
  await aiofiles_chmod(path, 0o444)
@@ -781,6 +937,9 @@ async def put_file(checksum: Annotated[Checksum, Path()], rq: Request) -> Respon
781
937
  except ClientDisconnect:
782
938
  LOGGER.warning("PUT %s client disconnected", checksum_str)
783
939
  return Response(status_code=400)
940
+ except (zlib.error, zstandard.ZstdError, ValueError):
941
+ LOGGER.warning("PUT %s invalid compressed payload", checksum_str)
942
+ return Response(status_code=400, content="Invalid compressed payload")
784
943
 
785
944
  finally:
786
945
  if added_to_put_requests:
@@ -865,6 +1024,9 @@ if as_commandline_tool:
865
1024
  print("OK")
866
1025
  try:
867
1026
  server.run()
1027
+ except KeyboardInterrupt:
1028
+ # Normal interactive shutdown path (CTRL+C / forwarded SIGINT).
1029
+ pass
868
1030
  except BaseException:
869
1031
  if status_tracker and not status_tracker.running_written:
870
1032
  status_tracker.write_failed()
@@ -4,16 +4,17 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "hashserver"
7
- version = "1.0.2"
7
+ version = "1.1.0"
8
8
  description = "Simple FastAPI-based hash server"
9
9
  readme = "README.md"
10
10
  license = "MIT"
11
11
  authors = [{name = "Sjoerd de Vries"}]
12
- requires-python = ">=3.9"
12
+ requires-python = ">=3.10"
13
13
  dependencies = [
14
14
  "fastapi",
15
15
  "uvicorn[standard]",
16
- "typing-extensions"
16
+ "typing-extensions",
17
+ "zstandard",
17
18
  ]
18
19
  keywords = ["seamless", "content-addressed", "file server", "HTTP"]
19
20
  classifiers = [
@@ -35,7 +36,7 @@ Repository = "https://github.com/sjdv1982/hashserver"
35
36
  dev = []
36
37
 
37
38
  [tool.setuptools]
38
- py-modules = ["hashserver", "hash_file_response"]
39
+ py-modules = ["hashserver", "hash_file_response", "compression_utils"]
39
40
 
40
41
  [project.scripts]
41
42
  hashserver = "hashserver:main"
@@ -0,0 +1,259 @@
1
+ import gzip
2
+ from hashlib import sha256
3
+ import sys
4
+
5
+ import requests
6
+ import zstandard
7
+
8
+ from .utils import start_server, wait_for_server
9
+
10
+
11
+ BUFFER = (b"compressed payload\n" * 128) + b"tail"
12
+
13
+
14
+ def calculate_checksum(buffer: bytes) -> str:
15
+ return sha256(buffer).digest().hex()
16
+
17
+
18
+ def compress_zstd(buffer: bytes) -> bytes:
19
+ return zstandard.ZstdCompressor().compress(buffer)
20
+
21
+
22
+ def put_buffer(port: int, checksum: str, buffer: bytes, *, content_encoding=None):
23
+ headers = {}
24
+ if content_encoding is not None:
25
+ headers["Content-Encoding"] = content_encoding
26
+ response = requests.put(
27
+ f"http://127.0.0.1:{port}/{checksum}",
28
+ data=buffer,
29
+ headers=headers,
30
+ timeout=10,
31
+ )
32
+ return response
33
+
34
+
35
+ def get_buffer_raw(port: int, checksum: str, *, headers=None):
36
+ response = requests.get(
37
+ f"http://127.0.0.1:{port}/{checksum}",
38
+ stream=True,
39
+ headers=headers,
40
+ timeout=10,
41
+ )
42
+ response.raw.decode_content = False
43
+ body = response.raw.read()
44
+ return response, body
45
+
46
+
47
+ def has_buffers(port: int, checksums):
48
+ response = requests.get(f"http://127.0.0.1:{port}/has", json=checksums, timeout=10)
49
+ return response
50
+
51
+
52
+ def buffer_lengths(port: int, checksums):
53
+ response = requests.get(
54
+ f"http://127.0.0.1:{port}/buffer-length", json=checksums, timeout=10
55
+ )
56
+ return response
57
+
58
+
59
+ def _server_command(write_dir, port):
60
+ env = {
61
+ "HASHSERVER_DIRECTORY": str(write_dir),
62
+ "HASHSERVER_PORT": str(port),
63
+ "HASHSERVER_WRITABLE": "1",
64
+ "HASHSERVER_LAYOUT": "flat",
65
+ }
66
+ command = [
67
+ sys.executable,
68
+ "-m",
69
+ "uvicorn",
70
+ "hashserver:app",
71
+ "--host",
72
+ "127.0.0.1",
73
+ "--port",
74
+ str(port),
75
+ "--log-level",
76
+ "warning",
77
+ ]
78
+ return command, env
79
+
80
+
81
+ def _readonly_server_command(read_dir, port):
82
+ env = {
83
+ "HASHSERVER_DIRECTORY": str(read_dir),
84
+ "HASHSERVER_PORT": str(port),
85
+ "HASHSERVER_LAYOUT": "flat",
86
+ }
87
+ command = [
88
+ sys.executable,
89
+ "-m",
90
+ "uvicorn",
91
+ "hashserver:app",
92
+ "--host",
93
+ "127.0.0.1",
94
+ "--port",
95
+ str(port),
96
+ "--log-level",
97
+ "warning",
98
+ ]
99
+ return command, env
100
+
101
+
102
+ def test_put_zstd_stores_compressed_and_sidecar(tmp_path, available_port):
103
+ write_dir = tmp_path / "writedir"
104
+ write_dir.mkdir()
105
+ checksum = calculate_checksum(BUFFER)
106
+ compressed = compress_zstd(BUFFER)
107
+
108
+ command, env = _server_command(write_dir, available_port)
109
+
110
+ with start_server(command, env=env) as _server:
111
+ wait_for_server(available_port)
112
+
113
+ response = put_buffer(
114
+ available_port,
115
+ checksum,
116
+ compressed,
117
+ content_encoding="zstd",
118
+ )
119
+ assert response.status_code == 200, response.content
120
+
121
+ stored = write_dir / f"{checksum}.zst"
122
+ assert stored.read_bytes() == compressed
123
+ assert (write_dir / f"{checksum}.BUFFERLENGTH").read_text() == str(len(BUFFER))
124
+
125
+
126
+ def test_put_compressed_checksum_mismatch_returns_400(tmp_path, available_port):
127
+ write_dir = tmp_path / "writedir"
128
+ write_dir.mkdir()
129
+ checksum = calculate_checksum(BUFFER)
130
+ compressed = compress_zstd(BUFFER + b"x")
131
+
132
+ command, env = _server_command(write_dir, available_port)
133
+
134
+ with start_server(command, env=env) as _server:
135
+ wait_for_server(available_port)
136
+
137
+ response = put_buffer(
138
+ available_port,
139
+ checksum,
140
+ compressed,
141
+ content_encoding="zstd",
142
+ )
143
+ assert response.status_code == 400, response.content
144
+ assert not (write_dir / f"{checksum}.zst").exists()
145
+ assert not (write_dir / f"{checksum}.BUFFERLENGTH").exists()
146
+
147
+
148
+ def test_put_compressed_and_uncompressed_can_coexist(tmp_path, available_port):
149
+ write_dir = tmp_path / "writedir"
150
+ write_dir.mkdir()
151
+ checksum = calculate_checksum(BUFFER)
152
+ compressed = compress_zstd(BUFFER)
153
+
154
+ command, env = _server_command(write_dir, available_port)
155
+
156
+ with start_server(command, env=env) as _server:
157
+ wait_for_server(available_port)
158
+
159
+ response = put_buffer(
160
+ available_port,
161
+ checksum,
162
+ compressed,
163
+ content_encoding="zstd",
164
+ )
165
+ assert response.status_code == 200, response.content
166
+
167
+ response = put_buffer(available_port, checksum, BUFFER)
168
+ assert response.status_code == 200, response.content
169
+
170
+ assert (write_dir / checksum).read_bytes() == BUFFER
171
+ assert (write_dir / f"{checksum}.zst").read_bytes() == compressed
172
+ assert not (write_dir / f"{checksum}.BUFFERLENGTH").exists()
173
+
174
+
175
+ def test_get_prefers_compressed_when_only_compressed_exists(tmp_path, available_port):
176
+ read_dir = tmp_path / "bufferdir"
177
+ read_dir.mkdir()
178
+ checksum = calculate_checksum(BUFFER)
179
+ compressed = compress_zstd(BUFFER)
180
+ (read_dir / f"{checksum}.zst").write_bytes(compressed)
181
+ (read_dir / f"{checksum}.BUFFERLENGTH").write_text(str(len(BUFFER)))
182
+
183
+ command, env = _readonly_server_command(read_dir, available_port)
184
+
185
+ with start_server(command, env=env) as _server:
186
+ wait_for_server(available_port)
187
+
188
+ response, body = get_buffer_raw(available_port, checksum)
189
+ assert response.status_code == 200, body
190
+ assert response.headers["Content-Encoding"] == "zstd"
191
+ assert body == compressed
192
+
193
+
194
+ def test_get_prefers_uncompressed_when_identity_is_requested(
195
+ tmp_path, available_port
196
+ ):
197
+ read_dir = tmp_path / "bufferdir"
198
+ read_dir.mkdir()
199
+ checksum = calculate_checksum(BUFFER)
200
+ compressed = gzip.compress(BUFFER)
201
+ (read_dir / checksum).write_bytes(BUFFER)
202
+ (read_dir / f"{checksum}.gz").write_bytes(compressed)
203
+
204
+ command, env = _readonly_server_command(read_dir, available_port)
205
+
206
+ with start_server(command, env=env) as _server:
207
+ wait_for_server(available_port)
208
+
209
+ response, body = get_buffer_raw(
210
+ available_port, checksum, headers={"Accept-Encoding": "identity"}
211
+ )
212
+ assert response.status_code == 200, body
213
+ assert "Content-Encoding" not in response.headers
214
+ assert body == BUFFER
215
+
216
+
217
+ def test_get_prefers_requested_encoding_when_available(
218
+ tmp_path, available_port
219
+ ):
220
+ read_dir = tmp_path / "bufferdir"
221
+ read_dir.mkdir()
222
+ checksum = calculate_checksum(BUFFER)
223
+ compressed = gzip.compress(BUFFER)
224
+ (read_dir / checksum).write_bytes(BUFFER)
225
+ (read_dir / f"{checksum}.gz").write_bytes(compressed)
226
+
227
+ command, env = _readonly_server_command(read_dir, available_port)
228
+
229
+ with start_server(command, env=env) as _server:
230
+ wait_for_server(available_port)
231
+
232
+ response, body = get_buffer_raw(
233
+ available_port, checksum, headers={"Accept-Encoding": "gzip"}
234
+ )
235
+ assert response.status_code == 200, body
236
+ assert response.headers["Content-Encoding"] == "gzip"
237
+ assert body == compressed
238
+
239
+
240
+ def test_has_and_buffer_length_recognize_compressed_form(tmp_path, available_port):
241
+ read_dir = tmp_path / "bufferdir"
242
+ read_dir.mkdir()
243
+ checksum = calculate_checksum(BUFFER)
244
+ compressed = compress_zstd(BUFFER)
245
+ (read_dir / f"{checksum}.zst").write_bytes(compressed)
246
+ (read_dir / f"{checksum}.BUFFERLENGTH").write_text(str(len(BUFFER)))
247
+
248
+ command, env = _readonly_server_command(read_dir, available_port)
249
+
250
+ with start_server(command, env=env) as _server:
251
+ wait_for_server(available_port)
252
+
253
+ has_response = has_buffers(available_port, [checksum, "0" * 64])
254
+ assert has_response.status_code == 200, has_response.text
255
+ assert has_response.json() == [True, False]
256
+
257
+ length_response = buffer_lengths(available_port, [checksum])
258
+ assert length_response.status_code == 200, length_response.text
259
+ assert length_response.json() == [len(BUFFER)]
@@ -0,0 +1,21 @@
1
+ import gzip
2
+
3
+ import zstandard
4
+
5
+ from compression_utils import compress_bytes, decompress_bytes
6
+
7
+
8
+ def test_compression_utils_round_trip():
9
+ payloads = [b"", b"hello", b"0123456789" * 10000]
10
+ for suffix in (".zst", ".gz"):
11
+ for payload in payloads:
12
+ assert decompress_bytes(compress_bytes(payload, suffix), suffix) == payload
13
+
14
+
15
+ def test_compression_utils_match_standard_codecs():
16
+ payload = b"payload" * 100
17
+ assert (
18
+ decompress_bytes(zstandard.ZstdCompressor().compress(payload), ".zst")
19
+ == payload
20
+ )
21
+ assert decompress_bytes(gzip.compress(payload), ".gz") == payload
File without changes
File without changes