hashserver 1.0__py3-none-any.whl

hash_file_response.py

@@ -0,0 +1,237 @@
+import os
+import stat
+import time
+import typing
+from hashlib import sha3_256, sha256
+
+import anyio
+
+from starlette.background import BackgroundTask
+from starlette.types import Receive, Scope, Send
+from starlette.responses import FileResponse
+
+HASH_ALGORITHMS = {
+    "sha3-256": sha3_256,
+    "sha-256": sha256,
+}
+DEFAULT_HASH_ALGORITHM = "sha-256"
+_current_hash_algorithm = DEFAULT_HASH_ALGORITHM
+_hash_constructor = HASH_ALGORITHMS[DEFAULT_HASH_ALGORITHM]
+
+
+def set_hash_algorithm(algorithm: str) -> None:
+    global _current_hash_algorithm, _hash_constructor
+    try:
+        _hash_constructor = HASH_ALGORITHMS[algorithm]
+    except KeyError as exc:
+        raise ValueError(
+            f"Unsupported hash algorithm '{algorithm}'. "
+            f"Choose one of: {', '.join(HASH_ALGORITHMS)}"
+        ) from exc
+    _current_hash_algorithm = algorithm
+
+
+def get_hash_algorithm() -> str:
+    return _current_hash_algorithm
+
+
+def parse_checksum(checksum) -> str:
+    """Parses checksum and returns it as string.
+
+    Adapted from the Seamless source code (fair use)"""
+    if isinstance(checksum, bytes):
+        checksum = checksum.hex()
+    if isinstance(checksum, str):
+        if len(checksum) % 2:
+            raise ValueError("Wrong length")
+        checksum = bytes.fromhex(checksum)
+
+    if isinstance(checksum, bytes):
+        if len(checksum) != 32:
+            raise ValueError("Wrong length")
+        return checksum.hex()
+
+    if checksum is None:
+        return
+    raise TypeError(type(checksum))
+
+
+class HashFileResponse(FileResponse):
+    """FileResponse that validates files against their checksum-derived filename."""
+
+    _PREFIX = False
+
+    lock_timeout = 120
+    chunk_size = 640 * 1024
+
+    def __init__(
+        self,
+        checksum: str,
+        directory: str,
+        status_code: int = 200,
+        headers: typing.Optional[typing.Mapping[str, str]] = None,
+        media_type: typing.Optional[str] = None,
+        background: typing.Optional[BackgroundTask] = None,
+        stat_result: typing.Optional[os.stat_result] = None,
+        method: typing.Optional[str] = None,
+        content_disposition_type: str = "attachment",
+        extra_dirs: typing.Optional[typing.List[str]] = None,
+    ) -> None:
+        filename = parse_checksum(checksum)
+        self.prefix = filename[:2]
+        stat_result = None
+        if self._PREFIX:
+            path = os.path.join(directory, self.prefix, filename)
+        else:
+            path = os.path.join(directory, filename)
+        super().__init__(
+            path=path,
+            status_code=status_code,
+            headers=headers,
+            media_type=media_type,
+            background=background,
+            filename=filename,
+            stat_result=stat_result,
+            method=method,
+            content_disposition_type=content_disposition_type,
+        )
+        self.directory = directory
+        self.extra_dirs = extra_dirs
+        extra_dirs_layout = {}
+        for extra_dir in extra_dirs:
+            prefix_file = os.path.join(extra_dir, ".HASHSERVER_PREFIX")
+            if os.path.exists(prefix_file):
+                layout = "prefix"
+            else:
+                layout = "flat"
+            extra_dirs_layout[extra_dir] = layout
+        self.extra_dirs_layout = extra_dirs_layout
+
+    async def refresh_stat_headers(self):
+        if self.extra_dirs and not await anyio.Path(self.path).exists():
+            for extra_dir in self.extra_dirs:
+                layout = self.extra_dirs_layout[extra_dir]
+                if layout == "prefix":
+                    path0 = os.path.join(extra_dir, self.prefix, self.filename)
+                else:
+                    path0 = os.path.join(extra_dir, self.filename)
+                if await anyio.Path(path0).exists():
+                    self.path = path0
+                    break
+
+        try:
+            stat_result = await anyio.to_thread.run_sync(os.stat, self.path)
+            del self.headers["content-length"]
+            del self.headers["last-modified"]
+            del self.headers["etag"]
+
+            self.set_stat_headers(stat_result)
+        except FileNotFoundError:
+            raise FileNotFoundError(
+                f"File at path {self.path} does not exist."
+            ) from None
+        else:
+            mode = stat_result.st_mode
+            if not stat.S_ISREG(mode):
+                raise RuntimeError(f"File at path {self.path} is not a file.")
+        return stat_result
+
+    async def _until_no_lock(self, lockpaths):
+        for lockpath in lockpaths:
+            while 1:
+                try:
+                    lock_stat_result = await anyio.to_thread.run_sync(os.stat, lockpath)
+                except FileNotFoundError:
+                    break
+                lock_mtime = lock_stat_result.st_mtime
+                if time.time() - lock_mtime > self.lock_timeout:
+                    break
+                await anyio.sleep(1)
+
+    async def until_no_lock(self):
+        lockpaths = [os.path.join(self.directory, ".LOCK")]
+        if self.path is not None:
+            lockpaths.append(self.path + ".LOCK")
+        return await self._until_no_lock(lockpaths)
+
+    async def calculate_checksum(self):
+        """Return checksum for the configured algorithm."""
+        checksum = _hash_constructor()
+        async with await anyio.open_file(self.path, mode="rb") as file:
+            more_body = True
+            while more_body:
+                chunk = await file.read(self.chunk_size)
+                checksum.update(chunk)
+                more_body = len(chunk) == self.chunk_size
+
+            checksum = checksum.digest().hex()
+        return checksum
+
+    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+        if self.stat_result is None:
+            try:
+                stat_result = await self.refresh_stat_headers()
+            except FileNotFoundError:
+                await self.until_no_lock()
+                stat_result = await self.refresh_stat_headers()
+            self.stat_result = stat_result
+
+        checksum = await self.calculate_checksum()
+        if checksum != self.filename:
+            await self.until_no_lock()
+            stat_result = await self.refresh_stat_headers()
+            self.stat_result = stat_result
+            checksum2 = await self.calculate_checksum()
+            if checksum2 != self.filename:
+                raise RuntimeError(
+                    f"File corruption: file at path {self.path} does not have the correct {_current_hash_algorithm} checksum."
+                )
+
+        await super().__call__(scope=scope, receive=receive, send=send)
+
+
+class PrefixHashFileResponse(HashFileResponse):
+    """Same as HashFileResponse but files are stored under a two-character prefix.
+
+    File has the same name as checksum.
+    File is stored as $PREFIX/$CHECKSUM, where $PREFIX is the first two
+      characters of $CHECKSUM
+    """
+
+    _PREFIX = True
+
+    def __init__(
+        self,
+        checksum: str,
+        directory: str,
+        status_code: int = 200,
+        headers: typing.Optional[typing.Mapping[str, str]] = None,
+        media_type: typing.Optional[str] = None,
+        background: typing.Optional[BackgroundTask] = None,
+        stat_result: typing.Optional[os.stat_result] = None,
+        method: typing.Optional[str] = None,
+        content_disposition_type: str = "attachment",
+        extra_dirs: typing.Optional[typing.List[str]] = None,
+    ) -> None:
+
+        super().__init__(
+            checksum=checksum,
+            directory=directory,
+            status_code=status_code,
+            headers=headers,
+            media_type=media_type,
+            background=background,
+            stat_result=stat_result,
+            method=method,
+            content_disposition_type=content_disposition_type,
+            extra_dirs=extra_dirs,
+        )
+        prefix_file = os.path.join(directory, ".HASHSERVER_PREFIX")
+        with open(prefix_file, mode="wb") as f:
+            f.write(b"1\n")
+
+    async def until_no_lock(self):
+        lockpaths = [os.path.join(self.directory, self.prefix, ".LOCK")]
+        if self.path is not None:
+            lockpaths.append(self.path + ".LOCK")
+        return await self._until_no_lock(lockpaths)

hashserver-1.0.dist-info/METADATA

@@ -0,0 +1,174 @@
+Metadata-Version: 2.4
+Name: hashserver
+Version: 1.0
+Summary: Simple FastAPI-based hash server
+Author: Sjoerd de Vries
+License-Expression: MIT
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE.txt
+Requires-Dist: fastapi
+Requires-Dist: uvicorn[standard]
+Requires-Dist: typing-extensions
+Dynamic: license-file
+
+# Hashserver
+
+A lightweight, content-addressed file server over HTTP.
+
+Hashserver stores and serves opaque binary buffers keyed by their cryptographic checksum. You PUT a buffer with its checksum in the URL; you GET it back by the same checksum. There are no filenames, no directories, no metadata — just content and its hash.
+
+The hash algorithm is configurable: SHA-256 (default) or SHA3-256.
+
+## Why content-addressed storage?
+
+Content-addressed storage (CAS) is a well-established pattern used by Git, IPFS, Docker registries, and many other systems. Identifying data by its cryptographic hash gives you automatic deduplication, trivially verifiable integrity, and strong reproducibility guarantees.
+
+Hashserver brings these benefits to any project that needs a simple HTTP-based buffer store. It is intentionally minimal: a single ASGI application backed by a directory of files, designed to be easy to deploy, easy to integrate, and easy to reason about.
+
+## Relationship to Seamless
+
+Hashserver was originally developed as the buffer-serving component of [Seamless](https://github.com/sjdv1982/seamless), a framework for reproducible, reactive computational workflows. In Seamless, all data — inputs, source code, and results — is represented as a tree of checksums, and hashserver provides the storage layer that maps those checksums back to actual data.
+
+However, **hashserver has no dependency on Seamless** and no knowledge of it. It is a generic content-addressed file server that is useful in any context where you need to store and retrieve buffers by hash — caching layers, artifact stores, reproducible pipelines, or your own CAS-backed application. It is published as an independent PyPI package for exactly this reason.
+
+## Features
+
+- **Content-addressed**: buffers are stored and retrieved by their cryptographic checksum.
+- **Configurable hash algorithm**: SHA-256 (default) or SHA3-256, selected at startup.
+- **Integrity-verified reads**: every buffer is re-checksummed on GET to detect corruption.
+- **Prefix directory layout**: by default, buffers are stored under a two-character prefix subdirectory (e.g. `ab/ab3f7c...`) to avoid filesystem performance problems with large flat directories. A flat layout is also supported.
+- **Extra read-only directories**: additional buffer directories can be mounted as fallback read sources.
+- **Promises**: a client can announce that a buffer will be uploaded soon via `PUT /promise/{checksum}`. Other clients reading that checksum will wait for the upload rather than getting a 404.
+- **Concurrent-safe**: in-flight PUT requests are tracked so concurrent GETs and batch queries return consistent results. Lock files are respected for external writers.
+- **Multiple instances**: several hashserver processes can safely share the same buffer directory.
+- **Lightweight**: built on FastAPI/Starlette — no database, no external services.
+- **Flexible deployment**: run as a CLI tool, under any ASGI server, or via Docker Compose.
+
+## Installation
+
+```bash
+pip install hashserver
+```
+
+Or with conda:
+
+```bash
+mamba env create --file environment.yml
+conda activate hashserver
+```
+
+## Quick start
+
+Serve buffers from a local directory:
+
+```bash
+hashserver ./my-buffers
+```
+
+This starts the server under uvicorn on port 8000. Run `hashserver -h` for all options.
+
+### Storing and retrieving a buffer
+
+```bash
+# Start a writable server
+hashserver ./my-buffers --writable
+
+# Compute the SHA-256 checksum and upload
+CHECKSUM=$(python3 -c "
+import hashlib, sys
+print(hashlib.sha256(open(sys.argv[1],'rb').read()).hexdigest())
+" myfile.bin)
+curl -X PUT --data-binary @myfile.bin http://localhost:8000/$CHECKSUM
+
+# Download
+curl -O http://localhost:8000/$CHECKSUM
+```
+
+To use SHA3-256 instead, start the server with `--hash-algorithm sha3-256` and hash your files with `hashlib.sha3_256`.
+
+## API
+
+### Retrieving buffers
+
+**`GET /{checksum}`** — Retrieve a buffer by its hex checksum. The server verifies the checksum before sending the response. Returns the raw buffer (200), or 404 if not found.
+
+### Storing buffers
+
+Requires `--writable`.
+
+**`PUT /{checksum}`** — Upload a buffer. The request body is the raw data; the server verifies that its checksum matches the URL. Returns 200 on success, 201 if the buffer already existed, or 400 on checksum mismatch.
+
+**`PUT /promise/{checksum}`** — Announce that a buffer will be uploaded soon. Returns 202 with the promise TTL. While a promise is active, GET requests for that checksum will wait rather than returning 404, and `/has` queries will report the checksum as present.
+
+### Querying availability
+
+**`GET /has`** — Batch existence check. Send a JSON list of checksums in the request body. Returns a JSON list of booleans. Includes both on-disk buffers and active promises.
+
+**`GET /has-now`** — Same as `/has`, but excludes promises — only reports buffers that are already on disk.
+
+**`GET /buffer-length`** — Batch size query. Send a JSON list of checksums in the request body. Returns a JSON list of integers: the buffer size in bytes, or 0 if not present. Promised checksums are reported as `true`.
+
+### Health
+
+**`GET /healthcheck`** — Returns "OK". Useful for load balancer probes.
+
+## Configuration
+
+### CLI flags
+
+| Flag | Description | Default |
+|------|-------------|---------|
+| `directory` | Buffer storage directory (positional, required) | — |
+| `--writable` | Enable PUT endpoints | off |
+| `--hash-algorithm` | Hash algorithm: `sha3-256` or `sha-256` | `sha-256` |
+| `--layout` | Directory layout: `prefix` or `flat` | `prefix` |
+| `--extra-dirs` | Semicolon-separated list of extra read-only buffer directories | — |
+| `--host` | Listen address | `127.0.0.1` |
+| `--port` | Listen port | `8000` |
+| `--port-range START END` | Pick a random free port in range (mutually exclusive with `--port`) | — |
+| `--status-file` | JSON file for reporting server status | — |
+| `--timeout` | Shut down after this many seconds of inactivity | — |
+
+### Environment variables
+
+When running under an external ASGI server (e.g. `uvicorn hashserver:app`), configure via environment variables instead:
+
+| Variable | Equivalent flag |
+|----------|----------------|
+| `HASHSERVER_DIRECTORY` | `directory` |
+| `HASHSERVER_WRITABLE` | `--writable` (set to `1` or `true`) |
+| `HASHSERVER_HASH_ALGORITHM` | `--hash-algorithm` |
+| `HASHSERVER_LAYOUT` | `--layout` |
+| `HASHSERVER_EXTRA_DIRS` | `--extra-dirs` |
+
+### Docker Compose
+
+```bash
+export HASHSERVER_PORT=8000
+export HASHSERVER_HOST=0.0.0.0
+export HASHSERVER_DIRECTORY=./buffers
+export HASHSERVER_WRITABLE=1
+docker compose up -d
+```
+
+Container user/group ID can be set with `HASHSERVER_USER_ID` and `HASHSERVER_GROUP_ID` (both default to 0).
+
+## Directory layouts
+
+In **prefix** layout (the default), a buffer with checksum `ab3f7c...` is stored as `<directory>/ab/ab3f7c...`. A sentinel file `.HASHSERVER_PREFIX` is written to the directory. This avoids performance issues when storing large numbers of buffers.
+
+In **flat** layout, the same buffer is stored as `<directory>/ab3f7c...`.
+
+Extra directories auto-detect their layout by checking for the `.HASHSERVER_PREFIX` sentinel.
+
+## Running tests
+
+```bash
+pip install requests
+pytest tests/
+```
+
+## License
+
+See [LICENSE.txt](LICENSE.txt).

hashserver-1.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+hash_file_response.py,sha256=3H5snJYz8FWiIfIugI5i-zQPhgF0jUVjr2GDRrX7DX8,8247
+hashserver.py,sha256=LDAYpZI7plcT6I5Lpe-nfFQwC3T54u-Z2-by2vPpBEY,26715
+hashserver-1.0.dist-info/licenses/LICENSE.txt,sha256=aFUunT7WYX_fR7ryljRBKqoltt4dSYe-PB15Hz9GeyA,1117
+hashserver-1.0.dist-info/METADATA,sha256=xc1WzqyX1Bb6Y7IlU9PIVUiLyz7_AYf4IPVtGVZTa3Y,7630
+hashserver-1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+hashserver-1.0.dist-info/entry_points.txt,sha256=APGs23yr75suYAn6nQFAmAUL0GC1MARS6SFntYzhdvo,47
+hashserver-1.0.dist-info/top_level.txt,sha256=oBgEDscAxsuQKVjeI30QOwRavz0iM5MLWh__ffn3mB0,30
+hashserver-1.0.dist-info/RECORD,,

hashserver-1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

hashserver-1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+hashserver = hashserver:main

hashserver-1.0.dist-info/licenses/LICENSE.txt

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Author: Sjoerd de Vries, MBI platform.
+Copyright (c) 2023-2026 CNRS. 
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

hashserver-1.0.dist-info/top_level.txt

@@ -0,0 +1,2 @@
+hash_file_response
+hashserver

hashserver.py

@@ -0,0 +1,865 @@
+import os
+import sys
+import argparse
+import random
+import socket
+import json
+import asyncio
+import logging
+import aiofiles
+import aiofiles.os
+import aiofiles.tempfile
+import contextlib
+import copy
+from dataclasses import dataclass
+from typing import Iterable, List, Optional, Set, Union
+from fastapi import FastAPI, Path, Body, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import Response, JSONResponse
+from fastapi.exceptions import RequestValidationError
+from fastapi.encoders import jsonable_encoder
+from starlette.requests import ClientDisconnect
+
+from functools import partial
+
+from hash_file_response import (
+    parse_checksum,
+    HashFileResponse,
+    PrefixHashFileResponse,
+    HASH_ALGORITHMS,
+    DEFAULT_HASH_ALGORITHM,
+    set_hash_algorithm,
+)
+
+from typing_extensions import Annotated
+from pydantic.functional_validators import BeforeValidator
+
+import anyio
+import pathlib
+import time
+
+Checksum = Annotated[str, BeforeValidator(parse_checksum)]
+
+checksum_constructor = HASH_ALGORITHMS[DEFAULT_HASH_ALGORITHM]
+
+STATUS_FILE_WAIT_TIMEOUT = 20.0
+INACTIVITY_CHECK_INTERVAL = 1.0
+
+
+INACTIVITY_STATE = {
+    "timeout": None,
+    "last_request": None,
+    "task": None,
+    "server": None,
+}
+
+aiofiles_chmod = aiofiles.os.wrap(os.chmod)  # aiofiles.os lacks chmod
+
+
+def calculate_checksum(buffer):
+    """Return checksum in the configured hash algorithm."""
+    return checksum_constructor(buffer).digest().hex()
+
+
+def calculate_checksum_stream():
+    return checksum_constructor()
+
+
+def wait_for_status_file(path: str, timeout: float = STATUS_FILE_WAIT_TIMEOUT):
+    deadline = time.monotonic() + timeout
+    while True:
+        try:
+            with open(path, "r", encoding="utf-8") as status_stream:
+                contents = json.load(status_stream)
+                break
+        except FileNotFoundError:
+            if time.monotonic() >= deadline:
+                print(
+                    f"Status file '{path}' not found after {int(timeout)} seconds",
+                    file=sys.stderr,
+                )
+                sys.exit(1)
+            time.sleep(0.1)
+            continue
+        except json.JSONDecodeError as exc:
+            print(
+                f"Status file '{path}' is not valid JSON: {exc}",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+
+    if not isinstance(contents, dict):
+        print(
+            f"Status file '{path}' must contain a JSON object",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    return contents
+
+
+class StatusFileTracker:
+    def __init__(self, path: str, base_contents: dict, port: int):
+        self.path = path
+        self._base_contents = dict(base_contents)
+        self.port = port
+        self.running_written = False
+
+    def _write(self, payload: dict):
+        tmp_path = f"{self.path}.tmp"
+        with open(tmp_path, "w", encoding="utf-8") as status_stream:
+            json.dump(payload, status_stream)
+            status_stream.write("\n")
+        os.replace(tmp_path, self.path)
+
+    def write_running(self):
+        payload = dict(self._base_contents)
+        payload["port"] = self.port
+        payload["status"] = "running"
+        self._write(payload)
+        self._base_contents = payload
+        self.running_written = True
+
+    def write_failed(self):
+        payload = dict(self._base_contents)
+        payload["status"] = "failed"
+        self._write(payload)
+
+
+def raise_startup_error(exc: BaseException):
+    if status_tracker and not status_tracker.running_written:
+        status_tracker.write_failed()
+    raise exc
+
+
+def configure_hash_algorithm(algorithm: str):
+    global checksum_constructor
+    try:
+        checksum_constructor = HASH_ALGORITHMS[algorithm]
+    except KeyError:
+        raise_startup_error(
+            RuntimeError(
+                f"--hash-algorithm must be one of: {', '.join(HASH_ALGORITHMS.keys())}"
+            )
+        )
+    set_hash_algorithm(algorithm)
+
+
+def configure_lock_timeout(lock_timeout_seconds: float):
+    if lock_timeout_seconds <= 0:
+        raise_startup_error(RuntimeError("--lock-timeout must be a positive number"))
+    HashFileResponse.lock_timeout = lock_timeout_seconds
+    PrefixHashFileResponse.lock_timeout = lock_timeout_seconds
+
+
+def setup_inactivity_timeout(timeout_seconds: float, server):
+    INACTIVITY_STATE["timeout"] = timeout_seconds
+    INACTIVITY_STATE["server"] = server
+
+    async def monitor_inactivity():
+        try:
+            while True:
+                await asyncio.sleep(INACTIVITY_CHECK_INTERVAL)
+                last_request = INACTIVITY_STATE.get("last_request")
+                if last_request is None:
+                    continue
+                if time.monotonic() - last_request >= timeout_seconds:
+                    server.should_exit = True
+                    break
+        except asyncio.CancelledError:
+            raise
+
+    async def start_monitor():
+        INACTIVITY_STATE["last_request"] = time.monotonic()
+        loop = asyncio.get_running_loop()
+        INACTIVITY_STATE["task"] = loop.create_task(monitor_inactivity())
+
+    async def stop_monitor():
+        task = INACTIVITY_STATE.get("task")
+        if task:
+            task.cancel()
+            with contextlib.suppress(asyncio.CancelledError):
+                await task
+            INACTIVITY_STATE["task"] = None
+        INACTIVITY_STATE["last_request"] = None
+        INACTIVITY_STATE["server"] = None
+        INACTIVITY_STATE["timeout"] = None
+
+    app.add_event_handler("startup", start_monitor)
+    app.add_event_handler("shutdown", stop_monitor)
+
+
+def pick_random_free_port(host: str, start: int, end: int) -> int:
+    if start < 0 or end > 65535:
+        raise RuntimeError("--port-range values must be between 0 and 65535")
+    if start > end:
+        raise RuntimeError("--port-range START must be less than or equal to END")
+
+    span = end - start + 1
+    attempted = set()
+    while len(attempted) < span:
+        port = random.randint(start, end)
+        if port in attempted:
+            continue
+        attempted.add(port)
+        try:
+            with socket.create_server((host, port), reuse_port=False):
+                pass
+        except OSError:
+            continue
+        return port
+
+    raise RuntimeError(f"No free port available in range {start}-{end}")
+
+
+DEFAULT_LOCK_TIMEOUT = 120.0
+CHUNK_SIZE = 640 * 1024  # for now, hardcoded
+PROMISE_TTL_SECONDS = 10 * 60.0
+
+env = os.environ
+as_commandline_tool = True
+status_tracker = None
+status_file_path = None
+status_file_contents = None
+timeout_seconds = None
+
+if "HASHSERVER_DIRECTORY" in os.environ:
+    directory = os.environ["HASHSERVER_DIRECTORY"]
+    writable = False
+    if "HASHSERVER_WRITABLE" in os.environ:
+        env_writable = os.environ["HASHSERVER_WRITABLE"]
+        assert env_writable.lower() in ("true", "false", "0", "1", ""), env_writable
+        if env_writable.lower() in ("true", "1"):
+            writable = True
+    as_commandline_tool = False
+
+    extra_dirs: list[str] = []
+    extra_dirs0 = os.environ.get("HASHSERVER_EXTRA_DIRS")
+    if extra_dirs0:
+
+        def _filt(d):
+            d = d.strip()
+            if d == '""' or d == "''":
+                return ""
+
+        extra_dirs00 = [_filt(d) for d in extra_dirs0.split(";")]
+        extra_dirs = [d for d in extra_dirs00 if d]
+
+    layout = os.environ.get("HASHSERVER_LAYOUT", "prefix")
+    status_file_path = None
+    status_file_contents = None
+    timeout_seconds = None
+    algorithm = os.environ.get("HASHSERVER_HASH_ALGORITHM", DEFAULT_HASH_ALGORITHM)
+    configure_hash_algorithm(algorithm)
+    lock_timeout = os.environ.get("HASHSERVER_LOCK_TIMEOUT")
+    if lock_timeout is not None:
+        try:
+            configure_lock_timeout(float(lock_timeout))
+        except ValueError:
+            raise_startup_error(
+                RuntimeError("HASHSERVER_LOCK_TIMEOUT must be a number")
+            )
+
+else:
+    if (
+        len(sys.argv)
+        and sys.argv[0].find("uvicorn") > -1
+        and not os.path.isdir(sys.argv[0])
+    ):
+        print(
+            "Running hashserver under uvicorn CLI requires at least HASHSERVER_DIRECTORY to be defined",
+            file=sys.stderr,
+        )
+        exit(1)
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "directory",
+        help="""Directory where buffers are located.
+
+    Buffers have the same file name as their checksum (sha3-256 by default).""",
+    )
+    parser.add_argument(
+        "--extra-dirs",
+        help="""Extra directories where read-only buffers are located.
+
+This must be a list of directories separated by semi-colons (;).
+If not specified, this argument is read from HASHSERVER_EXTRA_DIRS, if present""",
+    )
+
+    parser.add_argument(
+        "--writable",
+        action="store_true",
+        help="Allow HTTP PUT requests",
+    )
+
+    port_group = parser.add_mutually_exclusive_group()
+    port_group.add_argument(
+        "--port",
+        type=int,
+        help="Network port",
+    )
+    port_group.add_argument(
+        "--port-range",
+        type=int,
+        nargs=2,
+        metavar=("START", "END"),
+        help="Inclusive port range to select a random free port from",
+    )
+
+    parser.add_argument(
+        "--host",
+        type=str,
+        help="Network host",
+        default="127.0.0.1",
+    )
+
+    parser.add_argument(
+        "--layout",
+        type=str,
+        help="""Directory layout.
+        One of:
+        - "flat". 
+        A buffer with checksum CS is stored as file "$DIRECTORY/$CS".
+
+        - "prefix". 
+        A buffer with checksum CS is stored as file "$DIRECTORY/$PREFIX/$CS",
+        where PREFIX is the first two characters of CS.
+                
+        """,
+        default="prefix",
+    )
+
+    parser.add_argument(
+        "--hash-algorithm",
+        type=str,
+        choices=tuple(HASH_ALGORITHMS.keys()),
+        default=DEFAULT_HASH_ALGORITHM,
+        help="Hash algorithm used for checksum calculations (default: %(default)s)",
+    )
+    parser.add_argument(
+        "--lock-timeout",
+        type=float,
+        default=DEFAULT_LOCK_TIMEOUT,
+        help="Wait this many seconds for stale lock files (default: %(default)s)",
+    )
+
+    parser.add_argument(
+        "--status-file",
+        type=str,
+        help="JSON file used to report server status",
+    )
+
+    parser.add_argument(
+        "--timeout",
+        type=float,
+        help="Stop the server after this many seconds of inactivity",
+    )
+
+    args = parser.parse_args()
+    directory = args.directory
+    writable = args.writable
+    extra_dirs = args.extra_dirs
+    configure_hash_algorithm(args.hash_algorithm)
+    configure_lock_timeout(args.lock_timeout)
+    status_file_path = args.status_file
+    timeout_seconds = args.timeout
+    if status_file_path:
+        status_file_contents = wait_for_status_file(status_file_path)
+        status_tracker = StatusFileTracker(
+            status_file_path, status_file_contents, args.port
+        )
+    if timeout_seconds is not None and timeout_seconds <= 0:
+        raise_startup_error(RuntimeError("--timeout must be a positive number"))
+    if not extra_dirs:
+        extra_dirs = os.environ.get("HASHSERVER_EXTRA_DIRS")
+    if extra_dirs:
+        extra_dirs = [d.strip() for d in extra_dirs.split(";")]
+    else:
+        extra_dirs = []
+    layout = args.layout
+    if args.port_range:
+        start, end = args.port_range
+        try:
+            selected_port = pick_random_free_port(args.host, start, end)
+        except BaseException as exc:
+            raise_startup_error(exc)
+    else:
+        selected_port = args.port if args.port is not None else 8000
+    args.port = selected_port
+    if status_tracker:
+        status_tracker.port = selected_port
+
+
+if not os.path.exists(directory):
+    raise_startup_error(FileExistsError(f"Directory '{directory}' does not exist"))
+if not os.path.isdir(directory):
+    raise_startup_error(FileExistsError(f"Directory '{directory}' is not a directory"))
+
+try:
+    os.chmod(directory, 0o3775)
+except Exception:
+    pass
+
+if layout not in ("flat", "prefix"):
+    raise_startup_error(RuntimeError("Layout must be 'flat' or 'prefix'"))
+
+if layout == "prefix":
+    prefix_file = os.path.join(directory, ".HASHSERVER_PREFIX")
+    try:
+        if not os.path.exists(prefix_file):
+            with open(prefix_file, "wb") as f:
+                f.write(b"1\n")
+    except Exception:
+        pass
+
+app = FastAPI()
+LOGGER = logging.getLogger("hashserver")
+
+
+@app.exception_handler(RequestValidationError)
+async def validation_exception_handler(request, exc):
+    inner_exc = exc.args[0][0]
+    inner_exc = jsonable_encoder(inner_exc)
+    inner_exc.pop("ctx", None)
+    return JSONResponse(
+        status_code=400,
+        content={"message": "Invalid data", "exception": inner_exc},
+    )
+
+
+@app.exception_handler(FileNotFoundError)
+async def filenotfound_exception_handler(request, exc):
+    return Response(status_code=404, content="Not found")
+
+
+@app.exception_handler(RuntimeError)
+async def runtime_exception_handler(request, exc):
+    return JSONResponse(
+        status_code=400,
+        content={"message": f"{exc}"},
+    )
+
+
+@app.middleware("http")
+async def record_last_request(request: Request, call_next):
+    if INACTIVITY_STATE["timeout"] is not None:
+        INACTIVITY_STATE["last_request"] = time.monotonic()
+    response = await call_next(request)
+    if INACTIVITY_STATE["timeout"] is not None:
+        INACTIVITY_STATE["last_request"] = time.monotonic()
+    return response
+
+
+@app.get("/buffer-length")
+async def buffer_length(checksums: Annotated[List[Checksum], Body()]) -> JSONResponse:
+    checksums2 = [parse_checksum(checksum) for checksum in checksums]
+    await _wait_for_current_put_requests(checksums2)
+    curr_results = [0] * len(checksums)
+
+    async def stat_all(paths):
+        futures = []
+        for _, path in paths:
+            fut = anyio.Path(path).stat()
+            futures.append(fut)
+        result0 = await asyncio.gather(*futures, return_exceptions=True)
+        for (nr, path), stat in zip(paths, result0):
+            if isinstance(stat, Exception):
+                continue
+            curr_results[nr] = stat.st_size
+
+    paths = []
+    for nr, checksum in enumerate(checksums2):
+        assert isinstance(checksum, str)
+        if layout == "prefix":
+            prefix = checksum[:2]
+            path = os.path.join(directory, prefix, checksum)
+        else:
+            path = os.path.join(directory, checksum)
+        paths.append((nr, path))
+
+    await stat_all(paths)
+
+    for extra_dir in extra_dirs:
+        for nr, checksum in enumerate(checksums2):
+            if curr_results[nr]:
+                continue
+            path = os.path.join(extra_dir, checksum)
+            paths.append((nr, path))
+        if not len(paths):
+            break
+        await stat_all(paths)
+
+    promised = await _promise_registry.promised_indices(checksums2)
+    for idx in promised:
+        curr_results[idx] = True
+
+    return curr_results
+
+
+async def _has(checksums: List[Checksum], include_promises: bool) -> List[bool]:
+    checksums2 = [parse_checksum(checksum) for checksum in checksums]
+    curr_results = [False] * len(checksums)
+
+    # Flag any in-flight uploads immediately.
+    for idx, checksum in enumerate(checksums2):
+        if checksum in _current_put_requests:
+            curr_results[idx] = True
+
+    async def exists_all(paths):
+        futures = []
+        for _, path in paths:
+            fut = anyio.Path(path).exists()
+            futures.append(fut)
+        result0 = await asyncio.gather(*futures, return_exceptions=True)
+        for (nr, path), exists in zip(paths, result0):
+            if isinstance(exists, Exception):
+                continue
+            if exists:
+                curr_results[nr] = True
+
+    paths = []
+    for nr, checksum in enumerate(checksums2):
+        assert isinstance(checksum, str)
+        if curr_results[nr]:
+            continue
+        if layout == "prefix":
+            prefix = checksum[:2]
+            path = os.path.join(directory, prefix, checksum)
+        else:
+            path = os.path.join(directory, checksum)
+        paths.append((nr, path))
+
+    if paths:
+        await exists_all(paths)
+
+    for extra_dir in extra_dirs:
+        paths = []
+        for nr, checksum in enumerate(checksums2):
+            if curr_results[nr]:
+                continue
+            path = os.path.join(extra_dir, checksum)
+            paths.append((nr, path))
+        if not paths:
+            break
+        await exists_all(paths)
+
+    if include_promises:
+        promised = await _promise_registry.promised_indices(checksums2)
+        for idx in promised:
+            curr_results[idx] = True
+
+    return curr_results
+
+
+@app.get("/has")
+async def has(checksums: Annotated[List[Checksum], Body()]) -> JSONResponse:
+    return await _has(checksums, include_promises=True)
+
+
+@app.get("/has-now")
+async def has(checksums: Annotated[List[Checksum], Body()]) -> JSONResponse:
+    return await _has(checksums, include_promises=False)
+
+
+class PromiseAwareResponseMixin:
+    def __init__(self, *, checksum: str, **kwargs):
+        self._promise_checksum = checksum
+        super().__init__(checksum=checksum, **kwargs)
+
+    async def __call__(self, scope, receive, send):
+        while True:
+            try:
+                await super().__call__(scope, receive, send)
+                return
+            except FileNotFoundError:
+                should_retry = await _promise_registry.wait_for(self._promise_checksum)
+                if not should_retry:
+                    raise
+
+
+class PromiseAwareHashFileResponse(PromiseAwareResponseMixin, HashFileResponse):
+    pass
+
+
+class PromiseAwarePrefixHashFileResponse(
+    PromiseAwareResponseMixin, PrefixHashFileResponse
+):
+    pass
+
+
+_response_classes_get_file = {
+    "flat": PromiseAwareHashFileResponse,
+    "prefix": PromiseAwarePrefixHashFileResponse,
+}
+
+
+@app.get("/healthcheck")
+async def healthcheck() -> Response:
+    return Response(content="OK")
+
+
+@app.get("/{checksum}")
+async def get_file(checksum: Annotated[Checksum, Path()]) -> HashFileResponse:
+    checksum2 = parse_checksum(checksum)
+    LOGGER.info("GET %s", checksum2)
+    await _wait_for_current_put_requests((checksum2,))
+    ResponseClass = _response_classes_get_file[layout]
+    response = ResponseClass(
+        directory=directory, checksum=checksum2, extra_dirs=extra_dirs
+    )
+    return response
+
+
+async def promise(checksum: Annotated[Checksum, Path()]) -> JSONResponse:
+    checksum2 = parse_checksum(checksum)
+    await _promise_registry.add(checksum2)
+    return JSONResponse(
+        status_code=202,
+        content={"checksum": checksum2, "expires_in": PROMISE_TTL_SECONDS},
+    )
+
+
+_current_put_requests: set[str] = set()
+_current_put_condition = asyncio.Condition()
+
+
+@dataclass
+class _PromiseEntry:
+    event: asyncio.Event
+    expires_at: float
+
+
+class PromiseRegistry:
+    def __init__(self, ttl_seconds: float = PROMISE_TTL_SECONDS):
+        self._ttl_seconds = ttl_seconds
+        self._promises: dict[str, _PromiseEntry] = {}
+        self._lock = asyncio.Lock()
+
+    def _cleanup_locked(self, now: Optional[float] = None) -> None:
+        if now is None:
+            now = time.monotonic()
+        expired = [
+            cs for cs, entry in self._promises.items() if entry.expires_at <= now
+        ]
+        for checksum in expired:
+            self._promises.pop(checksum, None)
+
+    async def add(self, checksum: str) -> float:
+        now = time.monotonic()
+        expires_at = now + self._ttl_seconds
+        async with self._lock:
+            self._cleanup_locked(now)
+            entry = self._promises.get(checksum)
+            if entry is None:
+                entry = _PromiseEntry(asyncio.Event(), expires_at)
+                self._promises[checksum] = entry
+            else:
+                entry.expires_at = expires_at
+        return expires_at
+
+    async def resolve(self, checksum: str) -> None:
+        async with self._lock:
+            entry = self._promises.pop(checksum, None)
+        if entry:
+            entry.event.set()
+
+    async def promised_indices(self, checksums: List[str]) -> Set[int]:
+        async with self._lock:
+            self._cleanup_locked()
+            promised = {idx for idx, cs in enumerate(checksums) if cs in self._promises}
+        return promised
+
+    async def wait_for(self, checksum: str) -> bool:
+        while True:
+            async with self._lock:
+                self._cleanup_locked()
+                entry = self._promises.get(checksum)
+                if entry is None:
+                    return False
+                timeout = entry.expires_at - time.monotonic()
+                if timeout <= 0:
+                    self._promises.pop(checksum, None)
+                    return False
+                event = entry.event
+            try:
+                await asyncio.wait_for(event.wait(), timeout)
+                return True
+            except asyncio.TimeoutError:
+                async with self._lock:
+                    current = self._promises.get(checksum)
+                    if current is not entry:
+                        continue
+                    remaining = current.expires_at - time.monotonic()
+                    if remaining <= 0:
+                        self._promises.pop(checksum, None)
+                        return False
+                continue
+
+
+_promise_registry = PromiseRegistry()
+
+
+async def _wait_for_current_put_requests(checksums: Iterable[str]) -> None:
+    if isinstance(checksums, (str, bytes)):
+        checksum_set = {checksums}
+    else:
+        checksum_set = set(checksums)
+    async with _current_put_condition:
+        while _current_put_requests.intersection(checksum_set):
+            await _current_put_condition.wait()
+
+
+async def put_file(checksum: Annotated[Checksum, Path()], rq: Request) -> Response:
+
+    checksum_str = parse_checksum(checksum)
+    LOGGER.info("PUT %s start", checksum_str)
+
+    if layout == "prefix":
+        prefix = checksum_str[:2]
+        target_dir = os.path.join(directory, prefix)
+    else:
+        target_dir = directory
+    path = os.path.join(target_dir, checksum_str)
+
+    if layout == "prefix":
+        target_directory = anyio.Path(target_dir)
+        if not await target_directory.exists():
+            await target_directory.mkdir(exist_ok=True)
+        await aiofiles_chmod(target_dir, 0o3775)
+    if await aiofiles.ospath.exists(path):
+        LOGGER.info("PUT %s already exists", checksum_str)
+        await aiofiles_chmod(path, 0o444)
+        await _promise_registry.resolve(checksum_str)
+        return Response(status_code=201)
+
+    ok = False
+    added_to_put_requests = False
+    cs_stream = calculate_checksum_stream()
+    temp_path = None
+    buffer_checksum = None
+    try:
+        async with _current_put_condition:
+            if checksum_str in _current_put_requests:
+                LOGGER.info("PUT %s already in progress", checksum_str)
+                return Response(status_code=202)
+            _current_put_requests.add(checksum_str)
+            added_to_put_requests = True
+        async with aiofiles.tempfile.NamedTemporaryFile(
+            dir=target_dir,
+            prefix=checksum_str + "-",
+            delete=False,
+        ) as file:
+            async for chunk in rq.stream():
+                cs_stream.update(chunk)
+                await file.write(chunk)
+            buffer_checksum = cs_stream.hexdigest()
+            temp_path = file.name
+        if buffer_checksum != checksum_str:
+            LOGGER.warning("PUT %s incorrect checksum", checksum_str)
+            return Response(status_code=400, content="Incorrect checksum")
+        if not await aiofiles.ospath.exists(path):
+            try:
+                await aiofiles.os.replace(temp_path, path)
+            except Exception:
+                if not await aiofiles.ospath.exists(path):
+                    raise
+        ok = True
+        try:
+            await aiofiles_chmod(path, 0o444)
+        except Exception:
+            pass
+
+    except ClientDisconnect:
+        LOGGER.warning("PUT %s client disconnected", checksum_str)
+        return Response(status_code=400)
+
+    finally:
+        if added_to_put_requests:
+            async with _current_put_condition:
+                _current_put_requests.remove(checksum_str)
+                _current_put_condition.notify_all()
+        if temp_path is not None:
+            try:
+                await aiofiles.os.unlink(temp_path)
+            except FileNotFoundError:
+                pass
+            except Exception:
+                pass
+        if added_to_put_requests and not ok:
+            try:
+                pathlib.Path(path).unlink()
+            except FileNotFoundError:
+                pass
+
+    if ok:
+        LOGGER.info("PUT %s completed", checksum_str)
+        await _promise_registry.resolve(checksum_str)
+    return Response(content="OK")
+
+
+if writable:
+    put_file = app.put("/{checksum}")(put_file)
+    promise = app.put("/promise/{checksum}")(promise)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+def main():
+    """Console-script shim; server launch happens during module import."""
+    return 0
+
+
+def _timestamped_log_config():
+    try:
+        from uvicorn.config import LOGGING_CONFIG
+    except Exception:  # pragma: no cover - uvicorn import guard
+        return None
+
+    log_config = copy.deepcopy(LOGGING_CONFIG)
+    formatters = log_config.get("formatters", {})
+    for name in ("default", "access"):
+        formatter = formatters.get(name)
+        if not formatter:
+            continue
+        fmt = formatter.get("fmt")
+        if fmt:
+            formatter["fmt"] = f"%(asctime)s {fmt}"
+        else:
+            formatter["fmt"] = "%(asctime)s %(message)s"
+    return log_config
+
+
+if as_commandline_tool:
+    import uvicorn
+
+    log_config = _timestamped_log_config()
+    config_kwargs = dict(app=app, port=args.port, host=args.host)
+    if log_config is not None:
+        config_kwargs["log_config"] = log_config
+    config = uvicorn.Config(**config_kwargs)
+    server = uvicorn.Server(config)
+
+    if status_tracker:
+
+        @app.on_event("startup")
+        async def _hashserver_status_file_running():
+            await anyio.to_thread.run_sync(status_tracker.write_running)
+
+    if timeout_seconds is not None:
+        setup_inactivity_timeout(timeout_seconds, server)
+
+    print("OK")
+    try:
+        server.run()
+    except BaseException:
+        if status_tracker and not status_tracker.running_written:
+            status_tracker.write_failed()
+        raise
+else:
+    # uvicorn (or some other ASGI launcher) will take care of it
+    pass