theblob 0.1.0__py3-none-any.whl

theblob/__init__.py

@@ -0,0 +1 @@
+"""theblob: an S3-compatible object store."""

theblob/asgi.py

@@ -0,0 +1,238 @@
+"""The ASGI application: the one place where S3 errors become HTTP responses."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import os
+import secrets
+import time
+from contextlib import aclosing, suppress
+from dataclasses import replace
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from theblob import handlers, maintenance, meta
+from theblob.blobstore import BlobStore
+from theblob.handlers import ClientDisconnectedError, Context, Request, Response
+from theblob.meta import Database
+from theblob.model import HttpRequest
+from theblob.s3 import errors, routing, sigv4, xml
+from theblob.s3.errors import S3Error
+from theblob.s3.routing import BucketRoute, ObjectRoute, Operation, ServiceRoute
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping
+    from typing import Any
+
+    type Scope = Mapping[str, Any]
+    type Message = Mapping[str, Any]
+    type Receive = Callable[[], Awaitable[Message]]
+    type Send = Callable[[Message], Awaitable[None]]
+    type App = Callable[[Scope, Receive, Send], Awaitable[None]]
+
+ENV_DATA_DIR = "THEBLOB_DATA_DIR"
+ENV_BASE_DOMAIN = "THEBLOB_BASE_DOMAIN"
+META_FILE = "meta.db"
+
+_CONNECTION_CLOSE = ("connection", "close")
+_FIRST_ERROR_STATUS = 400
+_FIRST_SERVER_ERROR_STATUS = 500
+_STATUS_CLIENT_GONE = 499
+_STATUSES_WITHOUT_BODY = frozenset({204, 304})
+
+logger = logging.getLogger("theblob.access")
+
+
+def create_app_from_env() -> App:
+    """Build the application in a worker process, from what `serve` set up."""
+    # Here and not in `serve`: each worker is its own process, with its own logging.
+    logging.basicConfig(level=logging.INFO, format="%(message)s")
+    data_dir = Path(os.environ[ENV_DATA_DIR])
+    context = Context(Database(data_dir / META_FILE), BlobStore(data_dir))
+    return create_app(context, os.environ.get(ENV_BASE_DOMAIN))
+
+
+def create_app(context: Context, base_domain: str | None) -> App:
+    async def app(scope: Scope, receive: Receive, send: Send) -> None:
+        if scope["type"] == "lifespan":
+            await _run_lifespan(context, receive, send)
+        else:
+            await _handle_http(context, base_domain, scope, receive, send)
+
+    return app
+
+
+async def _run_lifespan(context: Context, receive: Receive, send: Send) -> None:
+    await receive()
+    task = asyncio.create_task(maintenance.run_forever(context.db, context.blobs))
+    await send({"type": "lifespan.startup.complete"})
+    await receive()
+    task.cancel()
+    with suppress(asyncio.CancelledError):
+        await task
+    context.db.close()
+    await send({"type": "lifespan.shutdown.complete"})
+
+
+async def _handle_http(
+    context: Context,
+    base_domain: str | None,
+    scope: Scope,
+    receive: Receive,
+    send: Send,
+) -> None:
+    request_id = secrets.token_hex(8)
+    started = time.monotonic()
+    http_request = _parse_http(scope)
+    access = {
+        "request_id": request_id,
+        "method": http_request.method,
+        "path": http_request.raw_path.decode("latin-1"),
+        "operation": "unknown",
+    }
+    try:
+        route = routing.route(http_request, base_domain)
+        access["operation"] = route.operation.name
+        response = await _dispatch(context, http_request, route, receive)
+    except S3Error as error:
+        response = _error_response(error, http_request.raw_path, request_id)
+    except ClientDisconnectedError:
+        _log_access(access, _STATUS_CLIENT_GONE, started)
+        return
+    except Exception:
+        logger.exception("Unhandled error in request %s", request_id)
+        response = _error_response(
+            S3Error(errors.INTERNAL_ERROR), http_request.raw_path, request_id
+        )
+
+    if "expect" in http_request.headers and response.status >= _FIRST_ERROR_STATUS:
+        # The client held its body back and will send it after no 100 Continue. If
+        # the connection stayed open, those bytes would parse as the next request.
+        response = replace(response, headers=(*response.headers, _CONNECTION_CLOSE))
+    try:
+        await _send_response(response, http_request.method, request_id, receive, send)
+    finally:
+        # Including a stream that fails midway: its status was already sent.
+        _log_access(access, response.status, started)
+
+
+def _parse_http(scope: Scope) -> HttpRequest:
+    headers: dict[str, str] = {}
+    for raw_name, raw_value in scope["headers"]:
+        name = raw_name.decode("latin-1").lower()
+        value = raw_value.decode("latin-1")
+        headers[name] = f"{headers[name]},{value}" if name in headers else value
+    raw_query: bytes = scope["query_string"]
+    return HttpRequest(
+        scope["method"],
+        scope["raw_path"],
+        raw_query,
+        routing.parse_query(raw_query),
+        headers,
+    )
+
+
+async def _dispatch(
+    context: Context,
+    http_request: HttpRequest,
+    route: routing.Route,
+    receive: Receive,
+) -> Response:
+    if route.operation is Operation.HEALTH:
+        await meta.ping(context.db)
+        return Response(body=b"ok\n", headers=(("content-type", "text/plain"),))
+
+    claim = sigv4.parse_claim(http_request)
+    secret = await meta.get_secret(context.db, claim.credential.access_key)
+    seed = sigv4.verify(http_request, claim, secret, time.time())
+    request = Request(http_request.headers, http_request.query, receive, seed)
+
+    match route:
+        case ServiceRoute(operation=operation):
+            return await handlers.SERVICE_HANDLERS[operation](context, request)
+        case BucketRoute(operation=operation, bucket=bucket):
+            return await handlers.BUCKET_HANDLERS[operation](context, request, bucket)
+        case ObjectRoute(operation=operation, bucket=bucket, key=key):
+            return await handlers.OBJECT_HANDLERS[operation](
+                context, request, bucket, key
+            )
+
+
+def _error_response(error: S3Error, raw_path: bytes, request_id: str) -> Response:
+    if error.error.status >= _FIRST_SERVER_ERROR_STATUS:
+        logger.error("%s: %s", request_id, error)
+    body = xml.render_error(
+        error.error.code,
+        error.message,
+        raw_path.decode("latin-1"),
+        request_id,
+    )
+    return Response(
+        error.error.status,
+        (("content-type", "application/xml"), *error.headers),
+        body,
+    )
+
+
+async def _send_response(
+    response: Response, method: str, request_id: str, receive: Receive, send: Send
+) -> None:
+    headers = [
+        (name.encode("latin-1"), value.encode("latin-1"))
+        for name, value in response.headers
+    ]
+    headers.append((b"x-amz-request-id", request_id.encode()))
+    body = response.body
+    has_length = any(name == b"content-length" for name, _ in headers)
+    if (
+        isinstance(body, bytes)
+        and not has_length
+        and response.status not in _STATUSES_WITHOUT_BODY
+    ):
+        headers.append((b"content-length", str(len(body)).encode()))
+    await send({
+        "type": "http.response.start",
+        "status": response.status,
+        "headers": headers,
+    })
+    if isinstance(body, bytes):
+        payload = b"" if method == "HEAD" else body
+        await send({"type": "http.response.body", "body": payload})
+    else:
+        await _stream_body(body, receive, send)
+
+
+async def _stream_body(
+    body: AsyncGenerator[bytes], receive: Receive, send: Send
+) -> None:
+    """Send a blob stream, and stop reading the disk once the client has gone.
+
+    uvicorn silently drops `send` after a disconnect, so nothing else would stop
+    an abandoned download from reading the whole object.
+    """
+    disconnected = asyncio.create_task(_wait_for_disconnect(receive))
+    try:
+        async with aclosing(body) as chunks:
+            async for chunk in chunks:
+                if disconnected.done():
+                    return
+                await send({
+                    "type": "http.response.body",
+                    "body": chunk,
+                    "more_body": True,
+                })
+        await send({"type": "http.response.body", "body": b""})
+    finally:
+        disconnected.cancel()
+
+
+async def _wait_for_disconnect(receive: Receive) -> None:
+    while (await receive())["type"] != "http.disconnect":
+        pass
+
+
+def _log_access(access: Mapping[str, str], status: int, started: float) -> None:
+    seconds = round(time.monotonic() - started, 6)
+    logger.info("%s", json.dumps({**access, "status": status, "seconds": seconds}))

theblob/blobstore.py

@@ -0,0 +1,190 @@
+"""Blob files: durable writes, range reads and deletion.
+
+A blob's ID is random, so user-supplied keys never reach the filesystem.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import secrets
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Self
+
+from theblob.model import Blob
+from theblob.s3 import errors
+from theblob.s3.errors import S3Error
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncGenerator, AsyncIterable, Iterable, Mapping
+    from types import TracebackType
+
+    from theblob.model import BlobSlice
+    from theblob.s3.payload import Hasher
+
+BUFFER_BYTES = 1024 * 1024
+TMP_MAX_AGE_SECONDS = 24 * 3600
+
+_SHARD_LENGTH = 3
+_SHARD_COUNT = 16**_SHARD_LENGTH
+
+
+def existing_files(directory: Path) -> Iterable[tuple[str, os.stat_result]]:
+    """Yield the files in a directory that other workers are changing concurrently.
+
+    A file listed a moment ago may have been committed or collected since.
+    """
+    with os.scandir(directory) as entries:
+        for entry in entries:
+            try:
+                yield entry.path, entry.stat()
+            except FileNotFoundError:
+                continue
+
+
+def _fsync_directory(path: Path) -> None:
+    descriptor = os.open(path, os.O_RDONLY)
+    try:
+        os.fsync(descriptor)
+    finally:
+        os.close(descriptor)
+
+
+@dataclass(frozen=True, slots=True)
+class BlobStore:
+    """A directory of immutable blob files."""
+
+    root: Path
+
+    @property
+    def tmp_dir(self) -> Path:
+        return self.root / "tmp"
+
+    @property
+    def blobs_dir(self) -> Path:
+        return self.root / "blobs"
+
+    def initialize(self) -> None:
+        """Create every shard up front, so writes never create a directory."""
+        self.tmp_dir.mkdir(parents=True, exist_ok=True)
+        for number in range(_SHARD_COUNT):
+            shard = f"{number:0{_SHARD_LENGTH}x}"
+            (self.blobs_dir / shard).mkdir(parents=True, exist_ok=True)
+
+    def path(self, blob_id: str) -> Path:
+        return self.blobs_dir / blob_id[:_SHARD_LENGTH] / blob_id
+
+    def writer(self, hashers: Mapping[str, Hasher]) -> BlobWriter:
+        return BlobWriter(self, hashers)
+
+    def delete(self, blob_id: str) -> None:
+        self.path(blob_id).unlink(missing_ok=True)
+
+    def sweep_tmp(self, now: float) -> int:
+        """Remove temp files abandoned by a crash. Returns how many were removed."""
+        removed = 0
+        for path, stat in existing_files(self.tmp_dir):
+            if now - stat.st_mtime > TMP_MAX_AGE_SECONDS:
+                Path(path).unlink(missing_ok=True)
+                removed += 1
+        return removed
+
+    def scan(self) -> Iterable[tuple[str, int, float]]:
+        """Yield (blob ID, size, modification time) for every blob on disk."""
+        for shard in self.blobs_dir.iterdir():
+            for path, stat in existing_files(shard):
+                yield Path(path).name, stat.st_size, stat.st_mtime
+
+
+class BlobWriter:
+    """One blob being written. It only becomes durable and named on `commit`."""
+
+    def __init__(self, store: BlobStore, hashers: Mapping[str, Hasher]) -> None:
+        self._store = store
+        self._hashers = hashers
+        self.blob_id = secrets.token_hex(16)
+        self._tmp_path = store.tmp_dir / self.blob_id
+        self._file = self._tmp_path.open("xb")
+        self._committed = False
+        self.size = 0
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        traceback: TracebackType | None,
+    ) -> None:
+        self._file.close()
+        if not self._committed:
+            self._tmp_path.unlink(missing_ok=True)
+
+    def write(self, data: bytes) -> None:
+        """Hash and write one buffer. Runs on a worker thread."""
+        for hasher in self._hashers.values():
+            hasher.update(data)
+        self._file.write(data)
+        self.size += len(data)
+
+    def digests(self) -> dict[str, bytes]:
+        return {name: hasher.digest() for name, hasher in self._hashers.items()}
+
+    def commit(self) -> Blob:
+        """Make the blob durable under its final name. Runs on a worker thread."""
+        self._file.flush()
+        # ponytail: on macOS fsync doesn't flush the drive cache. Linux is the
+        # target; add fcntl F_FULLFSYNC if it is ever deployed on macOS.
+        os.fsync(self._file.fileno())
+        self._file.close()
+
+        final_path = self._store.path(self.blob_id)
+        self._tmp_path.rename(final_path)
+        _fsync_directory(final_path.parent)
+        self._committed = True
+        return Blob(self.blob_id, self.size, self._hashers["md5"].digest().hex())
+
+
+async def write_stream(
+    writer: BlobWriter, chunks: AsyncIterable[bytes], max_bytes: int
+) -> None:
+    """Feed a stream to a writer, one thread-pool trip per buffer of data."""
+    buffer = bytearray()
+    received = 0
+    async for chunk in chunks:
+        received += len(chunk)
+        if received > max_bytes:
+            raise S3Error(errors.ENTITY_TOO_LARGE)
+        buffer += chunk
+        if len(buffer) >= BUFFER_BYTES:
+            await asyncio.to_thread(writer.write, bytes(buffer))
+            buffer.clear()
+    if buffer:
+        await asyncio.to_thread(writer.write, bytes(buffer))
+
+
+async def read_slices(
+    store: BlobStore, slices: Iterable[BlobSlice]
+) -> AsyncGenerator[bytes]:
+    """Stream byte ranges of blobs, opening each blob only when it is reached."""
+    for blob_slice in slices:
+        descriptor = await asyncio.to_thread(
+            os.open, store.path(blob_slice.blob_id), os.O_RDONLY
+        )
+        try:
+            position = blob_slice.offset
+            remaining = blob_slice.length
+            while remaining > 0:
+                data = await asyncio.to_thread(
+                    os.pread, descriptor, min(BUFFER_BYTES, remaining), position
+                )
+                if not data:
+                    message = f"Blob {blob_slice.blob_id} is shorter than its metadata."
+                    raise OSError(message)
+                position += len(data)
+                remaining -= len(data)
+                yield data
+        finally:
+            os.close(descriptor)

theblob/cli.py

@@ -0,0 +1,211 @@
+"""The `theblob` command."""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import secrets
+import string
+import sys
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Annotated
+
+import cyclopts
+import uvicorn
+
+from theblob import asgi, maintenance, meta
+from theblob.blobstore import BlobStore
+from theblob.meta import Database
+
+if TYPE_CHECKING:
+    from collections.abc import Awaitable, Callable
+    from typing import Concatenate
+
+_DEFAULT_DATA_DIR = Path("./data")
+_ACCESS_KEY_ALPHABET = string.ascii_uppercase + string.digits
+_ACCESS_KEY_LENGTH = 20
+_SECRET_BYTES = 30
+
+DataDir = Annotated[Path, cyclopts.Parameter(help="Directory holding all stored data.")]
+
+app = cyclopts.App(name="theblob", help="An S3-compatible object store.")
+key_app = cyclopts.App(name="key", help="Manage access keys.")
+app.command(key_app)
+
+
+@cyclopts.Parameter(name="*")
+@dataclass(frozen=True, kw_only=True)
+class ServeOptions:
+    """Options of `theblob serve`.
+
+    Parameters
+    ----------
+    data_dir
+        Directory holding all stored data.
+    host
+        Address to listen on.
+    port
+        Port to listen on.
+    workers
+        Number of worker processes; defaults to the number of CPUs.
+    domain
+        Base domain enabling virtual-hosted addressing (bucket.domain).
+    ssl_certfile
+        TLS certificate, to serve HTTPS directly.
+    ssl_keyfile
+        TLS private key.
+    shutdown_grace
+        Seconds to let requests finish on shutdown. Without a limit, one client
+        that stops reading would prevent the server from ever exiting.
+    """
+
+    data_dir: Path = _DEFAULT_DATA_DIR
+    host: str = "127.0.0.1"
+    port: int = 9000
+    workers: int | None = None
+    domain: str | None = None
+    ssl_certfile: Path | None = None
+    ssl_keyfile: Path | None = None
+    shutdown_grace: int = 20
+
+
+@app.command
+def serve(options: ServeOptions | None = None) -> None:
+    """Run the S3 server."""
+    options = options or ServeOptions()
+    _prepare(options.data_dir)
+    asyncio.run(_create_first_key_if_needed(options.data_dir))
+
+    os.environ[asgi.ENV_DATA_DIR] = str(options.data_dir.resolve())
+    if options.domain:
+        os.environ[asgi.ENV_BASE_DOMAIN] = options.domain
+    else:
+        os.environ.pop(asgi.ENV_BASE_DOMAIN, None)
+
+    uvicorn.run(
+        "theblob.asgi:create_app_from_env",
+        factory=True,
+        host=options.host,
+        port=options.port,
+        workers=options.workers or os.cpu_count(),
+        http="httptools",
+        loop="uvloop",
+        lifespan="on",
+        ws="none",
+        access_log=False,
+        ssl_certfile=options.ssl_certfile,
+        ssl_keyfile=options.ssl_keyfile,
+        timeout_graceful_shutdown=options.shutdown_grace,
+    )
+
+
+@key_app.command(name="create")
+def key_create(*, data_dir: DataDir = _DEFAULT_DATA_DIR) -> None:
+    """Create an access key and print it. The secret isn't shown again."""
+    _prepare(data_dir)
+    access_key, secret = asyncio.run(_create_key(data_dir))
+    print(f"access key: {access_key}")
+    print(f"secret key: {secret}")
+
+
+@key_app.command(name="list")
+def key_list(*, data_dir: DataDir = _DEFAULT_DATA_DIR) -> None:
+    """List access keys."""
+    _open_existing(data_dir)
+    for access_key in asyncio.run(_with_db(data_dir, meta.list_access_keys)):
+        print(access_key)
+
+
+@key_app.command(name="delete")
+def key_delete(access_key: str, /, *, data_dir: DataDir = _DEFAULT_DATA_DIR) -> None:
+    """Delete an access key."""
+    _open_existing(data_dir)
+    if not asyncio.run(_with_db(data_dir, meta.delete_access_key, access_key)):
+        sys.exit(f"theblob: no such access key: {access_key}")
+
+
+@app.command
+def fsck(
+    *, data_dir: DataDir = _DEFAULT_DATA_DIR, delete_orphans: bool = False
+) -> None:
+    """Compare the metadata with the disk; exit non-zero if they disagree.
+
+    Parameters
+    ----------
+    delete_orphans
+        Remove blob files that have no metadata and are over a day old.
+    """
+    _open_existing(data_dir)
+    blobs = BlobStore(data_dir)
+    report = asyncio.run(_with_db(data_dir, _run_fsck, blobs))
+    for label, blob_ids in (
+        ("orphan", report.orphans),
+        ("recent orphan (may be in flight)", report.young_orphans),
+        ("missing", report.missing),
+        ("wrong size", report.wrong_size),
+    ):
+        for blob_id in blob_ids:
+            print(f"{label}: {blob_id}")
+    if delete_orphans:
+        for blob_id in report.orphans:
+            blobs.delete(blob_id)
+        print(f"deleted {len(report.orphans)} orphans")
+    if report.missing or report.wrong_size or (report.orphans and not delete_orphans):
+        sys.exit(1)
+    print("clean")
+
+
+async def _run_fsck(db: Database, blobs: BlobStore) -> maintenance.FsckReport:
+    return await maintenance.fsck(db, blobs, time.time())
+
+
+def _prepare(data_dir: Path) -> None:
+    """Create the directory layout and schema, before any worker can start."""
+    data_dir.mkdir(parents=True, exist_ok=True)
+    data_dir.chmod(0o700)  # meta.db holds secrets, even if the directory existed.
+    BlobStore(data_dir).initialize()
+    meta.initialize(data_dir / asgi.META_FILE)
+
+
+def _open_existing(data_dir: Path) -> None:
+    """Refuse a path that holds no store: reporting on it would be false reassurance."""
+    if not (data_dir / asgi.META_FILE).exists():
+        sys.exit(f"theblob: no store found in {data_dir}")
+    meta.initialize(data_dir / asgi.META_FILE)
+
+
+async def _with_db[**P, R](
+    data_dir: Path,
+    function: Callable[Concatenate[Database, P], Awaitable[R]],
+    *args: P.args,
+    **kwargs: P.kwargs,
+) -> R:
+    db = Database(data_dir / asgi.META_FILE)
+    try:
+        return await function(db, *args, **kwargs)
+    finally:
+        db.close()
+
+
+async def _create_key(data_dir: Path) -> tuple[str, str]:
+    access_key = "".join(
+        secrets.choice(_ACCESS_KEY_ALPHABET) for _ in range(_ACCESS_KEY_LENGTH)
+    )
+    secret = secrets.token_urlsafe(_SECRET_BYTES)
+    await _with_db(data_dir, meta.add_access_key, access_key, secret)
+    return access_key, secret
+
+
+async def _create_first_key_if_needed(data_dir: Path) -> None:
+    """Make a fresh store usable at once: no key means nobody could ever connect."""
+    if await _with_db(data_dir, meta.list_access_keys):
+        return
+    access_key, secret = await _create_key(data_dir)
+    print(
+        "Created the first access key; the secret isn't shown again:\n"
+        f"  access key: {access_key}\n"
+        f"  secret key: {secret}",
+        flush=True,
+    )