theblob 0.1.0__tar.gz

theblob-0.1.0/PKG-INFO

@@ -0,0 +1,53 @@
+Metadata-Version: 2.3
+Name: theblob
+Version: 0.1.0
+Summary: An S3-compatible object store
+Author: Stefane Fermigier
+Author-email: Stefane Fermigier <sf@abilian.com>
+Requires-Dist: cyclopts>=5.3.3
+Requires-Dist: defusedxml>=0.7.1
+Requires-Dist: httptools>=0.6.4
+Requires-Dist: uvicorn>=0.54.0
+Requires-Dist: uvloop>=0.21.0
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+
+# theblob
+
+An S3-compatible object store written in Python. It starts as one command on one machine, and is designed to grow into a redundant cluster.
+
+This is an experiment. Phase 1 (one server, one disk) is under construction.
+
+## Quick start
+
+```sh
+uv sync
+uv run theblob serve --data-dir ./data
+```
+
+The first start creates an access key and prints it once. Use it with any S3 client:
+
+```python
+import boto3
+
+s3 = boto3.client(
+    "s3",
+    endpoint_url="http://127.0.0.1:9000",
+    aws_access_key_id="...",
+    aws_secret_access_key="...",
+    region_name="us-east-1",
+)
+s3.create_bucket(Bucket="my-bucket")
+s3.put_object(Bucket="my-bucket", Key="hello.txt", Body=b"hello")
+```
+
+Other commands: `theblob key create|list|delete` and `theblob fsck`. Run `theblob serve --help` for the server options, including TLS and virtual-hosted addressing.
+
+## Development
+
+```sh
+make test      # the whole test pyramid
+make lint      # ruff, ty, pyrefly, mypy
+```
+
+Tests are in `tests/`: `a_unit` for pure logic, `b_integration` for real SQLite and files, `c_e2e` for real S3 clients (boto3, AWS CLI, rclone, s5cmd) against a real server over HTTP and HTTPS. The rclone and s5cmd tests are skipped if those tools are not installed.

theblob-0.1.0/README.md

@@ -0,0 +1,39 @@
+# theblob
+
+An S3-compatible object store written in Python. It starts as one command on one machine, and is designed to grow into a redundant cluster.
+
+This is an experiment. Phase 1 (one server, one disk) is under construction.
+
+## Quick start
+
+```sh
+uv sync
+uv run theblob serve --data-dir ./data
+```
+
+The first start creates an access key and prints it once. Use it with any S3 client:
+
+```python
+import boto3
+
+s3 = boto3.client(
+    "s3",
+    endpoint_url="http://127.0.0.1:9000",
+    aws_access_key_id="...",
+    aws_secret_access_key="...",
+    region_name="us-east-1",
+)
+s3.create_bucket(Bucket="my-bucket")
+s3.put_object(Bucket="my-bucket", Key="hello.txt", Body=b"hello")
+```
+
+Other commands: `theblob key create|list|delete` and `theblob fsck`. Run `theblob serve --help` for the server options, including TLS and virtual-hosted addressing.
+
+## Development
+
+```sh
+make test      # the whole test pyramid
+make lint      # ruff, ty, pyrefly, mypy
+```
+
+Tests are in `tests/`: `a_unit` for pure logic, `b_integration` for real SQLite and files, `c_e2e` for real S3 clients (boto3, AWS CLI, rclone, s5cmd) against a real server over HTTP and HTTPS. The rclone and s5cmd tests are skipped if those tools are not installed.

theblob-0.1.0/pyproject.toml

@@ -0,0 +1,58 @@
+[project]
+name = "theblob"
+version = "0.1.0"
+description = "An S3-compatible object store"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "cyclopts>=5.3.3",
+    "defusedxml>=0.7.1",
+    "httptools>=0.6.4",
+    "uvicorn>=0.54.0",
+    "uvloop>=0.21.0",
+]
+
+[[project.authors]]
+name = "Stefane Fermigier"
+email = "sf@abilian.com"
+
+[project.scripts]
+theblob = "theblob.cli:app"
+
+[build-system]
+requires = ["uv_build>=0.13.4,<0.14.0"]
+build-backend = "uv_build"
+
+[dependency-groups]
+dev = [
+    "anyio>=4.16.0",
+    "awscli>=1.46.2",
+    "boto3>=1.43.124",
+    "httpx>=0.28.1",
+    "mypy>=2.4.1",
+    "nox>=2026.10.20",
+    "pre-commit>=4.7.0",
+    "pyrefly>=1.4.1",
+    "pytest>=9.2.0",
+    "pytest-cov>=7.1.0",
+    "pytest-timeout>=2.4.0",
+    "ruff>=0.17.3",
+    "ty>=0.0.97",
+    "types-defusedxml>=0.7.0.20260504",
+]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+timeout = 120
+markers = [
+    "unit: fast tests of pure logic, no I/O",
+    "integration: components against real SQLite and real files",
+    "e2e: real S3 clients against a real server process",
+    "slow: takes more than a few seconds",
+]
+
+[tool.pyrefly]
+project-includes = [
+    "**/*.py*",
+    "**/*.ipynb",
+]

theblob-0.1.0/pyproject.toml.orig

@@ -0,0 +1,57 @@
+[project]
+name = "theblob"
+version = "0.1.0"
+description = "An S3-compatible object store"
+readme = "README.md"
+authors = [
+    { name = "Stefane Fermigier", email = "sf@abilian.com" }
+]
+requires-python = ">=3.12"
+dependencies = [
+    "cyclopts>=5.3.3",
+    "defusedxml>=0.7.1",
+    "httptools>=0.6.4",
+    "uvicorn>=0.54.0",
+    "uvloop>=0.21.0",
+]
+
+[project.scripts]
+theblob = "theblob.cli:app"
+
+[build-system]
+requires = ["uv_build>=0.13.4,<0.14.0"]
+build-backend = "uv_build"
+
+[dependency-groups]
+dev = [
+    "anyio>=4.16.0",
+    "awscli>=1.46.2",
+    "boto3>=1.43.124",
+    "httpx>=0.28.1",
+    "mypy>=2.4.1",
+    "nox>=2026.10.20",
+    "pre-commit>=4.7.0",
+    "pyrefly>=1.4.1",
+    "pytest>=9.2.0",
+    "pytest-cov>=7.1.0",
+    "pytest-timeout>=2.4.0",
+    "ruff>=0.17.3",
+    "ty>=0.0.97",
+    "types-defusedxml>=0.7.0.20260504",
+]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+timeout = 120
+markers = [
+    "unit: fast tests of pure logic, no I/O",
+    "integration: components against real SQLite and real files",
+    "e2e: real S3 clients against a real server process",
+    "slow: takes more than a few seconds",
+]
+
+[tool.pyrefly]
+project-includes = [
+    "**/*.py*",
+    "**/*.ipynb",
+]

theblob-0.1.0/src/theblob/__init__.py

@@ -0,0 +1 @@
+"""theblob: an S3-compatible object store."""

theblob-0.1.0/src/theblob/asgi.py

@@ -0,0 +1,238 @@
+"""The ASGI application: the one place where S3 errors become HTTP responses."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import os
+import secrets
+import time
+from contextlib import aclosing, suppress
+from dataclasses import replace
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from theblob import handlers, maintenance, meta
+from theblob.blobstore import BlobStore
+from theblob.handlers import ClientDisconnectedError, Context, Request, Response
+from theblob.meta import Database
+from theblob.model import HttpRequest
+from theblob.s3 import errors, routing, sigv4, xml
+from theblob.s3.errors import S3Error
+from theblob.s3.routing import BucketRoute, ObjectRoute, Operation, ServiceRoute
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping
+    from typing import Any
+
+    type Scope = Mapping[str, Any]
+    type Message = Mapping[str, Any]
+    type Receive = Callable[[], Awaitable[Message]]
+    type Send = Callable[[Message], Awaitable[None]]
+    type App = Callable[[Scope, Receive, Send], Awaitable[None]]
+
+ENV_DATA_DIR = "THEBLOB_DATA_DIR"
+ENV_BASE_DOMAIN = "THEBLOB_BASE_DOMAIN"
+META_FILE = "meta.db"
+
+_CONNECTION_CLOSE = ("connection", "close")
+_FIRST_ERROR_STATUS = 400
+_FIRST_SERVER_ERROR_STATUS = 500
+_STATUS_CLIENT_GONE = 499
+_STATUSES_WITHOUT_BODY = frozenset({204, 304})
+
+logger = logging.getLogger("theblob.access")
+
+
+def create_app_from_env() -> App:
+    """Build the application in a worker process, from what `serve` set up."""
+    # Here and not in `serve`: each worker is its own process, with its own logging.
+    logging.basicConfig(level=logging.INFO, format="%(message)s")
+    data_dir = Path(os.environ[ENV_DATA_DIR])
+    context = Context(Database(data_dir / META_FILE), BlobStore(data_dir))
+    return create_app(context, os.environ.get(ENV_BASE_DOMAIN))
+
+
+def create_app(context: Context, base_domain: str | None) -> App:
+    async def app(scope: Scope, receive: Receive, send: Send) -> None:
+        if scope["type"] == "lifespan":
+            await _run_lifespan(context, receive, send)
+        else:
+            await _handle_http(context, base_domain, scope, receive, send)
+
+    return app
+
+
+async def _run_lifespan(context: Context, receive: Receive, send: Send) -> None:
+    await receive()
+    task = asyncio.create_task(maintenance.run_forever(context.db, context.blobs))
+    await send({"type": "lifespan.startup.complete"})
+    await receive()
+    task.cancel()
+    with suppress(asyncio.CancelledError):
+        await task
+    context.db.close()
+    await send({"type": "lifespan.shutdown.complete"})
+
+
+async def _handle_http(
+    context: Context,
+    base_domain: str | None,
+    scope: Scope,
+    receive: Receive,
+    send: Send,
+) -> None:
+    request_id = secrets.token_hex(8)
+    started = time.monotonic()
+    http_request = _parse_http(scope)
+    access = {
+        "request_id": request_id,
+        "method": http_request.method,
+        "path": http_request.raw_path.decode("latin-1"),
+        "operation": "unknown",
+    }
+    try:
+        route = routing.route(http_request, base_domain)
+        access["operation"] = route.operation.name
+        response = await _dispatch(context, http_request, route, receive)
+    except S3Error as error:
+        response = _error_response(error, http_request.raw_path, request_id)
+    except ClientDisconnectedError:
+        _log_access(access, _STATUS_CLIENT_GONE, started)
+        return
+    except Exception:
+        logger.exception("Unhandled error in request %s", request_id)
+        response = _error_response(
+            S3Error(errors.INTERNAL_ERROR), http_request.raw_path, request_id
+        )
+
+    if "expect" in http_request.headers and response.status >= _FIRST_ERROR_STATUS:
+        # The client held its body back and will send it after no 100 Continue. If
+        # the connection stayed open, those bytes would parse as the next request.
+        response = replace(response, headers=(*response.headers, _CONNECTION_CLOSE))
+    try:
+        await _send_response(response, http_request.method, request_id, receive, send)
+    finally:
+        # Including a stream that fails midway: its status was already sent.
+        _log_access(access, response.status, started)
+
+
+def _parse_http(scope: Scope) -> HttpRequest:
+    headers: dict[str, str] = {}
+    for raw_name, raw_value in scope["headers"]:
+        name = raw_name.decode("latin-1").lower()
+        value = raw_value.decode("latin-1")
+        headers[name] = f"{headers[name]},{value}" if name in headers else value
+    raw_query: bytes = scope["query_string"]
+    return HttpRequest(
+        scope["method"],
+        scope["raw_path"],
+        raw_query,
+        routing.parse_query(raw_query),
+        headers,
+    )
+
+
+async def _dispatch(
+    context: Context,
+    http_request: HttpRequest,
+    route: routing.Route,
+    receive: Receive,
+) -> Response:
+    if route.operation is Operation.HEALTH:
+        await meta.ping(context.db)
+        return Response(body=b"ok\n", headers=(("content-type", "text/plain"),))
+
+    claim = sigv4.parse_claim(http_request)
+    secret = await meta.get_secret(context.db, claim.credential.access_key)
+    seed = sigv4.verify(http_request, claim, secret, time.time())
+    request = Request(http_request.headers, http_request.query, receive, seed)
+
+    match route:
+        case ServiceRoute(operation=operation):
+            return await handlers.SERVICE_HANDLERS[operation](context, request)
+        case BucketRoute(operation=operation, bucket=bucket):
+            return await handlers.BUCKET_HANDLERS[operation](context, request, bucket)
+        case ObjectRoute(operation=operation, bucket=bucket, key=key):
+            return await handlers.OBJECT_HANDLERS[operation](
+                context, request, bucket, key
+            )
+
+
+def _error_response(error: S3Error, raw_path: bytes, request_id: str) -> Response:
+    if error.error.status >= _FIRST_SERVER_ERROR_STATUS:
+        logger.error("%s: %s", request_id, error)
+    body = xml.render_error(
+        error.error.code,
+        error.message,
+        raw_path.decode("latin-1"),
+        request_id,
+    )
+    return Response(
+        error.error.status,
+        (("content-type", "application/xml"), *error.headers),
+        body,
+    )
+
+
+async def _send_response(
+    response: Response, method: str, request_id: str, receive: Receive, send: Send
+) -> None:
+    headers = [
+        (name.encode("latin-1"), value.encode("latin-1"))
+        for name, value in response.headers
+    ]
+    headers.append((b"x-amz-request-id", request_id.encode()))
+    body = response.body
+    has_length = any(name == b"content-length" for name, _ in headers)
+    if (
+        isinstance(body, bytes)
+        and not has_length
+        and response.status not in _STATUSES_WITHOUT_BODY
+    ):
+        headers.append((b"content-length", str(len(body)).encode()))
+    await send({
+        "type": "http.response.start",
+        "status": response.status,
+        "headers": headers,
+    })
+    if isinstance(body, bytes):
+        payload = b"" if method == "HEAD" else body
+        await send({"type": "http.response.body", "body": payload})
+    else:
+        await _stream_body(body, receive, send)
+
+
+async def _stream_body(
+    body: AsyncGenerator[bytes], receive: Receive, send: Send
+) -> None:
+    """Send a blob stream, and stop reading the disk once the client has gone.
+
+    uvicorn silently drops `send` after a disconnect, so nothing else would stop
+    an abandoned download from reading the whole object.
+    """
+    disconnected = asyncio.create_task(_wait_for_disconnect(receive))
+    try:
+        async with aclosing(body) as chunks:
+            async for chunk in chunks:
+                if disconnected.done():
+                    return
+                await send({
+                    "type": "http.response.body",
+                    "body": chunk,
+                    "more_body": True,
+                })
+        await send({"type": "http.response.body", "body": b""})
+    finally:
+        disconnected.cancel()
+
+
+async def _wait_for_disconnect(receive: Receive) -> None:
+    while (await receive())["type"] != "http.disconnect":
+        pass
+
+
+def _log_access(access: Mapping[str, str], status: int, started: float) -> None:
+    seconds = round(time.monotonic() - started, 6)
+    logger.info("%s", json.dumps({**access, "status": status, "seconds": seconds}))

theblob-0.1.0/src/theblob/blobstore.py

@@ -0,0 +1,190 @@
+"""Blob files: durable writes, range reads and deletion.
+
+A blob's ID is random, so user-supplied keys never reach the filesystem.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import secrets
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Self
+
+from theblob.model import Blob
+from theblob.s3 import errors
+from theblob.s3.errors import S3Error
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncGenerator, AsyncIterable, Iterable, Mapping
+    from types import TracebackType
+
+    from theblob.model import BlobSlice
+    from theblob.s3.payload import Hasher
+
+BUFFER_BYTES = 1024 * 1024
+TMP_MAX_AGE_SECONDS = 24 * 3600
+
+_SHARD_LENGTH = 3
+_SHARD_COUNT = 16**_SHARD_LENGTH
+
+
+def existing_files(directory: Path) -> Iterable[tuple[str, os.stat_result]]:
+    """Yield the files in a directory that other workers are changing concurrently.
+
+    A file listed a moment ago may have been committed or collected since.
+    """
+    with os.scandir(directory) as entries:
+        for entry in entries:
+            try:
+                yield entry.path, entry.stat()
+            except FileNotFoundError:
+                continue
+
+
+def _fsync_directory(path: Path) -> None:
+    descriptor = os.open(path, os.O_RDONLY)
+    try:
+        os.fsync(descriptor)
+    finally:
+        os.close(descriptor)
+
+
+@dataclass(frozen=True, slots=True)
+class BlobStore:
+    """A directory of immutable blob files."""
+
+    root: Path
+
+    @property
+    def tmp_dir(self) -> Path:
+        return self.root / "tmp"
+
+    @property
+    def blobs_dir(self) -> Path:
+        return self.root / "blobs"
+
+    def initialize(self) -> None:
+        """Create every shard up front, so writes never create a directory."""
+        self.tmp_dir.mkdir(parents=True, exist_ok=True)
+        for number in range(_SHARD_COUNT):
+            shard = f"{number:0{_SHARD_LENGTH}x}"
+            (self.blobs_dir / shard).mkdir(parents=True, exist_ok=True)
+
+    def path(self, blob_id: str) -> Path:
+        return self.blobs_dir / blob_id[:_SHARD_LENGTH] / blob_id
+
+    def writer(self, hashers: Mapping[str, Hasher]) -> BlobWriter:
+        return BlobWriter(self, hashers)
+
+    def delete(self, blob_id: str) -> None:
+        self.path(blob_id).unlink(missing_ok=True)
+
+    def sweep_tmp(self, now: float) -> int:
+        """Remove temp files abandoned by a crash. Returns how many were removed."""
+        removed = 0
+        for path, stat in existing_files(self.tmp_dir):
+            if now - stat.st_mtime > TMP_MAX_AGE_SECONDS:
+                Path(path).unlink(missing_ok=True)
+                removed += 1
+        return removed
+
+    def scan(self) -> Iterable[tuple[str, int, float]]:
+        """Yield (blob ID, size, modification time) for every blob on disk."""
+        for shard in self.blobs_dir.iterdir():
+            for path, stat in existing_files(shard):
+                yield Path(path).name, stat.st_size, stat.st_mtime
+
+
+class BlobWriter:
+    """One blob being written. It only becomes durable and named on `commit`."""
+
+    def __init__(self, store: BlobStore, hashers: Mapping[str, Hasher]) -> None:
+        self._store = store
+        self._hashers = hashers
+        self.blob_id = secrets.token_hex(16)
+        self._tmp_path = store.tmp_dir / self.blob_id
+        self._file = self._tmp_path.open("xb")
+        self._committed = False
+        self.size = 0
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        traceback: TracebackType | None,
+    ) -> None:
+        self._file.close()
+        if not self._committed:
+            self._tmp_path.unlink(missing_ok=True)
+
+    def write(self, data: bytes) -> None:
+        """Hash and write one buffer. Runs on a worker thread."""
+        for hasher in self._hashers.values():
+            hasher.update(data)
+        self._file.write(data)
+        self.size += len(data)
+
+    def digests(self) -> dict[str, bytes]:
+        return {name: hasher.digest() for name, hasher in self._hashers.items()}
+
+    def commit(self) -> Blob:
+        """Make the blob durable under its final name. Runs on a worker thread."""
+        self._file.flush()
+        # ponytail: on macOS fsync doesn't flush the drive cache. Linux is the
+        # target; add fcntl F_FULLFSYNC if it is ever deployed on macOS.
+        os.fsync(self._file.fileno())
+        self._file.close()
+
+        final_path = self._store.path(self.blob_id)
+        self._tmp_path.rename(final_path)
+        _fsync_directory(final_path.parent)
+        self._committed = True
+        return Blob(self.blob_id, self.size, self._hashers["md5"].digest().hex())
+
+
+async def write_stream(
+    writer: BlobWriter, chunks: AsyncIterable[bytes], max_bytes: int
+) -> None:
+    """Feed a stream to a writer, one thread-pool trip per buffer of data."""
+    buffer = bytearray()
+    received = 0
+    async for chunk in chunks:
+        received += len(chunk)
+        if received > max_bytes:
+            raise S3Error(errors.ENTITY_TOO_LARGE)
+        buffer += chunk
+        if len(buffer) >= BUFFER_BYTES:
+            await asyncio.to_thread(writer.write, bytes(buffer))
+            buffer.clear()
+    if buffer:
+        await asyncio.to_thread(writer.write, bytes(buffer))
+
+
+async def read_slices(
+    store: BlobStore, slices: Iterable[BlobSlice]
+) -> AsyncGenerator[bytes]:
+    """Stream byte ranges of blobs, opening each blob only when it is reached."""
+    for blob_slice in slices:
+        descriptor = await asyncio.to_thread(
+            os.open, store.path(blob_slice.blob_id), os.O_RDONLY
+        )
+        try:
+            position = blob_slice.offset
+            remaining = blob_slice.length
+            while remaining > 0:
+                data = await asyncio.to_thread(
+                    os.pread, descriptor, min(BUFFER_BYTES, remaining), position
+                )
+                if not data:
+                    message = f"Blob {blob_slice.blob_id} is shorter than its metadata."
+                    raise OSError(message)
+                position += len(data)
+                remaining -= len(data)
+                yield data
+        finally:
+            os.close(descriptor)