leblob 0.4.0.dev1__py3-none-any.whl

leblob/__init__.py

@@ -0,0 +1 @@
+"""leblob: an S3-compatible object store."""

leblob/asgi.py

@@ -0,0 +1,376 @@
+"""The ASGI application: the one place where S3 errors become HTTP responses."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import os
+import secrets
+import time
+from contextlib import aclosing, suppress
+from dataclasses import replace
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from leblob import handlers, maintenance, meta, metrics
+from leblob.blobstore import BlobStore, VolumeError, open_store
+from leblob.handlers import Context, Request, Response
+from leblob.meta import Database
+from leblob.model import HttpRequest
+from leblob.s3 import access, errors, routing, sigv4, xml
+from leblob.s3.errors import S3Error
+from leblob.s3.routing import BucketRoute, ObjectRoute, Operation, ServiceRoute
+from leblob.streams import ClientDisconnectedError
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping
+    from typing import Any
+
+    type Scope = Mapping[str, Any]
+    type Message = Mapping[str, Any]
+    type Receive = Callable[[], Awaitable[Message]]
+    type Send = Callable[[Message], Awaitable[None]]
+    type App = Callable[[Scope, Receive, Send], Awaitable[None]]
+
+ENV_DATA_DIR = "LEBLOB_DATA_DIR"
+ENV_BASE_DOMAIN = "LEBLOB_BASE_DOMAIN"
+ENV_LIFECYCLE_DAY_SECONDS = "LEBLOB_LIFECYCLE_DAY_SECONDS"
+ENV_SSE_OVER_HTTP = "LEBLOB_ALLOW_SSE_C_OVER_HTTP"
+META_FILE = "meta.db"
+
+_CONNECTION_CLOSE = ("connection", "close")
+_FIRST_ERROR_STATUS = 400
+_FIRST_SERVER_ERROR_STATUS = 500
+_STATUS_CLIENT_GONE = 499
+_STATUSES_WITHOUT_BODY = frozenset({204, 304})
+_METRICS_CONTENT_TYPE = "text/plain; version=0.0.4; charset=utf-8"
+
+logger = logging.getLogger("leblob.access")
+
+
+def create_app_from_env() -> App:
+    """Build the application in a worker process, from what `serve` set up."""
+    # Here and not in `serve`: each worker is its own process, with its own logging.
+    logging.basicConfig(level=logging.INFO, format="%(message)s")
+    data_dir = Path(os.environ[ENV_DATA_DIR])
+    database = data_dir / META_FILE
+    context = Context(
+        Database(database),
+        open_blobs_for_worker(data_dir, database),
+        data_dir,
+        int(os.environ[ENV_LIFECYCLE_DAY_SECONDS]) * 1000,
+        ENV_SSE_OVER_HTTP in os.environ,
+    )
+    return create_app(context, os.environ.get(ENV_BASE_DOMAIN))
+
+
+def open_blobs_for_worker(data_dir: Path, database: Path) -> BlobStore:
+    """Open what can be opened, and never refuse to start.
+
+    `serve` has already refused a store that is wrong. What fails here is passing, such
+    as a full disk, and a worker that exits over it is started again and again, while
+    nothing answers clients. One with no volumes answers them with errors, until
+    maintenance finds it can open them after all.
+    """
+    try:
+        records, copies = meta.load_store_settings(database)
+        return open_store(data_dir, records, copies, allow_degraded=True)
+    except (meta.MetadataUnavailableError, VolumeError):
+        logger.exception("Starting with no volumes for now.")
+        return BlobStore({})
+
+
+def create_app(context: Context, base_domain: str | None) -> App:
+    async def app(scope: Scope, receive: Receive, send: Send) -> None:
+        if scope["type"] == "lifespan":
+            await _run_lifespan(context, receive, send)
+        else:
+            await _handle_http(context, base_domain, scope, receive, send)
+
+    return app
+
+
+async def _run_lifespan(context: Context, receive: Receive, send: Send) -> None:
+    await receive()
+    tasks = [
+        asyncio.create_task(job)
+        for job in (
+            maintenance.run_forever(context.db, context.blobs, context.data_dir),
+            maintenance.run_repairs(context.db, context.blobs),
+            maintenance.run_scrub(context.db, context.blobs),
+            maintenance.run_lifecycle(context.db, context.lifecycle_day_ms),
+            metrics.run_flusher(context.data_dir),
+        )
+    ]
+    await send({"type": "lifespan.startup.complete"})
+    await receive()
+    for task in tasks:
+        task.cancel()
+    for task in tasks:
+        with suppress(asyncio.CancelledError):
+            await task
+    context.db.close()
+    await send({"type": "lifespan.shutdown.complete"})
+
+
+async def _handle_http(
+    context: Context,
+    base_domain: str | None,
+    scope: Scope,
+    receive: Receive,
+    send: Send,
+) -> None:
+    request_id = secrets.token_hex(8)
+    started = time.monotonic()
+    http_request = _parse_http(scope)
+    access = {
+        "request_id": request_id,
+        "method": http_request.method,
+        "path": http_request.raw_path.decode("latin-1"),
+        "operation": "unknown",
+    }
+    route: routing.Route | None = None
+    try:
+        route = routing.route(http_request, base_domain)
+        access["operation"] = route.operation.name
+        response = await _dispatch(context, http_request, route, receive)
+    except S3Error as error:
+        response = _error_response(error, http_request.raw_path, request_id)
+    except ClientDisconnectedError:
+        _log_access(access, _STATUS_CLIENT_GONE, started)
+        return
+    except Exception:
+        logger.exception("Unhandled error in request %s", request_id)
+        response = _error_response(
+            S3Error(errors.INTERNAL_ERROR), http_request.raw_path, request_id
+        )
+
+    response = await _allow_browser_to_read(context, http_request, route, response)
+    if "expect" in http_request.headers and response.status >= _FIRST_ERROR_STATUS:
+        # The client held its body back and will send it after no 100 Continue. If
+        # the connection stayed open, those bytes would parse as the next request.
+        response = replace(response, headers=(*response.headers, _CONNECTION_CLOSE))
+    try:
+        await _send_response(response, http_request.method, request_id, receive, send)
+    finally:
+        # Including a stream that fails midway: its status was already sent.
+        _log_access(access, response.status, started)
+
+
+async def _allow_browser_to_read(
+    context: Context,
+    request: HttpRequest,
+    route: routing.Route | None,
+    response: Response,
+) -> Response:
+    """Add CORS headers, to errors too, or a page couldn't see why a request failed.
+
+    Only a request with an Origin pays for the lookup, which leaves other clients alone.
+    """
+    origin = request.headers.get("origin")
+    if (
+        origin is None
+        or not isinstance(route, BucketRoute | ObjectRoute)
+        or route.operation is Operation.CORS_PREFLIGHT
+    ):
+        return response
+    allowed = await handlers.cors_headers_for(
+        context, request.method, origin, route.bucket
+    )
+    return replace(response, headers=(*response.headers, *allowed))
+
+
+def _parse_http(scope: Scope) -> HttpRequest:
+    headers: dict[str, str] = {}
+    for raw_name, raw_value in scope["headers"]:
+        name = raw_name.decode("latin-1").lower()
+        value = raw_value.decode("latin-1")
+        headers[name] = f"{headers[name]},{value}" if name in headers else value
+    raw_query: bytes = scope["query_string"]
+    return HttpRequest(
+        scope["method"],
+        scope["raw_path"],
+        raw_query,
+        routing.parse_query(raw_query),
+        headers,
+        scope.get("scheme") == "https",
+    )
+
+
+async def _dispatch(
+    context: Context,
+    http_request: HttpRequest,
+    route: routing.Route,
+    receive: Receive,
+) -> Response:
+    unsigned = await _answer_unsigned(context, http_request, route, receive)
+    if unsigned is not None:
+        return unsigned
+
+    if sigv4.is_anonymous(http_request):
+        scope, seed = access.ANONYMOUS, None
+        await _authorize_anonymous(context, route)
+    else:
+        claim = sigv4.parse_claim(http_request)
+        secret, scope = await meta.get_access_key(
+            context.db, claim.credential.access_key
+        )
+        seed = sigv4.verify(http_request, claim, secret, time.time())
+        access.authorize(scope, route)
+    request = Request(
+        http_request.headers,
+        http_request.query,
+        receive,
+        seed,
+        scope,
+        http_request.secure,
+    )
+
+    match route:
+        case ServiceRoute(operation=operation):
+            return await handlers.SERVICE_HANDLERS[operation](context, request)
+        case BucketRoute(operation=operation, bucket=bucket):
+            return await handlers.BUCKET_HANDLERS[operation](context, request, bucket)
+        case ObjectRoute(operation=operation, bucket=bucket, key=key):
+            return await handlers.OBJECT_HANDLERS[operation](
+                context, request, bucket, key
+            )
+
+
+async def _answer_unsigned(
+    context: Context,
+    http_request: HttpRequest,
+    route: routing.Route,
+    receive: Receive,
+) -> Response | None:
+    """Answer a request that has no signature in the usual place, if this is one."""
+    if route.operation is Operation.HEALTH:
+        await meta.ping(context.db)
+        return Response(body=b"ok\n", headers=(("content-type", "text/plain"),))
+    if route.operation is Operation.METRICS:
+        return await _metrics_response(context)
+    if isinstance(route, ServiceRoute):
+        return None
+    if route.operation is Operation.CORS_PREFLIGHT:
+        return await handlers.cors_preflight(
+            context, http_request.headers, route.bucket
+        )
+    if route.operation is Operation.POST_OBJECT:
+        # The form itself holds the signature, which comes with the body.
+        form = Request(
+            http_request.headers,
+            http_request.query,
+            receive,
+            None,
+            access.ANONYMOUS,
+        )
+        return await handlers.post_object(context, form, route.bucket)
+    return None
+
+
+async def _authorize_anonymous(context: Context, route: routing.Route) -> None:
+    """Let through only what an ACL could allow. Reads check the object's own ACL."""
+    refusal = S3Error(errors.ACCESS_DENIED, "Anonymous access is not allowed here.")
+    if isinstance(route, ObjectRoute):
+        if route.operation not in access.ANONYMOUS_OBJECT_OPERATIONS:
+            raise refusal
+    elif not (
+        isinstance(route, BucketRoute)
+        and route.operation in access.ANONYMOUS_BUCKET_OPERATIONS
+        and await meta.bucket_is_public(context.db, route.bucket)
+    ):
+        raise refusal
+
+
+async def _metrics_response(context: Context) -> Response:
+    """Answer without a signature, which Prometheus can't make: it names no bucket."""
+    gauges = await maintenance.current_gauges(context.db, context.data_dir)
+    samples = await asyncio.to_thread(metrics.all_workers, context.data_dir)
+    return Response(
+        body=metrics.render([*samples, *gauges]).encode(),
+        headers=(("content-type", _METRICS_CONTENT_TYPE),),
+    )
+
+
+def _error_response(error: S3Error, raw_path: bytes, request_id: str) -> Response:
+    if error.error.status >= _FIRST_SERVER_ERROR_STATUS:
+        logger.error("%s: %s", request_id, error)
+    body = xml.render_error(
+        error.error.code,
+        error.message,
+        raw_path.decode("latin-1"),
+        request_id,
+    )
+    return Response(
+        error.error.status,
+        (("content-type", "application/xml"), *error.headers),
+        body,
+    )
+
+
+async def _send_response(
+    response: Response, method: str, request_id: str, receive: Receive, send: Send
+) -> None:
+    headers = [
+        (name.encode("latin-1"), value.encode("latin-1"))
+        for name, value in response.headers
+    ]
+    headers.append((b"x-amz-request-id", request_id.encode()))
+    body = response.body
+    has_length = any(name == b"content-length" for name, _ in headers)
+    if (
+        isinstance(body, bytes)
+        and not has_length
+        and response.status not in _STATUSES_WITHOUT_BODY
+    ):
+        headers.append((b"content-length", str(len(body)).encode()))
+    await send({
+        "type": "http.response.start",
+        "status": response.status,
+        "headers": headers,
+    })
+    if isinstance(body, bytes):
+        payload = b"" if method == "HEAD" else body
+        await send({"type": "http.response.body", "body": payload})
+    else:
+        await _stream_body(body, request_id, receive, send)
+
+
+async def _stream_body(
+    body: AsyncGenerator[bytes], request_id: str, receive: Receive, send: Send
+) -> None:
+    """Send a blob stream, and stop reading the disk once the client has gone.
+
+    uvicorn silently drops `send` after a disconnect, so nothing else would stop
+    an abandoned download from reading the whole object.
+    """
+    disconnected = asyncio.create_task(_wait_for_disconnect(receive))
+    sent = 0
+    try:
+        async with aclosing(body) as chunks:
+            async for chunk in chunks:
+                if disconnected.done():
+                    # The status logged is still the one the client was sent.
+                    logger.info("%s: connection lost after %d bytes", request_id, sent)
+                    return
+                sent += len(chunk)
+                await send({
+                    "type": "http.response.body",
+                    "body": chunk,
+                    "more_body": True,
+                })
+        await send({"type": "http.response.body", "body": b""})
+    finally:
+        disconnected.cancel()
+
+
+async def _wait_for_disconnect(receive: Receive) -> None:
+    while (await receive())["type"] != "http.disconnect":
+        pass
+
+
+def _log_access(access: Mapping[str, str], status: int, started: float) -> None:
+    seconds = round(time.monotonic() - started, 6)
+    metrics.counters.observe_request(access["operation"], status, seconds)
+    logger.info("%s", json.dumps({**access, "status": status, "seconds": seconds}))