leblob 0.4.0.dev1__py3-none-any.whl

leblob/cli.py

@@ -0,0 +1,499 @@
+"""The `leblob` command."""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import secrets
+import string
+import sys
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Annotated
+
+import cyclopts
+import uvicorn
+
+from leblob import asgi, maintenance, meta, metrics
+from leblob.blobstore import (
+    VOLUME_FILE,
+    BlobStore,
+    Volume,
+    VolumeError,
+    create_volume,
+    open_store,
+    open_volume,
+)
+from leblob.meta import Database
+from leblob.s3.access import Level
+from leblob.s3.errors import S3Error
+
+if TYPE_CHECKING:
+    from collections.abc import Awaitable, Callable
+    from typing import Concatenate
+
+    from leblob.s3.access import Scope
+
+_DEFAULT_DATA_DIR = Path("./data")
+_ACCESS_KEY_ALPHABET = string.ascii_uppercase + string.digits
+_ACCESS_KEY_LENGTH = 20
+_SECRET_BYTES = 30
+_FOLLOWS_IN = "a couple of minutes"
+
+DataDir = Annotated[Path, cyclopts.Parameter(help="Directory holding all stored data.")]
+GrantLevel = Annotated[Level, cyclopts.Parameter(help="read, write or admin.")]
+
+app = cyclopts.App(name="leblob", help="An S3-compatible object store.")
+key_app = cyclopts.App(name="key", help="Manage access keys.")
+app.command(key_app)
+volume_app = cyclopts.App(name="volume", help="Manage the disks blobs are stored on.")
+app.command(volume_app)
+
+
+@cyclopts.Parameter(name="*")
+@dataclass(frozen=True, kw_only=True)
+class ServeOptions:
+    """Options of `leblob serve`.
+
+    Parameters
+    ----------
+    data_dir
+        Directory holding all stored data.
+    host
+        Address to listen on.
+    port
+        Port to listen on.
+    workers
+        Number of worker processes; defaults to the number of CPUs, 4 at most. Each
+        takes about 30 MiB. Writes wait on the disk and gain nothing from more; reads
+        do.
+    domain
+        Base domain enabling virtual-hosted addressing (bucket.domain).
+    ssl_certfile
+        TLS certificate, to serve HTTPS directly.
+    ssl_keyfile
+        TLS private key.
+    shutdown_grace
+        Seconds to let requests finish on shutdown. Without a limit, one client
+        that stops reading would prevent the server from ever exiting.
+    allow_sse_c_over_http
+        Accept encryption keys over plain HTTP. Only behind a proxy that terminates TLS,
+        or for tests: otherwise the key can be read off the network.
+    lifecycle_day_seconds
+        How long a day lasts for lifecycle rules. For tests, which can't wait for one.
+    """
+
+    data_dir: Path = _DEFAULT_DATA_DIR
+    host: str = "127.0.0.1"
+    port: int = 9000
+    workers: int | None = None
+    domain: str | None = None
+    ssl_certfile: Path | None = None
+    ssl_keyfile: Path | None = None
+    shutdown_grace: int = 20
+    lifecycle_day_seconds: int = meta.DAY_MS // 1000
+    allow_sse_c_over_http: bool = False
+
+
+_DEFAULT_MOST_WORKERS = 4
+
+
+@app.command
+def serve(options: ServeOptions | None = None) -> None:
+    """Run the S3 server."""
+    options = options or ServeOptions()
+    _prepare(options.data_dir)
+    # Fail here, not in every worker. With several copies, it can run without a volume.
+    _open_store(options.data_dir, allow_degraded=_copies(options.data_dir) > 1)
+    asyncio.run(_create_first_key_if_needed(options.data_dir))
+
+    metrics.clear(options.data_dir)
+    os.environ[asgi.ENV_DATA_DIR] = str(options.data_dir.resolve())
+    os.environ[asgi.ENV_LIFECYCLE_DAY_SECONDS] = str(options.lifecycle_day_seconds)
+    if options.allow_sse_c_over_http:
+        os.environ[asgi.ENV_SSE_OVER_HTTP] = "1"
+    else:
+        os.environ.pop(asgi.ENV_SSE_OVER_HTTP, None)
+    if options.domain:
+        os.environ[asgi.ENV_BASE_DOMAIN] = options.domain
+    else:
+        os.environ.pop(asgi.ENV_BASE_DOMAIN, None)
+
+    uvicorn.run(
+        "leblob.asgi:create_app_from_env",
+        factory=True,
+        host=options.host,
+        port=options.port,
+        workers=options.workers or min(os.cpu_count() or 1, _DEFAULT_MOST_WORKERS),
+        http="httptools",
+        loop="uvloop",
+        lifespan="on",
+        ws="none",
+        access_log=False,
+        ssl_certfile=options.ssl_certfile,
+        ssl_keyfile=options.ssl_keyfile,
+        timeout_graceful_shutdown=options.shutdown_grace,
+    )
+
+
+@key_app.command(name="create")
+def key_create(*, data_dir: DataDir = _DEFAULT_DATA_DIR, scoped: bool = False) -> None:
+    """Create an access key and print it. The secret isn't shown again.
+
+    Parameters
+    ----------
+    scoped
+        Give the key no rights until `key grant` gives it some, instead of full access.
+    """
+    _prepare(data_dir)
+    access_key, secret = asyncio.run(_create_key(data_dir, scoped=scoped))
+    print(f"access key: {access_key}")
+    print(f"secret key: {secret}")
+
+
+@key_app.command(name="list")
+def key_list(*, data_dir: DataDir = _DEFAULT_DATA_DIR) -> None:
+    """List access keys and what each may do."""
+    _open_existing(data_dir)
+    keys = asyncio.run(_with_db(data_dir, meta.list_access_keys))
+    for access_key, scope in keys.items():
+        print(f"{access_key}  {_describe_scope(scope)}")
+
+
+def _describe_scope(scope: Scope) -> str:
+    if scope.buckets is None:
+        return "full access"
+    grants = ", ".join(
+        f"{bucket}: {level.name.lower()}"
+        for bucket, level in sorted(scope.buckets.items())
+    )
+    return grants or "no rights"
+
+
+@key_app.command(name="grant")
+def key_grant(
+    access_key: str,
+    bucket: str,
+    level: GrantLevel,
+    /,
+    *,
+    data_dir: DataDir = _DEFAULT_DATA_DIR,
+) -> None:
+    """Give a scoped key a level on a bucket, replacing any it had."""
+    _open_existing(data_dir)
+    try:
+        asyncio.run(_with_db(data_dir, meta.grant_bucket, access_key, bucket, level))
+    except S3Error as error:
+        sys.exit(f"leblob: {access_key} on {bucket}: {error.message}")
+
+
+@key_app.command(name="revoke")
+def key_revoke(
+    access_key: str, bucket: str, /, *, data_dir: DataDir = _DEFAULT_DATA_DIR
+) -> None:
+    """Remove what a scoped key was granted on a bucket."""
+    _open_existing(data_dir)
+    if not asyncio.run(_with_db(data_dir, meta.revoke_bucket, access_key, bucket)):
+        sys.exit(f"leblob: {access_key} has no grant on {bucket}")
+
+
+@key_app.command(name="delete")
+def key_delete(access_key: str, /, *, data_dir: DataDir = _DEFAULT_DATA_DIR) -> None:
+    """Delete an access key."""
+    _open_existing(data_dir)
+    if not asyncio.run(_with_db(data_dir, meta.delete_access_key, access_key)):
+        sys.exit(f"leblob: no such access key: {access_key}")
+
+
+@app.command
+def fsck(
+    *, data_dir: DataDir = _DEFAULT_DATA_DIR, delete_orphans: bool = False
+) -> None:
+    """Compare the metadata with the disk; exit non-zero if they disagree.
+
+    Parameters
+    ----------
+    delete_orphans
+        Remove blob files that have no metadata and are over a day old.
+    """
+    _open_existing(data_dir)
+    blobs = _open_store(data_dir)
+    report = asyncio.run(_with_db(data_dir, _run_fsck, blobs))
+    if report.under_replicated:
+        print(f"blobs with too few copies: {len(report.under_replicated)}")
+    for blob_id in report.lost:
+        print(f"lost, no copy left: {blob_id}")
+    for label, files in (
+        ("orphan", report.orphans),
+        ("recent orphan (may be in flight)", report.young_orphans),
+        ("missing", report.missing),
+        ("wrong size", report.wrong_size),
+    ):
+        for file in files:
+            print(f"{label}: {file.blob_id} on volume {file.volume_id}")
+    if delete_orphans:
+        for file in report.orphans:
+            blobs.delete(file.blob_id, (file.volume_id,))
+        print(f"deleted {len(report.orphans)} orphans")
+    damaged = report.missing or report.wrong_size or report.lost
+    if damaged or (report.orphans and not delete_orphans):
+        sys.exit(1)
+    print("clean")
+
+
+@volume_app.command(name="add")
+def volume_add(path: Path, /, *, data_dir: DataDir = _DEFAULT_DATA_DIR) -> None:
+    """Add a volume for new blobs. A running server follows within a couple of minutes.
+
+    With one copy of each blob, losing a volume loses the objects with blobs on it:
+    see `volume copies`.
+
+    Parameters
+    ----------
+    path
+        An existing empty directory, usually the mount point of a disk.
+    """
+    _open_existing(data_dir)
+    root = path.resolve()
+    identity = secrets.token_hex(16)
+    try:
+        create_volume(root, identity)
+    except (VolumeError, OSError) as error:
+        sys.exit(f"leblob: {error}")
+    volume_id = asyncio.run(_with_db(data_dir, meta.add_volume, identity, str(root)))
+    print(f"added volume {volume_id} at {root}; a server uses it within {_FOLLOWS_IN}")
+
+
+@volume_app.command(name="copies")
+def volume_copies(count: int, /, *, data_dir: DataDir = _DEFAULT_DATA_DIR) -> None:
+    """Set how many volumes hold each blob. A server follows within a couple of minutes.
+
+    A running server adds the missing copies of blobs already stored, and leaves extra
+    ones in place. Each copy needs its own disk to protect against losing one.
+
+    Parameters
+    ----------
+    count
+        Copies of each new blob, at most the number of volumes.
+    """
+    _open_existing(data_dir)
+    records = meta.load_volumes(data_dir / asgi.META_FILE)
+    if not 1 <= count <= len(records):
+        sys.exit(f"leblob: {count} copies can't fit on {len(records)} volumes")
+    asyncio.run(_with_db(data_dir, meta.set_copies, count))
+    print(f"blobs get {count} copies; a server uses it within {_FOLLOWS_IN}")
+    devices = {(data_dir / record.path).stat().st_dev for record in records}
+    if len(devices) < count:
+        print(
+            f"warning: the volumes are on {len(devices)} filesystems, fewer than that"
+        )
+
+
+@volume_app.command(name="list")
+def volume_list(*, data_dir: DataDir = _DEFAULT_DATA_DIR) -> None:
+    """List volumes, with how full each is and any that can't be used."""
+    _open_existing(data_dir)
+    print(f"blobs get {_copies(data_dir)} copies")
+    print(f"repairs queued: {asyncio.run(_with_db(data_dir, meta.repairs_queued))}")
+    counts = asyncio.run(_with_db(data_dir, meta.blobs_per_volume))
+    for record in meta.load_volumes(data_dir / asgi.META_FILE):
+        count = counts.get(record.volume_id, 0)
+        try:
+            free = open_volume(data_dir, record).free_bytes() / 2**30
+        except VolumeError as error:
+            print(f"{record.volume_id}  {record.path}  {count} blobs  {error}")
+        else:
+            state = "  draining" if record.draining else ""
+            print(
+                f"{record.volume_id}  {record.path}  {count} blobs"
+                f"  {free:.1f} GiB free{state}"
+            )
+
+
+@volume_app.command(name="drain")
+def volume_drain(volume_id: int, /, *, data_dir: DataDir = _DEFAULT_DATA_DIR) -> None:
+    """Stop using a volume, and have a running server move its blobs elsewhere.
+
+    `volume list` shows when it is empty, and `volume forget` then removes it.
+    """
+    _open_existing(data_dir)
+    try:
+        asyncio.run(_with_db(data_dir, meta.drain_volume, volume_id))
+    except meta.VolumeChangeError as error:
+        sys.exit(f"leblob: {error}")
+    print(f"draining volume {volume_id}")
+
+
+@volume_app.command(name="forget")
+def volume_forget(
+    volume_id: int,
+    /,
+    *,
+    data_dir: DataDir = _DEFAULT_DATA_DIR,
+    disk_lost: bool = False,
+    accept_loss: bool = False,
+) -> None:
+    """Remove a volume that is empty, or whose disk is gone for good.
+
+    Parameters
+    ----------
+    disk_lost
+        The disk is gone: forget the copies on it, for a running server to replace.
+    accept_loss
+        Go ahead even though some blobs have no other copy, losing their objects.
+    """
+    _open_existing(data_dir)
+    try:
+        asyncio.run(
+            _with_db(
+                data_dir,
+                meta.forget_volume,
+                volume_id,
+                disk_lost=disk_lost,
+                accept_loss=accept_loss,
+            )
+        )
+    except meta.VolumeChangeError as error:
+        sys.exit(f"leblob: {error}")
+    print(f"forgot volume {volume_id}")
+
+
+@app.command
+def scrub(*, data_dir: DataDir = _DEFAULT_DATA_DIR) -> None:
+    """Check every copy against its MD5 now; a running server does this slowly anyway.
+
+    A bad copy is removed, and a running server replaces it. Exits non-zero if a blob's
+    only copy is bad.
+    """
+    _open_existing(data_dir)
+    blobs = _open_store(data_dir, allow_degraded=True)
+    report = asyncio.run(_with_db(data_dir, maintenance.scrub_everything, blobs))
+    print(f"checked {report.checked} copies, removed {report.removed} bad ones")
+    for blob_id in report.unrecoverable:
+        print(f"only copy is bad: {blob_id}")
+    if report.unrecoverable:
+        sys.exit(1)
+
+
+@app.command
+def backup(destination: Path, /, *, data_dir: DataDir = _DEFAULT_DATA_DIR) -> None:
+    """Copy the metadata to a new file, while the server may be running.
+
+    The file holds the access keys' secrets. Without it, losing the metadata loses
+    every object, because blob files have random names. It doesn't hold the blobs.
+
+    Parameters
+    ----------
+    destination
+        File to create. Put it on another disk.
+    """
+    _open_existing(data_dir)
+    try:
+        meta.backup(data_dir / asgi.META_FILE, destination)
+    except (meta.BackupError, OSError) as error:
+        sys.exit(f"leblob: {error}")
+    print(f"backed up to {destination}")
+
+
+@app.command
+def restore(backup_file: Path, /, *, data_dir: DataDir = _DEFAULT_DATA_DIR) -> None:
+    """Put a backup in place of lost metadata. Stop the server first.
+
+    Objects written since the backup are lost, and their files become orphans. Blobs
+    collected since then are missing, which can only happen once the backup is older
+    than the collector's grace period, an hour. `fsck` reports both.
+
+    Parameters
+    ----------
+    backup_file
+        File made by `leblob backup`.
+    """
+    data_dir.mkdir(parents=True, exist_ok=True)
+    data_dir.chmod(0o700)
+    try:
+        meta.restore(backup_file, data_dir / asgi.META_FILE)
+    except (meta.BackupError, OSError) as error:
+        sys.exit(f"leblob: {error}")
+    _prepare(data_dir)  # An older backup needs migrating, and a new disk its layout.
+    _open_store(data_dir)  # A backup of another store must not run on these files.
+    print(f"restored from {backup_file}; run `leblob fsck` to compare it with the disk")
+
+
+async def _run_fsck(db: Database, blobs: BlobStore) -> maintenance.FsckReport:
+    return await maintenance.fsck(db, blobs, time.time())
+
+
+def _prepare(data_dir: Path) -> None:
+    """Create the directory layout and schema, before any worker can start."""
+    data_dir.mkdir(parents=True, exist_ok=True)
+    data_dir.chmod(0o700)  # meta.db holds secrets, even if the directory existed.
+    meta.initialize(data_dir / asgi.META_FILE)
+    _prepare_first_volume(data_dir)
+
+
+def _open_existing(data_dir: Path) -> None:
+    """Refuse a path that holds no store: reporting on it would be false reassurance."""
+    if not (data_dir / asgi.META_FILE).exists():
+        sys.exit(f"leblob: no store found in {data_dir}")
+    meta.initialize(data_dir / asgi.META_FILE)
+    _prepare_first_volume(data_dir)
+
+
+def _prepare_first_volume(data_dir: Path) -> None:
+    """Lay out the data directory, whose blobs need no disk of their own to appear."""
+    first = meta.load_volumes(data_dir / asgi.META_FILE)[0]
+    if not (data_dir / VOLUME_FILE).exists():
+        volume = Volume(data_dir, first.volume_id)
+        volume.initialize()
+        volume.claim(first.identity)
+
+
+def _copies(data_dir: Path) -> int:
+    return meta.load_copies(data_dir / asgi.META_FILE)
+
+
+def _open_store(data_dir: Path, *, allow_degraded: bool = False) -> BlobStore:
+    """Refuse volumes that aren't what the metadata expects, and say which."""
+    records = meta.load_volumes(data_dir / asgi.META_FILE)
+    try:
+        return open_store(
+            data_dir, records, _copies(data_dir), allow_degraded=allow_degraded
+        )
+    except VolumeError as error:
+        sys.exit(f"leblob: {error}")
+
+
+async def _with_db[**P, R](
+    data_dir: Path,
+    function: Callable[Concatenate[Database, P], Awaitable[R]],
+    *args: P.args,
+    **kwargs: P.kwargs,
+) -> R:
+    db = Database(data_dir / asgi.META_FILE)
+    try:
+        return await function(db, *args, **kwargs)
+    finally:
+        db.close()
+
+
+async def _create_key(data_dir: Path, *, scoped: bool = False) -> tuple[str, str]:
+    access_key = "".join(
+        secrets.choice(_ACCESS_KEY_ALPHABET) for _ in range(_ACCESS_KEY_LENGTH)
+    )
+    secret = secrets.token_urlsafe(_SECRET_BYTES)
+    await _with_db(data_dir, meta.add_access_key, access_key, secret, scoped=scoped)
+    return access_key, secret
+
+
+async def _create_first_key_if_needed(data_dir: Path) -> None:
+    """Make a fresh store usable at once: no key means nobody could ever connect."""
+    if await _with_db(data_dir, meta.list_access_keys):
+        return
+    access_key, secret = await _create_key(data_dir)
+    print(
+        "Created the first access key; the secret isn't shown again:\n"
+        f"  access key: {access_key}\n"
+        f"  secret key: {secret}",
+        flush=True,
+    )