leblob 0.4.0.dev1__py3-none-any.whl

leblob/metrics.py

@@ -0,0 +1,226 @@
+"""Counters for `/-/metrics`, added up across worker processes through files.
+
+A scrape reaches one worker, chosen by the kernel, so what each counts alone would jump
+around between scrapes. Instead each worker writes its counters to a file every few
+seconds and on each scrape, and the one answering adds up the files. A worker that dies
+leaves its file, so totals never go down until the server restarts, when Prometheus
+expects it.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import os
+import shutil
+import threading
+from bisect import bisect_left
+from collections import defaultdict
+from contextlib import suppress
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable, Mapping
+    from pathlib import Path
+
+type Labels = tuple[tuple[str, str], ...]
+type Sample = tuple[str, Labels, float]
+
+logger = logging.getLogger(__name__)
+
+_DIRECTORY = "metrics"
+_FLUSH_SECONDS = 5
+_REQUESTS = "leblob_http_requests_total"
+_LATENCY = "leblob_http_request_seconds"
+_LATENCY_BUCKETS = (0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 30.0)
+
+BLOBS_COLLECTED = "leblob_blobs_collected_total"
+COPIES_REPAIRED = "leblob_copies_repaired_total"
+COPIES_SCRUBBED = "leblob_copies_scrubbed_total"
+BAD_COPIES_REMOVED = "leblob_bad_copies_removed_total"
+LIFECYCLE_OBJECTS_EXPIRED = "leblob_lifecycle_objects_expired_total"
+LIFECYCLE_VERSIONS_ERASED = "leblob_lifecycle_versions_erased_total"
+LIFECYCLE_UPLOADS_ABORTED = "leblob_lifecycle_uploads_aborted_total"
+ONLY_COPIES_BAD = "leblob_only_copies_bad_total"
+VOLUME_FAILURES = "leblob_volume_failures_total"
+REPAIRS_QUEUED = "leblob_repairs_queued"
+BLOBS_AWAITING_COLLECTION = "leblob_blobs_awaiting_collection"
+COPIES = "leblob_copies_wanted"
+VOLUME_UP = "leblob_volume_up"
+VOLUME_DRAINING = "leblob_volume_draining"
+VOLUME_FREE_BYTES = "leblob_volume_free_bytes"
+
+_FAMILIES: Mapping[str, tuple[str, str]] = {
+    _REQUESTS: ("counter", "Requests answered, by S3 operation and status."),
+    _LATENCY: ("histogram", "Seconds to answer a request, by S3 operation."),
+    BLOBS_COLLECTED: ("counter", "Blobs deleted after their grace period."),
+    COPIES_REPAIRED: ("counter", "Copies of blobs restored by repair."),
+    COPIES_SCRUBBED: ("counter", "Copies checked against their MD5."),
+    BAD_COPIES_REMOVED: ("counter", "Bad copies scrub removed for repair to replace."),
+    ONLY_COPIES_BAD: ("counter", "Times scrub found a blob's only copy bad."),
+    VOLUME_FAILURES: ("counter", "I/O errors on a volume."),
+    LIFECYCLE_OBJECTS_EXPIRED: ("counter", "Objects expired by lifecycle rules."),
+    LIFECYCLE_VERSIONS_ERASED: ("counter", "Versions erased by lifecycle rules."),
+    LIFECYCLE_UPLOADS_ABORTED: ("counter", "Uploads aborted by lifecycle rules."),
+    REPAIRS_QUEUED: ("gauge", "Blobs waiting for repair."),
+    BLOBS_AWAITING_COLLECTION: ("gauge", "Blobs queued for deletion."),
+    COPIES: ("gauge", "Copies each blob should have."),
+    VOLUME_UP: ("gauge", "Whether a volume can be used, 1 or 0."),
+    VOLUME_DRAINING: ("gauge", "Whether a volume is being emptied, 1 or 0."),
+    VOLUME_FREE_BYTES: ("gauge", "Bytes free on a volume."),
+}
+
+
+@dataclass(slots=True)
+class _Latency:
+    """One operation's histogram, kept in separate buckets until a scrape adds them."""
+
+    buckets: list[int] = field(
+        default_factory=lambda: [0] * (len(_LATENCY_BUCKETS) + 1)
+    )
+    seconds: float = 0.0
+    count: int = 0
+
+    def samples(self, operation: str) -> list[Sample]:
+        result: list[Sample] = []
+        below = 0
+        for bound, seen in zip((*_LATENCY_BUCKETS, "+Inf"), self.buckets, strict=True):
+            below += seen
+            labels = (("le", str(bound)), ("operation", operation))
+            result.append((f"{_LATENCY}_bucket", labels, below))
+        by_operation = (("operation", operation),)
+        return [
+            *result,
+            (f"{_LATENCY}_sum", by_operation, self.seconds),
+            (f"{_LATENCY}_count", by_operation, self.count),
+        ]
+
+
+class Counters:
+    """This process's counters. Worker threads and the event loop both add to them."""
+
+    def __init__(self) -> None:
+        self._values: dict[tuple[str, Labels], float] = defaultdict(float)
+        self._requests: dict[tuple[str, int], int] = defaultdict(int)
+        self._latency: dict[str, _Latency] = defaultdict(_Latency)
+        self._lock = threading.Lock()
+
+    def add(self, name: str, amount: float = 1, **labels: str) -> None:
+        key = name, tuple(sorted(labels.items()))
+        with self._lock:
+            self._values[key] += amount
+
+    def observe_request(self, operation: str, status: int, seconds: float) -> None:
+        """Count a request. On every request, so this costs little until a scrape."""
+        bucket = bisect_left(_LATENCY_BUCKETS, seconds)
+        with self._lock:
+            self._requests[operation, status] += 1
+            latency = self._latency[operation]
+            latency.buckets[bucket] += 1
+            latency.seconds += seconds
+            latency.count += 1
+
+    def samples(self) -> list[Sample]:
+        with self._lock:
+            samples: list[Sample] = [
+                (name, labels, value) for (name, labels), value in self._values.items()
+            ]
+            for (operation, status), count in self._requests.items():
+                labels = (("operation", operation), ("status", str(status)))
+                samples.append((_REQUESTS, labels, count))
+            for operation, latency in self._latency.items():
+                samples += latency.samples(operation)
+            return samples
+
+
+counters = Counters()
+
+
+def clear(data_dir: Path) -> None:
+    """Forget the workers of an earlier run, before any worker starts."""
+    shutil.rmtree(data_dir / _DIRECTORY, ignore_errors=True)
+
+
+def _worker_file(data_dir: Path) -> Path:
+    return data_dir / _DIRECTORY / f"{os.getpid()}.json"
+
+
+def flush(data_dir: Path) -> None:
+    """Replace this worker's file. It needn't survive a crash, so there is no fsync."""
+    path = _worker_file(data_dir)
+    path.parent.mkdir(exist_ok=True)
+    partial = path.with_suffix(".partial")
+    partial.write_text(json.dumps(counters.samples()), encoding="utf-8")
+    partial.rename(path)
+
+
+async def run_flusher(data_dir: Path) -> None:
+    """Write this worker's counters regularly. A full disk mustn't stop it for good."""
+    try:
+        while True:
+            await asyncio.sleep(_FLUSH_SECONDS)
+            try:
+                await asyncio.to_thread(flush, data_dir)
+            except OSError:
+                logger.exception("Can't write metrics; they will be written later.")
+    finally:
+        # Shutting down: the last few seconds count too.
+        with suppress(OSError):
+            flush(data_dir)
+
+
+def all_workers(data_dir: Path) -> list[Sample]:
+    """Every worker's counters, from their files, this worker's written just now.
+
+    Only ever from the files, so that any worker gives the same answer. Adding this
+    worker's live counts to the others' older files would make totals go down when the
+    next scrape reached another worker, which Prometheus takes for a restart.
+    """
+    with suppress(OSError):  # On a full disk, its file is simply a little old.
+        flush(data_dir)
+    samples: list[Sample] = []
+    directory = data_dir / _DIRECTORY
+    if directory.is_dir():
+        for path in directory.glob("*.json"):
+            samples += [
+                (name, tuple(map(tuple, labels)), value)
+                for name, labels, value in json.loads(path.read_text("utf-8"))
+            ]
+    return samples
+
+
+def _family_of(name: str) -> str:
+    for suffix in ("_bucket", "_sum", "_count"):
+        if name.endswith(suffix) and name.removesuffix(suffix) == _LATENCY:
+            return _LATENCY
+    return name
+
+
+def _bucket_order(labels: Labels) -> tuple[Labels, float]:
+    """Sort buckets by their bound, which Prometheus requires."""
+    rest = tuple(item for item in labels if item[0] != "le")
+    bounds = [float(value) for name, value in labels if name == "le"]
+    return rest, bounds[0] if bounds else 0.0
+
+
+def render(samples: Iterable[Sample]) -> str:
+    """Add up samples into the Prometheus text format."""
+    totals: dict[tuple[str, Labels], float] = defaultdict(float)
+    for name, labels, value in samples:
+        totals[name, labels] += value
+    by_family: dict[str, list[tuple[str, Labels, float]]] = defaultdict(list)
+    for (name, labels), value in totals.items():
+        by_family[_family_of(name)].append((name, labels, value))
+    lines = []
+    for family, members in sorted(by_family.items()):
+        kind, description = _FAMILIES[family]
+        lines += [f"# HELP {family} {description}", f"# TYPE {family} {kind}"]
+        members.sort(key=lambda member: (_bucket_order(member[1]), member[0]))
+        for name, labels, value in members:
+            text = ",".join(f'{key}="{item}"' for key, item in labels)
+            selector = f"{{{text}}}" if text else ""
+            # Not `g`, whose 6 digits would round a disk's free bytes.
+            lines.append(f"{name}{selector} {value:.17g}")
+    return "\n".join(lines) + "\n"

leblob/model.py

@@ -0,0 +1,339 @@
+"""Plain data shared by the pure core and the storage layers."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import StrEnum
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+
+NULL_VERSION_ID = "null"
+
+
+@dataclass(frozen=True, slots=True)
+class HttpRequest:
+    """The parts of a request that routing and signatures depend on."""
+
+    method: str
+    raw_path: bytes
+    raw_query: bytes
+    query: Mapping[str, str]
+    headers: Mapping[str, str]
+    secure: bool = False  # Whether it came over TLS.
+
+
+@dataclass(frozen=True, slots=True)
+class ObjectRef:
+    """A key in a bucket, whichever version of it."""
+
+    bucket: str
+    key: str
+
+
+@dataclass(frozen=True, slots=True)
+class ByteSpan:
+    """A range of an object's bytes, as its client knows them."""
+
+    start: int
+    length: int
+
+
+@dataclass(frozen=True, slots=True)
+class UploadRef:
+    """What identifies a multipart upload. All three parts must match."""
+
+    upload_id: str
+    bucket: str
+    key: str
+
+
+@dataclass(frozen=True, slots=True)
+class Blob:
+    """An immutable run of bytes on disk, named by a random ID."""
+
+    blob_id: str
+    size: int
+    md5: str
+    volume_ids: tuple[int, ...]
+
+
+class ChecksumType(StrEnum):
+    """How a multipart object's checksum relates to its parts."""
+
+    COMPOSITE = "COMPOSITE"
+    FULL_OBJECT = "FULL_OBJECT"
+
+
+@dataclass(frozen=True, slots=True)
+class KnownBlob:
+    """What the metadata says about a blob file."""
+
+    size: int
+    queued_for_collection: bool
+    volume_ids: tuple[int, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class BlobFile:
+    """A blob's file on one volume, as `fsck` reports it."""
+
+    volume_id: int
+    blob_id: str
+
+
+@dataclass(frozen=True, slots=True)
+class VolumeRecord:
+    """A volume as the metadata knows it. Its path is relative to the data directory."""
+
+    volume_id: int
+    identity: str
+    path: str
+    draining: bool = False
+
+
+@dataclass(frozen=True, slots=True)
+class RepairWork:
+    """A blob queued for repair, and what the metadata wants of it right now."""
+
+    blob: Blob
+    copies: int
+    draining: frozenset[int]
+
+
+@dataclass(frozen=True, slots=True)
+class Checksum:
+    """A checksum, in the form S3 headers carry it."""
+
+    algorithm: str
+    value: str
+
+    @property
+    def checksum_type(self) -> ChecksumType:
+        """Tell them apart by the `-N` a composite value ends in, not valid base64."""
+        return ChecksumType.COMPOSITE if "-" in self.value else ChecksumType.FULL_OBJECT
+
+
+@dataclass(frozen=True, slots=True)
+class ChecksumPlan:
+    """The checksum a multipart upload declared its object will have."""
+
+    algorithm: str
+    checksum_type: ChecksumType
+
+
+@dataclass(frozen=True, slots=True)
+class ObjectRecord:
+    """The current version of an object: its metadata and ordered blobs."""
+
+    key: str
+    size: int
+    etag: str
+    modified_at: int
+    headers: Mapping[str, str]
+    checksum: Checksum | None
+    blobs: tuple[Blob, ...]
+    version_id: str = NULL_VERSION_ID
+    tags: Mapping[str, str] = field(default_factory=dict)
+    public: bool = False
+    sse_check: str | None = None  # Set if encrypted with a key the client keeps.
+
+
+@dataclass(frozen=True, slots=True)
+class VersionSummary:
+    """The fields of an object version or delete marker that a listing returns."""
+
+    key: str
+    version_id: str
+    rank: int
+    is_delete_marker: bool
+    size: int
+    etag: str
+    modified_at: int
+    is_latest: bool = False
+    tags: Mapping[str, str] = field(default_factory=dict)
+
+
+class Versioning(StrEnum):
+    """A bucket's versioning state. A bucket that never had any has none."""
+
+    ENABLED = "Enabled"
+    SUSPENDED = "Suspended"
+
+
+@dataclass(frozen=True, slots=True)
+class DeleteTarget:
+    """What a bulk delete names: a key, and perhaps one version of it."""
+
+    key: str
+    version_id: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class DeleteOutcome:
+    """What a delete did. Unversioned buckets have no version to report."""
+
+    key: str
+    version_id: str | None = None
+    is_delete_marker: bool = False
+
+
+@dataclass(frozen=True, slots=True)
+class ObjectSummary:
+    """The fields of an object that a listing returns."""
+
+    key: str
+    size: int
+    etag: str
+    modified_at: int
+
+
+@dataclass(frozen=True, slots=True)
+class BucketRecord:
+    name: str
+    created_at: int
+
+
+@dataclass(frozen=True, slots=True)
+class Part:
+    """One uploaded part of a multipart upload."""
+
+    number: int
+    blob: Blob
+    checksum: Checksum | None
+    modified_at: int
+
+
+@dataclass(frozen=True, slots=True)
+class Upload:
+    """A multipart upload in progress."""
+
+    upload_id: str
+    bucket: str
+    key: str
+    headers: Mapping[str, str]
+    created_at: int
+    checksum: ChecksumPlan | None
+    tags: Mapping[str, str] = field(default_factory=dict)
+    public: bool = False
+    sse_check: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class UploadExtras:
+    """What an upload promises the object it will become, beyond its headers."""
+
+    tags: Mapping[str, str] = field(default_factory=dict)
+    public: bool = False
+    sse_check: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class RequestedPart:
+    """A part a client asks to assemble, and what it believes about that part."""
+
+    number: int
+    etag: str
+    checksums: tuple[Checksum, ...] = ()
+
+
+@dataclass(frozen=True, slots=True)
+class CompletionRequest:
+    """A CompleteMultipartUpload request: the parts, and claims about the object."""
+
+    parts: tuple[RequestedPart, ...]
+    checksum: Checksum | None = None
+    checksum_type: str | None = None
+    object_size: int | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class CompletedUpload:
+    """The result of completing an upload, which a retry must see unchanged."""
+
+    etag: str
+    checksum: Checksum | None
+    version_id: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class BlobSlice:
+    """A byte range inside one blob."""
+
+    blob: Blob
+    offset: int
+    length: int
+
+
+@dataclass(frozen=True, slots=True)
+class RuleScope:
+    """What a lifecycle rule applies to, and which form the client wrote it in."""
+
+    prefix: str | None = None
+    legacy_prefix: bool = False  # Written as <Prefix>, rather than inside <Filter>.
+    larger_than: int | None = None
+    smaller_than: int | None = None
+    tags: tuple[tuple[str, str], ...] = ()
+
+    def looks_inside_objects(self) -> bool:
+        """Whether it needs more than a key, as an upload has so far."""
+        return (
+            self.larger_than is not None
+            or self.smaller_than is not None
+            or bool(self.tags)
+        )
+
+
+@dataclass(frozen=True, slots=True)
+class RuleActions:
+    """What a lifecycle rule does. The default, doing nothing, isn't a valid rule."""
+
+    expire_days: int | None = None
+    expire_at: int | None = None  # Milliseconds since the epoch.
+    expire_lone_markers: bool = False
+    noncurrent_days: int | None = None
+    keep_noncurrent: int = 0
+    abort_upload_days: int | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class LifecycleRule:
+    rule_id: str
+    enabled: bool
+    scope: RuleScope
+    actions: RuleActions
+
+
+@dataclass(frozen=True, slots=True)
+class KeyVerdict:
+    """What lifecycle rules say to do with one key."""
+
+    expire_current: bool = False
+    erase: tuple[str, ...] = ()
+
+    @property
+    def removals(self) -> int:
+        return int(self.expire_current) + len(self.erase)
+
+
+@dataclass(frozen=True, slots=True)
+class LifecycleReport:
+    """What one batch of lifecycle work removed."""
+
+    expired: int = 0
+    erased: int = 0
+    aborted: int = 0
+
+
+@dataclass(frozen=True, slots=True)
+class CorsRule:
+    """Which browser pages may send which requests to a bucket."""
+
+    origins: tuple[str, ...]
+    methods: tuple[str, ...]
+    allowed_headers: tuple[str, ...] = ()
+    expose: tuple[str, ...] = ()
+    rule_id: str | None = None
+    max_age: int | None = None

leblob/s3/__init__.py

@@ -0,0 +1 @@
+"""The S3 protocol as pure functions: no sockets, files or databases here."""

leblob/s3/access.py

@@ -0,0 +1,110 @@
+"""What an access key may do: a pure decision, taken once per request."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import IntEnum
+from typing import TYPE_CHECKING
+
+from leblob.s3 import errors
+from leblob.s3.errors import S3Error
+from leblob.s3.routing import BucketRoute, ObjectRoute, Operation
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+    from leblob.s3.routing import Route
+
+
+class Level(IntEnum):
+    """A right on one bucket. Each level includes the ones below it."""
+
+    READ = 1
+    WRITE = 2
+    ADMIN = 3
+
+
+@dataclass(frozen=True, slots=True)
+class Scope:
+    """The buckets a scoped key was granted, or None for a key with full access."""
+
+    buckets: Mapping[str, Level] | None = None
+
+    def allows(self, bucket: str, level: Level) -> bool:
+        if self.buckets is None:
+            return True
+        granted = self.buckets.get(bucket)
+        return granted is not None and granted >= level
+
+    def sees(self, bucket: str) -> bool:
+        return self.buckets is None or bucket in self.buckets
+
+
+FULL = Scope()
+ANONYMOUS = Scope(buckets={})  # Sees nothing, unless an ACL says otherwise.
+ANONYMOUS_BUCKET_OPERATIONS = frozenset({
+    Operation.HEAD_BUCKET,
+    Operation.LIST_OBJECTS,
+    Operation.LIST_OBJECTS_V2,
+    Operation.LIST_OBJECT_VERSIONS,
+})
+ANONYMOUS_OBJECT_OPERATIONS = frozenset({Operation.GET_OBJECT, Operation.HEAD_OBJECT})
+
+# An operation left out of this table is for full-access keys only, so one added
+# to the router later is closed to scoped keys until someone decides otherwise.
+_REQUIRED = {
+    Operation.HEAD_BUCKET: Level.READ,
+    Operation.GET_BUCKET_LOCATION: Level.READ,
+    Operation.GET_BUCKET_VERSIONING: Level.READ,
+    Operation.GET_BUCKET_LIFECYCLE: Level.READ,
+    Operation.GET_BUCKET_CORS: Level.READ,
+    Operation.GET_BUCKET_ACL: Level.READ,
+    Operation.GET_OBJECT_ACL: Level.READ,
+    Operation.LIST_OBJECTS: Level.READ,
+    Operation.LIST_OBJECTS_V2: Level.READ,
+    Operation.LIST_OBJECT_VERSIONS: Level.READ,
+    Operation.GET_OBJECT: Level.READ,
+    Operation.HEAD_OBJECT: Level.READ,
+    Operation.GET_OBJECT_TAGGING: Level.READ,
+    Operation.PUT_OBJECT: Level.WRITE,
+    Operation.COPY_OBJECT: Level.WRITE,
+    Operation.DELETE_OBJECT: Level.WRITE,
+    Operation.DELETE_OBJECTS: Level.WRITE,
+    Operation.PUT_OBJECT_ACL: Level.WRITE,  # And ADMIN to make it public.
+    Operation.PUT_OBJECT_TAGGING: Level.WRITE,  # And ADMIN to tag an older version.
+    Operation.DELETE_OBJECT_TAGGING: Level.WRITE,
+    Operation.CREATE_MULTIPART_UPLOAD: Level.WRITE,
+    Operation.UPLOAD_PART: Level.WRITE,
+    Operation.UPLOAD_PART_COPY: Level.WRITE,  # And READ on the source, once it's known.
+    Operation.COMPLETE_MULTIPART_UPLOAD: Level.WRITE,
+    Operation.ABORT_MULTIPART_UPLOAD: Level.WRITE,
+    Operation.LIST_PARTS: Level.WRITE,
+    Operation.LIST_MULTIPART_UPLOADS: Level.WRITE,
+    Operation.DELETE_BUCKET: Level.ADMIN,
+    # Erasing history takes more than writing: a stolen write key can't destroy what
+    # versioning is there to keep.
+    Operation.DELETE_OBJECT_VERSION: Level.ADMIN,
+    Operation.PUT_BUCKET_VERSIONING: Level.ADMIN,
+    # A rule can erase history as surely as a request can.
+    Operation.PUT_BUCKET_LIFECYCLE: Level.ADMIN,
+    Operation.DELETE_BUCKET_LIFECYCLE: Level.ADMIN,
+    Operation.PUT_BUCKET_CORS: Level.ADMIN,
+    # Publishing takes more than writing: a stolen write key can't expose a bucket.
+    Operation.PUT_BUCKET_ACL: Level.ADMIN,
+    Operation.DELETE_BUCKET_CORS: Level.ADMIN,
+}
+
+
+def require(scope: Scope, bucket: str, level: Level) -> None:
+    if not scope.allows(bucket, level):
+        raise S3Error(errors.ACCESS_DENIED)
+
+
+def authorize(scope: Scope, route: Route) -> None:
+    """Refuse before any bucket is looked up, so a refusal says nothing about it."""
+    if scope.buckets is None or not isinstance(route, BucketRoute | ObjectRoute):
+        return  # Listing buckets is open to all: it shows what a key can see.
+    level = _REQUIRED.get(route.operation)
+    if level is None:
+        raise S3Error(errors.ACCESS_DENIED)
+    require(scope, route.bucket, level)