leblob 0.4.0.dev1__py3-none-any.whl

leblob/maintenance.py

@@ -0,0 +1,444 @@
+"""Background work, and the consistency check behind `leblob fsck`.
+
+Every worker runs the maintenance loop. There is no leader, because each step
+is safe to run concurrently: unlinking a blob twice is harmless.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import secrets
+import time
+from dataclasses import dataclass
+from functools import partial
+from typing import TYPE_CHECKING
+
+from leblob import meta, metrics
+from leblob.blobstore import (
+    TMP_MAX_AGE_SECONDS,
+    CorruptSourceError,
+    TargetVolumeError,
+    VolumeError,
+    copy_blob,
+    open_store,
+    open_volume,
+)
+from leblob.meta import GC_GRACE_MS
+from leblob.model import BlobFile
+
+if TYPE_CHECKING:
+    from collections.abc import Awaitable, Callable, Iterable, Mapping
+    from pathlib import Path
+
+    from leblob.blobstore import BlobStore, Volume
+    from leblob.meta import Database
+    from leblob.model import (
+        Blob,
+        KnownBlob,
+        LifecycleReport,
+        RepairWork,
+        VolumeRecord,
+    )
+
+logger = logging.getLogger(__name__)
+
+_INTERVAL_SECONDS = 60
+_NO_VOLUMES_RETRY_SECONDS = 2
+_BATCH_SIZE = 1000
+_REPAIR_POLL_SECONDS = 5
+_REPAIR_BATCH = 100
+_REPAIR_LEASE_MS = 600_000
+_SCRUB_START_SECONDS = 60
+_SCRUB_BATCH = 256
+# ponytail: one rate for any disks. Make it a setting when a store needs another.
+_SCRUB_BYTES_PER_SECOND = 32 * 1024 * 1024
+_LONGEST_PAUSE_MS = 60_000
+_MILLISECONDS = 1000
+_COMPLETED_UPLOAD_MEMORY_MS = meta.DAY_MS
+
+
+async def run_forever(db: Database, blobs: BlobStore, data_dir: Path) -> None:
+    """Run maintenance periodically, surviving failures of any one round."""
+    while True:
+        if blobs.volumes:
+            # Jitter, so that workers started together do not all run at once.
+            await asyncio.sleep(
+                _INTERVAL_SECONDS + secrets.randbelow(_INTERVAL_SECONDS)
+            )
+        else:
+            await asyncio.sleep(_NO_VOLUMES_RETRY_SECONDS)
+        try:
+            await follow_volumes(db, blobs, data_dir)
+            now = meta.now_ms()
+            await collect(db, blobs, now)
+            await meta.forget_completed_uploads(db, now - _COMPLETED_UPLOAD_MEMORY_MS)
+            await asyncio.to_thread(blobs.sweep_tmp, time.time())
+        except Exception:
+            logger.exception("Maintenance round failed; it will run again later.")
+
+
+async def follow_volumes(db: Database, blobs: BlobStore, data_dir: Path) -> None:
+    """Pick up changes made by commands, and disks that have come or gone."""
+    records, copies = await meta.current_volumes(db)
+    current = await asyncio.to_thread(
+        open_store, data_dir, records, copies, allow_degraded=True
+    )
+    blobs.follow(current)
+
+
+async def current_gauges(db: Database, data_dir: Path) -> list[metrics.Sample]:
+    """What is true right now of the store, whichever worker is asked."""
+    records, copies = await meta.current_volumes(db)
+    return [
+        (metrics.COPIES, (), copies),
+        (metrics.REPAIRS_QUEUED, (), await meta.repairs_queued(db)),
+        (
+            metrics.BLOBS_AWAITING_COLLECTION,
+            (),
+            await meta.blobs_awaiting_collection(db),
+        ),
+        *await asyncio.to_thread(_volume_gauges, data_dir, records),
+    ]
+
+
+def _volume_gauges(
+    data_dir: Path, records: Iterable[VolumeRecord]
+) -> list[metrics.Sample]:
+    gauges: list[metrics.Sample] = []
+    for record in records:
+        labels = (("volume", str(record.volume_id)),)
+        gauges.append((metrics.VOLUME_DRAINING, labels, int(record.draining)))
+        try:
+            free = open_volume(data_dir, record).free_bytes()
+        except (VolumeError, OSError):
+            gauges.append((metrics.VOLUME_UP, labels, 0))
+        else:
+            gauges += [
+                (metrics.VOLUME_UP, labels, 1),
+                (metrics.VOLUME_FREE_BYTES, labels, free),
+            ]
+    return gauges
+
+
+async def run_repairs(db: Database, blobs: BlobStore) -> None:
+    """Restore copies as they are needed, surviving failures of any one round."""
+    while True:
+        await asyncio.sleep(_REPAIR_POLL_SECONDS)
+        try:
+            while await meta.repairs_due(db, meta.now_ms()):
+                await repair_round(db, blobs, meta.now_ms())
+        except Exception:
+            logger.exception("Repair round failed; it will run again later.")
+
+
+async def repair_round(db: Database, blobs: BlobStore, now: int) -> int:
+    """Work through a batch of the queue. Returns how many blobs are done with."""
+    done = 0
+    for work in await meta.claim_repairs(db, now, _REPAIR_BATCH, _REPAIR_LEASE_MS):
+        if await _repair(db, blobs, work):
+            done += 1
+    return done
+
+
+async def _repair(db: Database, blobs: BlobStore, work: RepairWork) -> bool:
+    """Bring a blob up to its copies, and off any draining volume once it is safe.
+
+    The queue keeps a blob until this succeeds, so giving up here only delays it.
+    """
+    blob = work.blob
+    active = [key for key in blob.volume_ids if key not in work.draining]
+    needed = work.copies - len(active)
+    if needed > 0:
+        added, placed = await _add_copies(db, blobs, work, needed)
+        if added < needed:
+            return False
+        if not active and work.draining.intersection(blob.volume_ids):
+            # A read that started before this knows only the draining copy, so that
+            # stays for the grace period, as any deleted blob does.
+            await meta.postpone_repair(db, blob.blob_id, meta.now_ms() + GC_GRACE_MS)
+            return False
+        active += placed
+    # Copies on a disk that is away don't count: this one may be all that can be read.
+    if any(key in blobs.volumes for key in active):
+        for volume_id in work.draining.intersection(blob.volume_ids):
+            await _remove_drained_copy(db, blobs, blob.blob_id, volume_id)
+    return await meta.finish_repair(db, blob.blob_id)
+
+
+async def _add_copies(
+    db: Database, blobs: BlobStore, work: RepairWork, needed: int
+) -> tuple[int, list[int]]:
+    """Make new copies, and return how many it managed, and on which volumes."""
+    blob = work.blob
+    placed: list[int] = []
+    for target in blobs.placement_order(exclude=(*blob.volume_ids, *work.draining)):
+        if len(placed) == needed:
+            break
+        try:
+            await copy_blob(blobs, blob, target)
+        except TargetVolumeError as error:
+            blobs.report_failure(target.volume_id, OSError(str(error)))
+            continue
+        except CorruptSourceError:
+            logger.exception(
+                "Can't copy blob %s yet; scrub finds the bad one", blob.blob_id
+            )
+            break
+        except OSError as error:
+            logger.warning("Can't read blob %s to copy it: %s", blob.blob_id, error)
+            break
+        if not await meta.add_location(db, blob.blob_id, target.volume_id):
+            target.delete(blob.blob_id)  # Collected while it was being copied.
+            return needed, []
+        placed.append(target.volume_id)
+        metrics.counters.add(metrics.COPIES_REPAIRED)
+    return len(placed), placed
+
+
+async def _remove_drained_copy(
+    db: Database, blobs: BlobStore, blob_id: str, volume_id: int
+) -> None:
+    """Remove the row and then the file, so that a crash leaves at worst an orphan.
+
+    Unlike with a bad copy, no repair can be writing here: this volume takes no blobs.
+    """
+    volume = blobs.volumes.get(volume_id)
+    if volume is None:
+        return  # The disk is away. `volume forget --disk-lost` deals with it.
+    await meta.remove_location(db, blob_id, volume_id)
+    try:
+        await asyncio.to_thread(volume.delete, blob_id)
+    except OSError as error:
+        blobs.report_failure(volume_id, error)
+
+
+async def run_scrub(db: Database, blobs: BlobStore) -> None:
+    """Check every copy, slowly and for ever, surviving failures of any one round."""
+    await asyncio.sleep(_SCRUB_START_SECONDS + secrets.randbelow(_INTERVAL_SECONDS))
+    await _keep_doing(
+        "Scrub",
+        partial(meta.scrub_due_at, db),
+        lambda now: scrub_round(db, blobs, now),
+    )
+
+
+async def _keep_doing(
+    what: str,
+    due_at: Callable[[], Awaitable[int]],
+    work: Callable[[int], Awaitable[object]],
+) -> None:
+    """Do work whenever it is due, for ever, surviving failures of any one round."""
+    while True:
+        try:
+            pause = await _do_if_due(due_at, work)
+        except Exception:
+            logger.exception("%s round failed; it will run again later.", what)
+            pause = _INTERVAL_SECONDS
+        await asyncio.sleep(pause)
+
+
+async def _do_if_due(
+    due_at: Callable[[], Awaitable[int]], work: Callable[[int], Awaitable[object]]
+) -> float:
+    """Work if it is time. Returns the seconds to wait before looking again."""
+    now = meta.now_ms()
+    due = await due_at()
+    if due > now:
+        return min(due - now, _LONGEST_PAUSE_MS) / _MILLISECONDS
+    await work(now)
+    return 0
+
+
+async def run_lifecycle(db: Database, day_ms: int) -> None:
+    """Apply lifecycle rules once a day, surviving failures of any one round."""
+    await _keep_doing(
+        "Lifecycle",
+        partial(meta.lifecycle_due_at, db),
+        lambda now: lifecycle_round(db, now, day_ms),
+    )
+
+
+async def lifecycle_round(db: Database, now: int, day_ms: int) -> LifecycleReport:
+    report = await meta.expire_batch(db, now, day_ms)
+    metrics.counters.add(metrics.LIFECYCLE_OBJECTS_EXPIRED, report.expired)
+    metrics.counters.add(metrics.LIFECYCLE_VERSIONS_ERASED, report.erased)
+    metrics.counters.add(metrics.LIFECYCLE_UPLOADS_ABORTED, report.aborted)
+    return report
+
+
+async def scrub_round(db: Database, blobs: BlobStore, now: int) -> ScrubReport:
+    batch = await meta.claim_scrub_batch(db, now, _SCRUB_BATCH, _SCRUB_BYTES_PER_SECOND)
+    return await _scrub_blobs(db, blobs, batch)
+
+
+async def scrub_everything(db: Database, blobs: BlobStore) -> ScrubReport:
+    """Check every copy now, at full speed, for `leblob scrub`."""
+    report = ScrubReport()
+    after = ""
+    while batch := await meta.blobs_to_scrub(db, after, _SCRUB_BATCH):
+        report = report.plus(await _scrub_blobs(db, blobs, batch))
+        after = batch[-1].blob_id
+    return report
+
+
+async def _scrub_blobs(
+    db: Database, blobs: BlobStore, batch: Iterable[Blob]
+) -> ScrubReport:
+    report = ScrubReport()
+    for blob in batch:
+        for volume_id in blob.volume_ids:
+            volume = blobs.volumes.get(volume_id)
+            if volume is None:
+                continue
+            report = report.plus(ScrubReport(checked=1))
+            metrics.counters.add(metrics.COPIES_SCRUBBED)
+            if not await _copy_is_intact(volume, blob):
+                report = report.plus(await _drop_bad_copy(db, blobs, blob, volume))
+    return report
+
+
+async def _copy_is_intact(volume: Volume, blob: Blob) -> bool:
+    try:
+        return await asyncio.to_thread(volume.copy_is_intact, blob)
+    except OSError:
+        return False
+
+
+async def _drop_bad_copy(
+    db: Database, blobs: BlobStore, blob: Blob, volume: Volume
+) -> ScrubReport:
+    """Remove a bad copy, so that repair replaces it, unless it is all there is.
+
+    The file goes first, as with the collector: while the row remains repair leaves
+    this volume alone, so it can't write a new copy that this would then delete. It
+    also leaves everything as it was on a dead disk, which only a person should write
+    off. A crash between the two leaves a row with no file, which `fsck` reports
+    and the next scrub clears.
+    """
+    blob_id = blob.blob_id
+    others = await meta.other_copies(db, blob_id, volume.volume_id)
+    if others is None:
+        return ScrubReport()  # Collected meanwhile: not damage.
+    if not volume.identifies_itself():
+        blobs.report_failure(volume.volume_id, OSError(f"{volume.root} isn't mounted"))
+        return ScrubReport()
+    if others == 0:
+        logger.error("The only copy of blob %s is bad, on %s", blob_id, volume.root)
+        metrics.counters.add(metrics.ONLY_COPIES_BAD)
+        return ScrubReport(unrecoverable=(blob_id,))
+    try:
+        await asyncio.to_thread(volume.delete, blob_id)
+    except OSError as error:
+        blobs.report_failure(volume.volume_id, error)
+        return ScrubReport()
+    await meta.remove_location(db, blob_id, volume.volume_id)
+    logger.warning("Removed a bad copy of blob %s from %s", blob_id, volume.root)
+    metrics.counters.add(metrics.BAD_COPIES_REMOVED)
+    return ScrubReport(removed=1)
+
+
+async def collect(db: Database, blobs: BlobStore, now: int) -> int:
+    """Delete blobs whose grace period is over. Returns how many were deleted."""
+    collected = 0
+    while True:
+        due = await meta.blobs_due_for_collection(db, now, _BATCH_SIZE)
+        if not due:
+            return collected
+        # Files go first: a crash here leaves rows to retry, never lost files.
+        await asyncio.to_thread(_delete_all, blobs, due)
+        await meta.forget_blobs(db, list(due))
+        metrics.counters.add(metrics.BLOBS_COLLECTED, len(due))
+        collected += len(due)
+
+
+def _delete_all(blobs: BlobStore, due: Mapping[str, Iterable[int]]) -> None:
+    for blob_id, volume_ids in due.items():
+        blobs.delete(blob_id, volume_ids)
+
+
+@dataclass(frozen=True, slots=True)
+class ScrubReport:
+    """What checking copies found."""
+
+    checked: int = 0
+    removed: int = 0
+    unrecoverable: tuple[str, ...] = ()
+
+    def plus(self, other: ScrubReport) -> ScrubReport:
+        return ScrubReport(
+            self.checked + other.checked,
+            self.removed + other.removed,
+            self.unrecoverable + other.unrecoverable,
+        )
+
+
+@dataclass(frozen=True, slots=True)
+class FsckReport:
+    """How the metadata and the disk disagree."""
+
+    orphans: tuple[BlobFile, ...]
+    young_orphans: tuple[BlobFile, ...]
+    missing: tuple[BlobFile, ...]
+    wrong_size: tuple[BlobFile, ...]
+    under_replicated: tuple[str, ...] = ()
+    lost: tuple[str, ...] = ()
+
+    @property
+    def is_clean(self) -> bool:
+        return not (self.orphans or self.missing or self.wrong_size or self.lost)
+
+
+async def fsck(db: Database, blobs: BlobStore, now: float) -> FsckReport:
+    """Compare the blobs and their locations with every volume.
+
+    A recent file with no row may be a write about to commit, so it is reported
+    apart from older orphans, and `fsck --delete-orphans` leaves it alone.
+
+    A row with no file is damage, unless the blob is queued for collection: the
+    collector deletes the file first, and a crash or this very check can land before
+    it forgets the row. The grace period means that a blob it deletes during the scan
+    was already queued when the rows were read.
+    """
+    known = await meta.all_blobs(db)
+    return await asyncio.to_thread(_compare, known, blobs, now)
+
+
+def _compare(
+    known: Mapping[str, KnownBlob], blobs: BlobStore, now: float
+) -> FsckReport:
+    orphans: list[BlobFile] = []
+    young_orphans: list[BlobFile] = []
+    wrong_size: list[BlobFile] = []
+    present: set[tuple[int, str]] = set()
+    for volume_id, volume in blobs.volumes.items():
+        for blob_id, size, modified_at in volume.scan():
+            blob = known.get(blob_id)
+            found = BlobFile(volume_id, blob_id)
+            if blob is None or volume_id not in blob.volume_ids:
+                old = now - modified_at > TMP_MAX_AGE_SECONDS
+                (orphans if old else young_orphans).append(found)
+            elif blob.size != size:
+                wrong_size.append(found)
+            present.add((volume_id, blob_id))
+    return FsckReport(
+        tuple(orphans),
+        tuple(young_orphans),
+        tuple(
+            BlobFile(volume_id, blob_id)
+            for blob_id, blob in known.items()
+            for volume_id in blob.volume_ids
+            if (volume_id, blob_id) not in present and not blob.queued_for_collection
+        ),
+        tuple(wrong_size),
+        tuple(
+            blob_id
+            for blob_id, blob in known.items()
+            if len(blob.volume_ids) < blobs.copies and not blob.queued_for_collection
+        ),
+        tuple(
+            blob_id
+            for blob_id, blob in known.items()
+            if not blob.volume_ids and not blob.queued_for_collection
+        ),
+    )