f5-backup 0.1.0__tar.gz

f5_backup-0.1.0/PKG-INFO

@@ -0,0 +1,26 @@
+Metadata-Version: 2.4
+Name: f5-backup
+Version: 0.1.0
+Summary: Automatic backups to S3-compatible storage
+License: MIT
+Author: Will Hinson
+Author-email: will@hinson.sh
+Requires-Python: >=3.13
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3.15
+Requires-Dist: boto3 (>=1.43.105,<2.0.0)
+Requires-Dist: click (>=8.5.0,<9.0.0)
+Requires-Dist: humanize (>=4.16.0,<5.0.0)
+Requires-Dist: insights-core (>=3.8.2,<4.0.0)
+Requires-Dist: pydantic (>=2.13.5,<3.0.0)
+Requires-Dist: pytsk3 (>=20260715,<20260716)
+Requires-Dist: pyudev (>=0.24.5,<0.25.0)
+Requires-Dist: rich (>=15.0.0,<16.0.0)
+Requires-Dist: tenacity (>=9.1.4,<10.0.0)
+Description-Content-Type: text/markdown
+
+# f5
+

f5_backup-0.1.0/README.md

@@ -0,0 +1 @@
+# f5

f5_backup-0.1.0/f5/__init__.py

@@ -0,0 +1,7 @@
+__all__ = [
+    "backup",
+    "cli",
+]
+
+from . import backup
+from . import cli

f5_backup-0.1.0/f5/__main__.py

@@ -0,0 +1,3 @@
+from .cli import cli
+
+cli()

f5_backup-0.1.0/f5/backup/__init__.py

@@ -0,0 +1,212 @@
+import logging
+from queue import Queue
+from threading import Thread
+
+import humanize
+import rich
+from rich.console import Console
+from rich.progress import (
+    Progress,
+    TextColumn,
+    BarColumn,
+    DownloadColumn,
+    TransferSpeedColumn,
+    TimeRemainingColumn,
+    TaskID,
+)
+
+from ..devices import Partition, PartitionRemoved
+from .filekey import FileKey
+from .s3config import S3Config
+from .s3filesystem import S3Filesystem
+from .tskfilesystem import TskFilesystem
+from .uploadprogress import UploadProgress
+
+
+_logger: logging.Logger = logging.getLogger(__name__)
+
+
+def _make_progress() -> Progress:
+    console: Console = rich.get_console()
+
+    return Progress(
+        TextColumn("[bold]{task.description}", justify="right"),
+        BarColumn(),
+        DownloadColumn(binary_units=True),
+        TransferSpeedColumn(),
+        TimeRemainingColumn(),
+        console=console,
+        disable=not console.is_terminal,  # no bars when piped or run under a service
+    )
+
+
+def _get_target_files(
+    partition: Partition,
+    tsk: TskFilesystem,
+    s3: S3Filesystem,
+) -> tuple[set[FileKey], int]:
+    # discover all the files on the local filesystem
+    local_files: set[FileKey] = tsk.discover_files()
+    _logger.info(
+        "Discovered %s total file%s on %s",
+        f"{len(local_files):,}",
+        ("s" if len(local_files) != 1 else ""),
+        partition.device.device_node,
+    )
+
+    # check which of the objects exist on the remote and only return those that don't
+    target_files: set[FileKey] = local_files - s3.get_partition_files(
+        partition=partition
+    )
+    total_size: int = sum(
+        map(
+            lambda file_key: file_key.size,
+            target_files,
+        )
+    )
+    _logger.info(
+        "Found %s file%s to sync from %s to remote storage (%s)",
+        f"{len(target_files)}",
+        ("s" if len(target_files) != 1 else ""),
+        partition.device.device_node,
+        humanize.naturalsize(
+            total_size,
+            binary=True,
+        ),
+    )
+    return target_files, total_size
+
+
+def _upload_with_progress(
+    progress: Progress,
+    partition: Partition,
+    target_file_key: FileKey,
+    index: int,
+    total_files: int,
+    overall_task: TaskID,
+    s3: S3Filesystem,
+    tsk: TskFilesystem,
+) -> None:
+    try:
+        file_task: TaskID = progress.add_task(
+            f"{index + 1}/{total_files} {target_file_key.path.name}",
+            total=target_file_key.size,
+        )
+        _logger.info(
+            "Uploading %s of size %s from %s (%s/%s)",
+            target_file_key.path.as_posix(),
+            humanize.naturalsize(
+                target_file_key.size,
+                binary=True,
+            ),
+            partition.device.device_node,
+            index + 1,
+            total_files,
+        )
+
+        s3.upload_partition_file(
+            partition=partition,
+            filesystem=tsk,
+            file_key=target_file_key,
+            callback=UploadProgress(
+                progress,
+                overall_task,
+                file_task,
+            ),
+        )
+        progress.remove_task(file_task)
+
+    finally:
+        # always remove any remaining task entries to clear the console up
+        for task_id in list(progress.task_ids):
+            try:
+                progress.remove_task(task_id)
+            except KeyError:
+                ...
+
+
+def _backup_partition(
+    partition: Partition,
+    s3_config: S3Config,
+) -> None:
+    tsk: TskFilesystem = TskFilesystem(partition=partition)
+    s3: S3Filesystem = S3Filesystem(config=s3_config)
+
+    # get the target files to sync and the total size of all of them
+    target_files, total_size = _get_target_files(partition, tsk, s3)
+    with _make_progress() as progress:
+        overall_task: TaskID = progress.add_task(
+            "Total",
+            total=total_size,
+        )
+
+        for index, target_file_key in enumerate(
+            sorted(
+                # iterate over all of the file in reverse size order, starting
+                # with the largest file first
+                target_files,
+                key=lambda file_key: file_key.size,
+                reverse=True,
+            )
+        ):
+            _upload_with_progress(
+                progress=progress,
+                s3=s3,
+                tsk=tsk,
+                partition=partition,
+                target_file_key=target_file_key,
+                index=index,
+                total_files=len(target_files),
+                overall_task=overall_task,
+            )
+
+        try:
+            progress.remove_task(overall_task)
+        except KeyError:
+            ...
+
+
+def make_backup_thread(
+    partition_queue: Queue[Partition],
+    s3_config: S3Config,
+) -> Thread:
+    """
+    Start a thread to handle backups for new partitions.
+    """
+
+    def backup_worker() -> None:
+        while True:
+            partition = partition_queue.get()
+            if partition is None:
+                break
+
+            partition_name: str = partition.device.device_node
+            try:
+                partition.check_removed()
+                _backup_partition(
+                    partition,
+                    s3_config=s3_config,
+                )
+            except (PartitionRemoved, OSError) as exc:
+                if isinstance(exc, OSError) and not partition.removed.is_set():
+                    _logger.exception(f"Backup of partition {partition_name} failed")
+                else:
+                    _logger.warning(
+                        f"Backup of partition {partition_name} stopped: device removed"
+                    )
+            except Exception:
+                _logger.exception(f"Backup of partition {partition_name} failed")
+            else:
+                _logger.info(
+                    "Partition %s of device %s is fully backed up",
+                    partition.device.device_node,
+                    partition.parent_device.device_node,
+                )
+            finally:
+                partition_queue.task_done()
+
+    return Thread(
+        target=backup_worker,
+        name="backup-worker",
+        daemon=True,
+    )

f5_backup-0.1.0/f5/backup/filekey.py

@@ -0,0 +1,22 @@
+from pathlib import Path
+
+from pydantic import BaseModel
+
+
+class FileKey(BaseModel):
+    path: Path
+    size: int
+
+    def __eq__(self: "FileKey", other: object) -> bool:
+        if not isinstance(other, FileKey):
+            return NotImplemented
+
+        return self.path == other.path and self.size == other.size
+
+    def __hash__(self: "FileKey") -> int:
+        return hash(
+            (
+                self.path.as_posix(),
+                self.size,
+            )
+        )

f5_backup-0.1.0/f5/backup/s3config.py

@@ -0,0 +1,25 @@
+from pathlib import Path, PurePosixPath
+
+from pydantic import BaseModel
+
+from ..devices import Partition
+
+
+class S3Config(BaseModel):
+    profile: str
+    bucket: str
+    prefix: str
+
+    def get_partition_prefix(self: "S3Config", partition: Partition) -> str:
+        return (
+            (PurePosixPath("/") / self.prefix / partition.uuid)
+            .relative_to("/")
+            .as_posix()
+        )
+
+    def get_partition_file_key(
+        self: "S3Config", partition: Partition, file: Path
+    ) -> str:
+        return (
+            PurePosixPath("/") / self.prefix / partition.uuid / file.relative_to("/")
+        ).as_posix()

f5_backup-0.1.0/f5/backup/s3filesystem.py

@@ -0,0 +1,121 @@
+import logging
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import boto3
+from boto3.exceptions import S3UploadFailedError
+from botocore.exceptions import BotoCoreError, ClientError
+from tenacity import (
+    RetryCallState,
+    before_sleep_log,
+    retry,
+    retry_base,
+    retry_if_exception,
+    retry_if_exception_type,
+    stop_after_attempt,
+    wait_exponential,
+)
+
+
+from ..devices import Partition
+from .filekey import FileKey
+from .s3config import S3Config
+from .tskfilesystem import TskFilesystem
+from .tskreader import TskReader
+from .uploadprogress import UploadProgress
+
+if TYPE_CHECKING:
+    from mypy_boto3_s3 import S3Client
+
+
+_logger: logging.Logger = logging.getLogger(__name__)
+
+
+class _retry_unless_partition_removed(retry_base):
+    def __call__(self, retry_state: RetryCallState) -> bool:
+        partition: Partition = retry_state.kwargs["partition"]
+        return not partition.removed.is_set()
+
+
+class S3Filesystem:
+    _client: S3Client
+    _config: S3Config
+
+    def __init__(self: "S3Filesystem", config: S3Config) -> None:
+        self._config = config
+        self._client = boto3.Session(
+            profile_name=config.profile,
+        ).client("s3")
+
+    def get_partition_files(
+        self: "S3Filesystem",
+        partition: Partition,
+    ) -> set[FileKey]:
+        remote_files: set[FileKey] = set()
+
+        for page in self._client.get_paginator("list_objects_v2").paginate(
+            Bucket=self._config.bucket,
+            Prefix=self._config.get_partition_prefix(partition),
+        ):
+            for obj in page.get("Contents", []):
+                if "Key" not in obj or "Size" not in obj:
+                    continue
+
+                remote_files.add(
+                    FileKey(
+                        path=(
+                            # normalize the path to how it will appear in the actual
+                            # filesystem (i.e., replace the partition prefix with /)
+                            Path("/")
+                            / Path(obj["Key"]).relative_to(
+                                self._config.get_partition_prefix(partition)
+                            )
+                        ),
+                        size=obj["Size"],
+                    )
+                )
+
+        return remote_files
+
+    @retry(
+        stop=stop_after_attempt(5),
+        wait=wait_exponential(multiplier=2, max=60),
+        retry=(
+            retry_if_exception_type((S3UploadFailedError, BotoCoreError, ClientError))
+            & _retry_unless_partition_removed()
+        ),
+        before_sleep=before_sleep_log(_logger, logging.WARNING),
+        reraise=True,
+    )
+    def upload_partition_file(
+        self: "S3Filesystem",
+        partition: Partition,
+        filesystem: TskFilesystem,
+        file_key: FileKey,
+        callback: UploadProgress | None = None,
+    ) -> None:
+        partition.check_removed()
+
+        s3_key: str = self._config.get_partition_file_key(
+            partition,
+            file_key.path,
+        )
+
+        with TskReader(
+            filesystem.open(file_key.path),
+            cancel_event=partition.removed,
+        ) as reader:
+            self._client.upload_fileobj(
+                reader,  # type: ignore
+                Bucket=self._config.bucket,
+                Key=s3_key,
+                Callback=callback,
+            )
+
+        # check that the file actually exists. head_object() will raise
+        # ClientError for us if the file doesn't exist which @retry
+        # will catch
+        self._client.head_object(
+            Bucket=self._config.bucket,
+            Key=s3_key,
+        )

f5_backup-0.1.0/f5/backup/tskfilesystem.py

@@ -0,0 +1,67 @@
+from collections.abc import Generator
+from pathlib import Path
+
+import pytsk3
+
+from ..devices import Partition
+from .filekey import FileKey
+
+
+class TskFilesystem:
+    _partition: Partition
+    _filesystem: pytsk3.FS_Info
+
+    def __init__(self: "TskFilesystem", partition: Partition) -> None:
+        self._partition = partition
+        self._filesystem = pytsk3.FS_Info(
+            pytsk3.Img_Info(
+                partition.device.device_node,
+            )
+        )
+
+    def _walk_files(
+        self: "TskFilesystem",
+        directory: pytsk3.Directory,
+        path: Path = Path("/"),
+    ) -> Generator[FileKey]:
+        """
+        Discover files in a filesystem.
+        """
+
+        for entry in directory:
+            # get the name for this entry as string. if this is the current directory
+            # or parent directory, ignore it
+            entry_name: str = entry.info.name.name.decode("utf-8", "replace")
+            if entry_name in [".", ".."]:
+                continue
+
+            # ignore deleted/unallocated entries with no metadata
+            if entry.info.meta is None:
+                continue
+
+            entry_path: Path = path / entry_name
+
+            # if this is a directory, recurse into it
+            if entry.info.meta.type == pytsk3.TSK_FS_META_TYPE_DIR:
+                yield from self._walk_files(
+                    entry.as_directory(),
+                    entry_path,
+                )
+            else:
+                yield FileKey(
+                    path=entry_path,
+                    size=entry.info.meta.size,
+                )
+
+    def discover_files(
+        self: "TskFilesystem",
+        root_path: Path = Path("/"),
+    ) -> set[FileKey]:
+        return set(
+            self._walk_files(
+                self._filesystem.open_dir(str(root_path)),
+            )
+        )
+
+    def open(self: "TskFilesystem", path: Path):
+        return self._filesystem.open(path.as_posix())

f5_backup-0.1.0/f5/backup/tskreader.py

@@ -0,0 +1,73 @@
+import io
+from threading import Event, Lock
+from typing import Annotated, override, TYPE_CHECKING
+
+import pytsk3
+
+from ..backup import PartitionRemoved
+
+if TYPE_CHECKING:
+    from _typeshed import MaybeNone, WriteableBuffer
+
+_tsk_lock: Annotated[
+    Lock, "Lock for pytsk3 operations since pytsk3 is not threadsafe"
+] = Lock()
+
+
+class TskReader(io.RawIOBase):
+    _file: pytsk3.File
+    _size: int
+    _position: int
+    _cancel_event: Event | None
+
+    def __init__(
+        self: "TskReader", file: pytsk3.File, cancel_event: Event | None = None
+    ) -> None:
+        self._file = file
+        self._size = file.info.meta.size
+        self._position = 0
+        self._cancel_event = cancel_event
+
+    @override
+    def readable(self: "TskReader") -> bool:
+        return True
+
+    @override
+    def seekable(self: "TskReader") -> bool:
+        return True
+
+    def seek(self: "TskReader", offset: int, whence: int = io.SEEK_SET) -> int:
+        match whence:
+            case io.SEEK_SET:
+                self._position = 0 + offset
+            case io.SEEK_CUR:
+                self._position += offset
+            case io.SEEK_END:
+                self._position = self._size
+            case _:
+                raise ValueError(f"Unknown whence value {whence}")
+
+        return self._position
+
+    @override
+    def readinto(self, buffer: WriteableBuffer, /) -> int | MaybeNone:
+        if self._cancel_event is not None and self._cancel_event.is_set():
+            raise PartitionRemoved("Partition was removed during read operation")
+
+        # get the target number of bytes for this read. it's either the
+        # remaining bytes available of length of the target buffer,
+        # whichever's smaller
+        byte_count: int = min(len(buffer), self._size - self._position)
+        if byte_count <= 0:
+            return 0
+
+        # read the bytes with a mutex since pytsk3 isn't threadsafe
+        data: bytes
+        with _tsk_lock:
+            data = self._file.read_random(self._position, byte_count)
+
+        # put the result in the buffer and seek our position ahead
+        buffer[: len(data)] = data
+        self._position += len(data)
+
+        return len(data)

f5_backup-0.1.0/f5/backup/uploadprogress.py

@@ -0,0 +1,35 @@
+from threading import Lock
+
+from rich.progress import Progress, TaskID
+
+
+class UploadProgress:
+    """boto3 Callback that feeds a per-file task and an overall task."""
+
+    def __init__(self, progress: Progress, overall: TaskID, task: TaskID) -> None:
+        self._progress = progress
+        self._overall = overall
+        self._task = task
+        self._sent = 0
+        self._lock = Lock()
+
+    def __call__(self, n: int) -> None:
+        with self._lock:
+            self._sent += n
+
+        try:
+            self._progress.advance(self._task, n)
+            self._progress.advance(self._overall, n)
+        except KeyError:
+            ...
+
+    def reset(self) -> None:
+        """Undo this file's contribution before a retry."""
+        with self._lock:
+            sent, self._sent = self._sent, 0
+
+        try:
+            self._progress.advance(self._overall, -sent)
+            self._progress.reset(self._task)
+        except KeyError:
+            ...

f5_backup-0.1.0/f5/cli.py

@@ -0,0 +1,112 @@
+import logging
+from queue import Queue
+from threading import Thread
+
+import click
+
+from .backup import make_backup_thread, S3Config
+from .devices import MonitorConfig, Partition, make_monitor_thread
+from .logging import setup_logging
+
+
+_logger: logging.Logger = logging.getLogger(__name__)
+
+
+@click.command()
+@click.option(
+    "--exclude-device",
+    "-e",
+    multiple=True,
+    help="Exclude a specific block device",
+    required=False,
+    type=click.Path(
+        exists=False,
+        file_okay=True,
+        dir_okay=False,
+        readable=False,
+    ),
+)
+@click.option(
+    "--include-device",
+    "-i",
+    multiple=True,
+    help="Include a specific block device",
+    required=False,
+    type=click.Path(
+        exists=False,
+        file_okay=True,
+        dir_okay=False,
+        readable=True,
+    ),
+)
+@click.option(
+    "--profile",
+    "-p",
+    help="AWS profile to use for S3 access",
+    required=False,
+    type=str,
+    default="",
+)
+@click.option(
+    "--bucket",
+    "-b",
+    help="S3 bucket to use for backups",
+    required=True,
+    type=str,
+)
+@click.option(
+    "--prefix",
+    "-x",
+    help="S3 prefix to use for backups",
+    required=False,
+    type=str,
+    default="f5",
+)
+@click.option(
+    "--allow-fixed/--no-allow-fixed",
+    help="Allow partitions from fixed devices to be backed up",
+    required=False,
+    type=bool,
+    default=False,
+)
+@click.version_option()
+def cli(
+    include_device: tuple[str, ...],
+    exclude_device: tuple[str, ...],
+    profile: str,
+    bucket: str,
+    prefix: str,
+    allow_fixed: bool,
+) -> None:
+    """
+    f5 - Automatic backups to S3-compatible storage
+    """
+
+    setup_logging()
+
+    # start a monitor thread to watch for new partitions and a
+    # backup thread to handle backups of those partitions
+    partition_queue: Queue[Partition] = Queue()
+    threads: list[Thread] = [
+        make_backup_thread(
+            partition_queue=partition_queue,
+            s3_config=S3Config(
+                profile=profile,
+                bucket=bucket,
+                prefix=prefix,
+            ),
+        ),
+        make_monitor_thread(
+            partition_queue=partition_queue,
+            config=MonitorConfig(
+                allow_fixed=allow_fixed,
+                include_devices=set(include_device),
+                exclude_devices=set(exclude_device),
+            ),
+        ),
+    ]
+    for thread in threads:
+        thread.start()
+        _logger.info(f"Started thread {thread.name}")
+    for thread in threads:
+        thread.join()

f5_backup-0.1.0/f5/devices/__init__.py

@@ -0,0 +1,25 @@
+from queue import Queue
+from threading import Thread
+
+from .monitorconfig import MonitorConfig
+from .partition import Partition
+from .partitionremoved import PartitionRemoved
+
+__all__ = [
+    "MonitorConfig",
+    "Partition",
+    "PartitionRemoved",
+    "make_monitor_thread",
+]
+
+
+def make_monitor_thread(
+    partition_queue: Queue[Partition],
+    config: MonitorConfig,
+) -> Thread:
+    from .monitorworker import MonitorWorker
+
+    return MonitorWorker(
+        partition_queue=partition_queue,
+        config=config,
+    )

f5_backup-0.1.0/f5/devices/monitorconfig.py

@@ -0,0 +1,7 @@
+from pydantic import BaseModel
+
+
+class MonitorConfig(BaseModel):
+    allow_fixed: bool
+    include_devices: set[str]
+    exclude_devices: set[str]

f5_backup-0.1.0/f5/devices/monitorworker.py

@@ -0,0 +1,170 @@
+import logging
+from queue import Queue
+from threading import Lock, Thread
+
+import pyudev
+
+from .monitorconfig import MonitorConfig
+from ..devices import Partition
+
+_logger: logging.Logger = logging.getLogger(__name__)
+
+
+def _get_device_names(device: pyudev.Device) -> set[str]:
+    """All names a config entry might use to refer to this device."""
+
+    # get all of /dev/disk/by-{id,path,uuid,partuuid,label}/...
+    names: set[str] = set(device.device_links)
+
+    if device.device_node:
+        names.add(device.device_node)  # /dev/sdb, /dev/sdb1
+
+    # add the ID_PATH if it exists which is a unique identifier for the port
+    # the device is on
+    if id_path := device.properties.get("ID_PATH"):
+        names.add(id_path)  # pci-0000:00:14.0-usb-0:1:1.0-scsi-0:0:0:0
+
+    return names
+
+
+class MonitorWorker(Thread):
+    _active_partitions: dict[str, Partition]
+    _config: MonitorConfig
+    _device_context: pyudev.Context
+    _monitor: pyudev.Monitor
+    _partition_queue_mutex: Lock
+    _partition_queue: Queue[Partition]
+
+    def __init__(
+        self: "MonitorWorker",
+        partition_queue: Queue[Partition],
+        config: MonitorConfig,
+        daemon: bool = True,
+        **kwargs,
+    ) -> None:
+        super().__init__(daemon=daemon, **kwargs)
+
+        self._config = config
+        self._device_context = pyudev.Context()
+        self._monitor = pyudev.Monitor.from_netlink(self._device_context)
+        self._monitor.filter_by(subsystem="block")
+
+        self._active_partitions = {}
+        self._partition_queue_mutex = Lock()
+        self._partition_queue = partition_queue
+
+    def run(self: "MonitorWorker") -> None:
+        # push events for all connected partitions on startup
+        for device in self._device_context.list_devices(
+            subsystem="block",
+            DEVTYPE="partition",
+        ):
+            self._handle_device(device)
+
+        # then, start an observer to listen for new connection events
+        observer: pyudev.MonitorObserver = pyudev.MonitorObserver(
+            self._monitor,
+            callback=self._handle_device,
+        )
+        observer.start()
+        observer.join()
+
+    def _device_included(self: "MonitorWorker", *devices: pyudev.Device | None) -> bool:
+        # collate all of the names for the given devices into a single set
+        device_names: set[str] = set()
+        for device in devices:
+            if device is not None:
+                device_names |= _get_device_names(device)
+
+        # if the include list is non-empty, then the device must be in it to be included
+        if self._config.include_devices and device_names.isdisjoint(
+            self._config.include_devices
+        ):
+            return False
+
+        # otherwise, check that the device is not in the exclude list
+        return device_names.isdisjoint(self._config.exclude_devices)
+
+    def _handle_device(self: "MonitorWorker", device: pyudev.Device) -> None:
+        # ignore things that aren't partitions with filesystems
+        if device.device_type != "partition":
+            return
+
+        if not self._device_included(device, self._find_disk_parent(device)):
+            _logger.info(
+                f"Detected device {device.device_node} is excluded by config, ignoring"
+            )
+            return
+
+        if (
+            # device.action will be None when we feed in the partitions at startup
+            device.action is None or device.action == "add"
+        ) and device.device_type == "partition":
+            if not self._device_is_removable(device) and not self._config.allow_fixed:
+                _logger.info(
+                    f"Detected non-removable device {device.device_node}, ignoring"
+                )
+                return
+
+            self._enqueue_partition(device=device)
+        elif device.action == "remove" and device.device_type == "partition":
+            self._dequeue_partition(device=device)
+
+    def _device_is_removable(self: "MonitorWorker", device: pyudev.Device) -> bool:
+        # seek up to the parent device that is the actual disk
+        while device is not None and device.get("DEVTYPE") != "disk":
+            device = device.parent
+
+        if device is None:
+            return False
+
+        return bool(device.attributes.asint("removable"))
+
+    def _find_disk_parent(
+        self: "MonitorWorker", device: pyudev.Device
+    ) -> pyudev.Device | None:
+        return device.find_parent("block", "disk")
+
+    def _enqueue_partition(
+        self: "MonitorWorker",
+        device: pyudev.Device,
+    ) -> None:
+        parent_device: pyudev.Device = self._find_disk_parent(device)
+
+        _logger.info(
+            f"Detected new partition {device.device_node} on device "
+            f"{parent_device.device_node if parent_device else 'unknown'}"
+        )
+        new_partition: Partition = Partition(
+            device=device,
+            parent_device=parent_device,
+            uuid=device.properties["ID_FS_UUID"],
+        )
+
+        with self._partition_queue_mutex:
+            self._partition_queue.put(new_partition)
+            self._active_partitions[new_partition.uuid] = new_partition
+
+    def _dequeue_partition(
+        self: "MonitorWorker",
+        device: pyudev.Device,
+    ) -> None:
+        parent_device: pyudev.Device | None = self._find_disk_parent(device)
+
+        _logger.info(
+            f"Detected removal of partition {device.device_node} on device "
+            f"{parent_device.device_node if parent_device else 'unknown'}"
+        )
+
+        with self._partition_queue_mutex:
+            # remove the partition from the queue if it exists
+            for partition in list(self._partition_queue.queue):
+                if partition.device.device_node == device.device_node:
+                    self._partition_queue.queue.remove(partition)
+                    _logger.info(f"Removed partition {device.device_node} from queue")
+
+            # set the removed event for this partition
+            fs_uuid: str = device.properties["ID_FS_UUID"]
+            if fs_uuid in self._active_partitions:
+                self._active_partitions[fs_uuid].removed.set()
+                self._active_partitions.pop(fs_uuid)

f5_backup-0.1.0/f5/devices/partition.py

@@ -0,0 +1,22 @@
+from threading import Event
+from typing import ClassVar
+
+from pydantic import BaseModel, ConfigDict, Field
+import pyudev
+
+from .partitionremoved import PartitionRemoved
+
+
+class Partition(BaseModel):
+    device: pyudev.Device
+    parent_device: pyudev.Device
+    uuid: str
+    removed: Event = Field(default_factory=Event)
+
+    model_config: ClassVar[ConfigDict] = ConfigDict(
+        arbitrary_types_allowed=True,
+    )
+
+    def check_removed(self: "Partition") -> None:
+        if self.removed.is_set():
+            raise PartitionRemoved(f"{self.device.device_node} was removed")

f5_backup-0.1.0/f5/devices/partitionremoved.py

@@ -0,0 +1,4 @@
+class PartitionRemoved(RuntimeError):
+    """
+    Raised when a partition has been removed and check_removed() is called
+    """

f5_backup-0.1.0/f5/logging.py

@@ -0,0 +1,23 @@
+import logging
+from rich.logging import RichHandler
+
+
+def setup_logging(level: logging._Level = logging.INFO) -> None:
+    """
+    Setup logging configuration with RichHandler for better console output.
+
+    Args:
+        level (logging._Level): The logging level to set. Default is logging.INFO.
+    """
+
+    logging.basicConfig(
+        level=level,
+        format="[%(name)s] %(message)s",
+        datefmt="[%X]",
+        handlers=[
+            RichHandler(
+                rich_tracebacks=True,
+                markup=False,
+            )
+        ],
+    )

f5_backup-0.1.0/pyproject.toml

@@ -0,0 +1,40 @@
+[project]
+name = "f5-backup"
+version = "0.1.0"
+description = "Automatic backups to S3-compatible storage"
+authors = [
+    {name = "Will Hinson",email = "will@hinson.sh"}
+]
+license = {text = "MIT"}
+readme = "README.md"
+requires-python = ">=3.13"
+dependencies = [
+    "click (>=8.5.0,<9.0.0)",
+    "pydantic (>=2.13.5,<3.0.0)",
+    "boto3 (>=1.43.105,<2.0.0)",
+    "pyudev (>=0.24.5,<0.25.0)",
+    "rich (>=15.0.0,<16.0.0)",
+    "insights-core (>=3.8.2,<4.0.0)",
+    "pytsk3 (>=20260715,<20260716)",
+    "humanize (>=4.16.0,<5.0.0)",
+    "tenacity (>=9.1.4,<10.0.0)",
+]
+
+[tool.poetry]
+packages = [{ include = "f5" }]
+
+[tool.poetry.scripts]
+f5 = "f5.cli:cli"
+
+[dependency-groups]
+dev = [
+    "pytest (>=9.1.1,<10.0.0)",
+    "mypy (>=2.3.1,<3.0.0)",
+    "ruff (>=0.16.9,<0.17.0)",
+    "boto3-stubs[s3] (>=1.43.105,<2.0.0)"
+]
+
+
+[build-system]
+requires = ["poetry-core>=2.0.0,<3.0.0"]
+build-backend = "poetry.core.masonry.api"