simo 2.10.11__py3-none-any.whl → 2.11.2__py3-none-any.whl

simo/backups/tasks.py

@@ -140,26 +140,405 @@ def create_snap(lv_group, lv_name, snap_name=None, size=None, try_no=1):
 
 
 def get_lvm_partition(lsblk_data):
+    """Return the *lsblk* entry describing the logical volume mounted as "/".
+
+    The original implementation returned prematurely when the first top-level
+    device contained any children – even if none of them matched the search
+    criteria.  As a result the search stopped after inspecting just a single
+    branch of the device tree which broke setups where the root logical
+    volume was not located under the very first block device listed by
+    *lsblk* (e.g. when the machine had multiple drives).
+
+    The fixed version walks the whole tree depth-first and stops only after a
+    matching entry is found or the entire structure has been inspected.
+    """
+
     for device in lsblk_data:
-        if device['type'] == 'lvm' and device['mountpoint'] == '/':
+        # Check the current node first.
+        if device.get('type') == 'lvm' and device.get('mountpoint') == '/':
             return device
-        if 'children' in device:
-            return get_lvm_partition(device['children'])
+
+        # Recursively search children (if any).  The recursive call returns
+        # either the desired dictionary or *None* – propagate the first truthy
+        # value up the call stack so that the outermost caller gets the
+        # matching entry.
+        child_match = get_lvm_partition(device.get('children', [])) if device.get('children') else None
+        if child_match:
+            return child_match
+
+    # Nothing found on this branch.
+    return None
+
+
+def _has_backup_label(dev: dict) -> bool:
+    """Return ``True`` when the given *lsblk* device description represents
+    the desired "backup" partition.  The logic is kept in one place to make
+    future adjustments simpler.
+
+    The criteria as of now are:
+
+    The filesystem label (``label`` field) is exactly ``BACKUP`` – this is
+    how the pre-built *rescue.img* image names the 3rd partition that
+    will be used for storing backups.
+    """
+
+    label = (dev.get("label") or dev.get("partlabel") or "").upper()
+    if label == "BACKUP":
+        return True
+    return False
 
 
 def get_backup_device(lsblk_data):
+    """Locate a removable partition that should be used to store backups.
+
+    Priority is given to a partition explicitly labelled ``BACKUP``.  If such
+    a partition isn't found, the legacy rule – ‘any removable exFAT
+    partition’ – is used.
+    """
+
+    _MIN_SIZE_BYTES = 32 * 1024 * 1024 * 1024  # 32 GiB – keep in sync with
+                                               # _find_blank_removable_device.
+
+    def _device_size_bytes(dev_name: str):
+        """Return size of *dev_name* in bytes (or ``None`` on failure)."""
+
+        for cmd in (
+            f"blockdev --getsize64 /dev/{dev_name}",
+            f"lsblk -b -dn -o SIZE /dev/{dev_name}",
+        ):
+            try:
+                out = subprocess.check_output(
+                    cmd, shell=True, stderr=subprocess.DEVNULL
+                ).strip()
+                return int(out)
+            except Exception:
+                continue
+        return None
+
+    # ------------------------------------------------------------------
+    # Helper: does the filesystem already contain legacy backups?
+    # ------------------------------------------------------------------
+
+    def _entry_has_simo_backups(entry: dict) -> bool:
+        """Return *True* when *entry* hosts legacy ``simo_backups`` folder.
+
+        The implementation borrows heavily from the _fs_is_empty() helper –
+        we temporarily mount the filesystem read-only when it is not mounted
+        yet, inspect the directory listing and clean everything up.
+        """
+
+        mountpoint = entry.get("mountpoint")
+        cleanup = False
+
+        if not mountpoint:
+            tmp_dir = f"/tmp/simo-bk-{uuid.uuid4().hex[:8]}"
+            try:
+                os.makedirs(tmp_dir, exist_ok=True)
+                res = subprocess.run(
+                    f"mount -o ro /dev/{entry['name']} {tmp_dir}",
+                    shell=True,
+                    stderr=subprocess.PIPE,
+                )
+                if res.returncode:
+                    shutil.rmtree(tmp_dir, ignore_errors=True)
+                    return False
+                mountpoint = tmp_dir
+                cleanup = True
+            except Exception:
+                shutil.rmtree(tmp_dir, ignore_errors=True)
+                return False
+
+        has_backups = os.path.isdir(os.path.join(mountpoint, "simo_backups"))
+
+        if cleanup:
+            subprocess.run(f"umount {mountpoint}", shell=True)
+            shutil.rmtree(mountpoint, ignore_errors=True)
+
+        return has_backups
+
+    # ------------------------------------------------------------------
+    # Phase 1 – look for properly prepared BACKUP partition **>=32 GiB**.
+    #          This is the preferred modern approach.
+    # ------------------------------------------------------------------
+
+    for device in lsblk_data:
+        if not device.get("hotplug"):
+            continue
+
+        # Capacity check – skip devices smaller than the required threshold.
+        size_bytes = _device_size_bytes(device["name"])
+        if size_bytes is None:
+            print(f"Could not obtain capacity of: {device['name']}")
+            continue
+
+        if size_bytes < _MIN_SIZE_BYTES:
+            continue
+
+        # Prefer partitions explicitly labelled "BACKUP".
+        for child in device.get("children", []):
+            if _has_backup_label(child):
+                return child
+
+        # Legacy fallback (modern capacity) – whole-disk or partitioned
+        # exFAT volumes are still acceptable for backward compatibility when
+        # they are large enough.
+
+        if (device.get("fstype") or "").lower() == "exfat":
+            return device
+
+        for child in device.get("children", []):
+            if (child.get("fstype") or "").lower() == "exfat":
+                return child
+
+
+    # ------------------------------------------------------------------
+    # Phase 2 – look for **existing** legacy backup drives.
+    # ------------------------------------------------------------------
+
+    if _find_blank_removable_device(lsblk_data):
+        # New empty disk is available, let's use it instead of trying to find
+        # legacy media
+        return None
+
+    for device in lsblk_data:
+        if not device.get("hotplug"):
+            continue
+
+        # Check the whole device first.
+        if device.get("mountpoint") or device.get("fstype"):
+            if _entry_has_simo_backups(device):
+                return device
+
+        # Check its partitions (if any).
+        for child in device.get("children", []):
+            if _entry_has_simo_backups(child):
+                return child
+
+    # Nothing has been found.
+    return None
+
+
+def _find_blank_removable_device(lsblk_data):
+    """Return the first removable block *device* that looks empty.
+
+    A device is considered *blank* when one of the following conditions is
+    met:
+
+    1. It has no children (partitions) **and** no recognised filesystem – the
+       original behaviour that covers brand-new, uninitialised drives.
+    2. It has no children (partitions) **and** an existing filesystem that is
+       effectively empty (e.g. a freshly formatted card).
+
+    Determining if a filesystem is *empty* is tricky without mounting it, but
+    for the purpose of automatically provisioning backup media we can use a
+    pragmatic heuristic: if the device is not mounted we temporarily mount it
+    read-only to a throw-away directory, inspect its contents and then unmount
+    it again.  If it **is** already mounted we reuse the existing
+    mount-point.  In both cases we treat the device as blank when the root of
+    the filesystem contains no entries other than implementation-specific
+    placeholders like the *lost+found* directory created by *mkfs.ext4*.
+
+    This relaxed definition allows the backup subsystem to reuse drives that
+    have been pre-formatted by the user but never actually used to store any
+    files.
+    """
+
+# --- Helper inner functions ------------------------------------------------
+
+    def _device_size_bytes(dev_name: str):
+        """Return size of *dev_name* in bytes (or ``None`` on failure)."""
+
+        for cmd in (
+            f"blockdev --getsize64 /dev/{dev_name}",
+            f"lsblk -b -dn -o SIZE /dev/{dev_name}",
+        ):
+            try:
+                out = subprocess.check_output(
+                    cmd, shell=True, stderr=subprocess.DEVNULL
+                ).strip()
+                return int(out)
+            except Exception:
+                continue
+        return None
+
+    def _fs_is_empty(entry: dict) -> bool:
+        """Heuristic to decide if a filesystem represented by *entry* is empty."""
+
+        fstype = entry.get("fstype")
+        if not fstype:
+            # Unformatted – treat as empty.
+            return True
+
+        mountpoint = entry.get("mountpoint")
+        cleanup = False
+
+        if not mountpoint:
+            tmp_dir = f"/tmp/simo-bk-{uuid.uuid4().hex[:8]}"
+            try:
+                os.makedirs(tmp_dir, exist_ok=True)
+                res = subprocess.run(
+                    f"mount -o ro /dev/{entry['name']} {tmp_dir}",
+                    shell=True,
+                    stderr=subprocess.PIPE,
+                )
+                if res.returncode:
+                    shutil.rmtree(tmp_dir, ignore_errors=True)
+                    print(f"Unable to mount {entry['name']} to inspect contents – skip")
+                    return False
+                mountpoint = tmp_dir
+                cleanup = True
+            except Exception as exc:
+                shutil.rmtree(tmp_dir, ignore_errors=True)
+                print(f"Exception while mounting {entry['name']}: {exc}")
+                return False
+
+        try:
+            with os.scandir(mountpoint) as it:
+                entries = [e.name for e in it if not e.name.startswith('.')]
+        except Exception as exc:
+            print(f"Unable to read directory listing for {entry['name']}: {exc}")
+            if cleanup:
+                subprocess.run(f"umount {mountpoint}", shell=True)
+                shutil.rmtree(mountpoint, ignore_errors=True)
+            return False
+
+        if cleanup:
+            subprocess.run(f"umount {mountpoint}", shell=True)
+            shutil.rmtree(mountpoint, ignore_errors=True)
+
+        meaningful = [e for e in entries if e not in {"lost+found"}]
+        return not meaningful
+
+# ---------------------------------------------------------------------------
+
+    _MIN_SIZE_BYTES = 32 * 1024 * 1024 * 1024  # 32 GiB
+
     for device in lsblk_data:
-        if not device['hotplug']:
+        if not device.get("hotplug"):
+            continue
+
+        size_bytes = _device_size_bytes(device["name"])
+        if size_bytes is None:
+            print(f"Could not obtain capacity of: {device['name']}")
             continue
-        target_device = None
-        if device.get('fstype') == 'exfat':
-            target_device = device
-        elif device.get('children'):
-            for child in device.get('children'):
-                if child.get('fstype') == 'exfat':
-                    target_device = child
-        if target_device:
-            return target_device
+
+        if size_bytes < _MIN_SIZE_BYTES:
+            print(f"Too small (<32 GiB): {device['name']}")
+            continue
+
+        children = device.get("children") or []
+
+        if not children:
+            # Whole-disk filesystem.
+            if _fs_is_empty(device):
+                return device
+            print(f"Whole-disk filesystem on {device['name']} is not empty – skip")
+            continue
+
+        if len(children) == 1:
+            child = children[0]
+            if _fs_is_empty(child):
+                return device
+            print(f"Single partition {child['name']} on {device['name']} is not empty – skip")
+            continue
+
+        print(f"More than one partition on {device['name']} – skip")
+
+    return None
+
+
+def _ensure_rescue_image_written(blank_device_name: str):
+    """Write *rescue.img* to the given **whole-disk** device.
+
+    The function is intentionally idempotent – if writing fails the caller can
+    attempt to call it again (e.g. the next time the periodic task runs).
+
+    It raises an exception on irrecoverable errors so that the caller can log
+    the failure.
+    """
+
+    import tarfile, time
+
+    img_path = os.path.join(os.path.dirname(__file__), "rescue.img.xz")
+
+    # Write the image.  We deliberately avoid using *python-dd* wrappers and
+    # rely on the time-tested `dd(1)` command.
+    dd_cmd = (
+        f"xzcat {img_path} | dd of=/dev/{blank_device_name} bs=4M conv=fsync"
+    )
+    res = subprocess.run(dd_cmd, shell=True, stderr=subprocess.PIPE)
+    if res.returncode:
+        raise RuntimeError(
+            f"Writing rescue image failed: {res.stderr.decode(errors='ignore')}"
+        )
+
+    # Make sure the kernel notices the new partition table.
+    subprocess.run(f"partprobe /dev/{blank_device_name}", shell=True)
+
+    # Give the device a moment to settle.
+    time.sleep(2)
+
+    # Enlarge the 3rd partition (BACKUP) to the rest of the disk and create /
+    # extend the exFAT filesystem.  This is wrapped in a helper to keep the
+    # main flow readable.
+    _expand_backup_partition(blank_device_name)
+
+
+def _expand_backup_partition(device_name: str):
+    """Make partition 3 span leftover space and be ext4 labelled BACKUP.
+
+    Implementation is intentionally minimal and resilient:
+    – Use *sgdisk* only (no interactive prompts).
+    – Delete partition 3 (if present) and create a new one that fills all
+      remaining free space.
+    – Always create a fresh ext4 filesystem labelled BACKUP.
+    Because the rescue-image just flashed is empty, data loss is not a
+    concern and this deterministic route avoids edge-case errors.
+    """
+
+    import time, shutil
+
+    def _dev_path(base: str) -> str:
+        """Return /dev/<base>3 path handling devices that need 'p3'."""
+        direct = f"/dev/{base}3"
+        with_p = f"/dev/{base}p3"
+        return direct if os.path.exists(direct) else with_p
+
+    # 1. Ensure GPT headers cover the whole disk (harmless if already OK).
+    subprocess.run(f"sgdisk -e /dev/{device_name}", shell=True)
+
+    # 2. Drop existing partition 3 (ignore errors when it does not exist).
+    subprocess.run(f"sgdisk -d 3 /dev/{device_name}", shell=True)
+
+    # 3. Create new Linux filesystem partition occupying the rest of the disk.
+    create_cmd = f"sgdisk -n 3:0:0 -t 3:8300 -c 3:BACKUP /dev/{device_name}"
+    res = subprocess.run(create_cmd, shell=True, stderr=subprocess.PIPE)
+    if res.returncode:
+        raise RuntimeError(
+            "sgdisk failed to create BACKUP partition: " +
+            res.stderr.decode(errors="ignore")
+        )
+
+    # 4. Inform kernel and wait for udev.
+    subprocess.run(f"partprobe /dev/{device_name}", shell=True)
+    subprocess.run("udevadm settle", shell=True)
+
+    part_path = _dev_path(device_name)
+    for _ in range(5):
+        if os.path.exists(part_path):
+            break
+        time.sleep(1)
+    else:
+        raise RuntimeError("/dev node for new BACKUP partition did not appear")
+
+    # 5. Always create a fresh ext4 filesystem; wipe old signatures first.
+    subprocess.run(f"wipefs -a {part_path}", shell=True)
+    mkfs_cmd = f"mkfs.ext4 -F -L BACKUP {part_path}"
+    res = subprocess.run(mkfs_cmd, shell=True, stderr=subprocess.PIPE)
+    if res.returncode:
+        raise RuntimeError(
+            "mkfs.ext4 failed for BACKUP partition: " + res.stderr.decode(errors="ignore")
+        )
 
 
 def get_partitions():
@@ -205,16 +584,39 @@ def get_partitions():
 
     backup_device = get_backup_device(lsblk_data)
 
+    # If no suitable partition is available try to prepare one automatically.
+    if not backup_device:
+        blank_dev = _find_blank_removable_device(lsblk_data)
+        if blank_dev:
+            try:
+                _ensure_rescue_image_written(blank_dev["name"])
+            except Exception as exc:
+                BackupLog.objects.create(
+                    level="error",
+                    msg=(
+                        "Can't prepare backup drive automatically.\n\n" +
+                        str(exc)
+                    ),
+                )
+            else:
+                # Re-read block devices so that the freshly written partition
+                # table appears in *lsblk* output.
+                lsblk_data = json.loads(subprocess.check_output(
+                    'lsblk --output NAME,HOTPLUG,MOUNTPOINT,FSTYPE,TYPE,LABEL,PARTLABEL  --json',
+                    shell=True
+                ).decode())['blockdevices']
+                backup_device = get_backup_device(lsblk_data)
+
     if not backup_device:
         BackupLog.objects.create(
             level='warning',
-            msg="Can't backup. No external exFAT backup device on this machine."
+            msg="Can't backup. No external BACKUP partition found and no blank removable device was available."
         )
         return
 
-    if lvm_partition.get('partlabel'):
+    if backup_device.get('partlabel'):
         sd_mountpoint = f"/media/{backup_device['partlabel']}"
-    elif lvm_partition.get('label'):
+    elif backup_device.get('label'):
         sd_mountpoint = f"/media/{backup_device['label']}"
     else:
         sd_mountpoint = f"/media/{backup_device['name']}"
@@ -237,6 +639,8 @@ def get_partitions():
 
 @celery_app.task
 def perform_backup():
+    from simo.core.models import Instance
+    from simo.core.middleware import drop_current_instance
     from simo.backups.models import BackupLog
     try:
         lv_group, lv_name, sd_mountpoint = get_partitions()
@@ -267,6 +671,15 @@ def perform_backup():
     mac = str(hex(uuid.getnode()))
     device_backups_path = f'{sd_mountpoint}/simo_backups/hub-{mac}'
 
+    if not os.path.exists(device_backups_path):
+        os.makedirs(device_backups_path)
+
+    drop_current_instance()
+    hub_meta = {
+        'instances': [inst.name for inst in Instance.objects.all()]
+    }
+    with open(os.path.join(device_backups_path, 'hub_meta.json'), 'w') as f:
+        f.write(json.dumps(hub_meta))
 
     now = datetime.now()
     month_folder = os.path.join(
@@ -288,6 +701,29 @@ def perform_backup():
                 f'borg init --encryption=none {month_folder}', shell=True
             )
 
+    # ------------------------------------------------------------------
+    # Ensure that files stored on *separate* partitions – most importantly
+    # the /boot (kernel & initrd images) and /boot/efi (EFI System
+    # Partition) – are included in the snapshot.  Otherwise the rescue
+    # procedure restores an empty /boot which leaves the system un-bootable
+    # once GRUB hands over control to the (missing) kernel.
+    #
+    # We temporarily bind-mount those paths into the read-only snapshot so
+    # that Borg treats them as regular directories residing on the same
+    # filesystem tree.
+    # ------------------------------------------------------------------
+
+    bind_mounts = []
+    for path in ("/boot", "/boot/efi"):
+        target = os.path.join(snap_mount_point, path.lstrip("/"))
+        # Create the mount-point inside the snapshot and bind-mount the live
+        # directory if it exists.
+        if os.path.ismount(path):
+            os.makedirs(target, exist_ok=True)
+            subprocess.run(["mount", "--bind", path, target], check=True)
+            bind_mounts.append(target)
+
+    # Directories that are safe to exclude – keep /boot out of this list!
     exclude_dirs = (
         'tmp', 'lost+found', 'proc', 'cdrom', 'dev', 'mnt', 'sys', 'run',
         'var/tmp', 'var/cache', 'var/log', 'media',
@@ -326,6 +762,11 @@ def perform_backup():
         stdout=subprocess.PIPE, stderr=subprocess.PIPE
     )
 
+    # Unmount previously created bind-mounts (boot / boot/efi) *before*
+    # removing the snapshot so that no busy references remain.
+    for mnt in reversed(bind_mounts):
+        subprocess.run(["umount", mnt])
+
     subprocess.run(["umount", snap_mount_point])
     subprocess.run(
         f"lvremove -f {lv_group}/{snap_name}", shell=True