py-pve-cloud-backup 0.0.1__py3-none-any.whl

pve_cloud_backup/daemon/bdd.py

@@ -0,0 +1,169 @@
+from tinydb import TinyDB
+import subprocess
+from pathlib import Path
+import logging
+from shared import IMAGE_META_DB_PATH, STACK_META_DB_PATH, BACKUP_DIR, copy_backup_generic, RBD_REPO_TYPES
+import os
+from enum import Enum
+import asyncio
+import struct
+import pickle
+import zstandard as zstd
+
+
+log_level_str = os.getenv("LOG_LEVEL", "INFO").upper()
+log_level = getattr(logging, log_level_str, logging.INFO)
+
+logging.basicConfig(level=log_level)
+logger = logging.getLogger("bdd")
+
+ENV = os.getenv("ENV", "TESTING")
+
+FILE_REPO_TYPES = ["nextcloud", "git"] # different borg archives for each
+
+
+def init_backup_dir():
+  if ENV == "TESTING":
+    Path(BACKUP_DIR).mkdir(parents=True, exist_ok=True)
+
+  for repo_type in RBD_REPO_TYPES:
+    repo_path = f"{BACKUP_DIR}/borg-{repo_type}"
+    Path(repo_path).mkdir(parents=True, exist_ok=True)
+
+    # init borg repo, is ok to fail if it already exists
+    subprocess.run(["borg", "init", "--encryption=none", repo_path])
+
+  for file_type in FILE_REPO_TYPES:
+    repo_path = f"{BACKUP_DIR}/borg-{file_type}"
+    Path(repo_path).mkdir(parents=True, exist_ok=True)
+
+    # init borg repo, is ok to fail if it already exists
+    subprocess.run(["borg", "init", "--encryption=none", repo_path])
+
+  if ENV == 'PRODUCTION':
+    copy_backup_generic()
+
+
+class Command(Enum):
+  ARCHIVE = 1
+  IMAGE_META = 2
+  STACK_META = 3
+
+
+lock_dict = {}
+
+def get_lock(lock_type):
+  if lock_type not in RBD_REPO_TYPES and lock_type not in FILE_REPO_TYPES and lock_type not in ["stack", "image"]:
+    raise Exception(f"Unknown type {lock_type}")
+  
+  if lock_type not in lock_dict:
+    lock_dict[lock_type] = asyncio.Lock()
+  
+  return lock_dict[lock_type]
+
+
+async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWriter):
+  addr = writer.get_extra_info('peername')
+  logger.info(f"Connection from {addr}")
+  
+  command = Command(struct.unpack('B', await reader.read(1))[0])
+  logger.info(f"{addr} send command: {command}")
+
+  try:
+    match command:
+      case Command.ARCHIVE:
+        # each archive request starts with a pickled dict containing parameters
+        dict_size = struct.unpack('!I', (await reader.readexactly(4)))[0]
+        req_dict = pickle.loads((await reader.readexactly(dict_size)))
+        logger.info(req_dict)
+
+        # extract the parameters
+        borg_archive_type = req_dict["borg_archive_type"] # borg locks 
+        archive_name = req_dict["archive_name"]
+        timestamp = req_dict["timestamp"]
+
+        # lock locally, we have one borg archive per archive type
+        async with get_lock(borg_archive_type):
+          borg_archive = f"{BACKUP_DIR}/borg-{borg_archive_type}::{archive_name}_{timestamp}"
+          logger.info(f"accuired lock {borg_archive_type}")
+
+          # send continue signal, meaning we have the lock and export can start.
+          writer.write(b'\x01')  # signal = 0x01 means "continue"
+          await writer.drain()
+          logger.debug("send go")
+
+          # initialize the borg subprocess we will pipe the received content to
+          # decompressor = zlib.decompressobj()
+          decompressor = zstd.ZstdDecompressor().decompressobj()
+          borg_proc = await asyncio.create_subprocess_exec(
+            "borg", "create", "--compression", "zstd,1",
+            "--stdin-name", req_dict["stdin_name"],
+            borg_archive, "-",
+            stdin=asyncio.subprocess.PIPE
+          )
+
+          # read compressed chunks
+          while True:
+            # client first always sends chunk size
+            chunk_size = struct.unpack("!I", (await reader.readexactly(4)))[0]
+            if chunk_size == 0:
+              break # client sends 0 chunk size at the end to signal that its finished uploading
+            chunk = await reader.readexactly(chunk_size)
+            
+            # decompress and write
+            decompressed_chunk = decompressor.decompress(chunk)
+            if decompressed_chunk:
+              borg_proc.stdin.write(decompressed_chunk)
+              await borg_proc.stdin.drain()
+
+          # the decompressor does not always return a decompressed chunk but might retain 
+          # and return empty. at the end we need to call flush to get everything out
+          borg_proc.stdin.write(decompressor.flush())
+          await borg_proc.stdin.drain()
+
+          # close the proc stdin pipe, writer gets closed in finally
+          borg_proc.stdin.close()
+          exit_code = await borg_proc.wait()
+
+          if exit_code != 0:
+            raise Exception(f"Borg failed with code {exit_code}")
+
+      case Command.STACK_META:
+        # read meta dict size
+        dict_size = struct.unpack('!I', (await reader.readexactly(4)))[0]
+        meta_dict = pickle.loads((await reader.readexactly(dict_size)))
+
+        async with get_lock("stack"):
+          meta_db = TinyDB(STACK_META_DB_PATH)
+          meta_db.insert(meta_dict)
+
+      case Command.IMAGE_META:
+        dict_size = struct.unpack('!I', (await reader.readexactly(4)))[0]
+        meta_dict = pickle.loads((await reader.readexactly(dict_size)))
+
+        async with get_lock("image"):
+          meta_db = TinyDB(IMAGE_META_DB_PATH)
+          meta_db.insert(meta_dict)
+
+  except asyncio.IncompleteReadError as e:
+    logger.error("Client disconnected", e)
+  finally:
+    writer.close()
+    # dont await on server side
+
+
+async def run():
+  init_backup_dir()
+
+  server = await asyncio.start_server(handle_client, "0.0.0.0", 8888)
+  addr = server.sockets[0].getsockname()
+  logger.info(f"Serving on {addr}")
+  async with server:
+      await server.serve_forever()
+
+
+def main():
+  asyncio.run(run())
+
+  
+

pve_cloud_backup/daemon/brctl.py

@@ -0,0 +1,221 @@
+import argparse
+import logging
+from kubernetes import client
+from kubernetes.config.kube_config import KubeConfigLoader
+import shared
+from proxmoxer import ProxmoxAPI
+from tinydb import TinyDB, Query
+from pprint import pformat
+import yaml
+import pickle
+import base64
+import os
+
+log_level_str = os.getenv("LOG_LEVEL", "INFO").upper()
+log_level = getattr(logging, log_level_str, logging.INFO)
+
+logging.basicConfig(level=log_level)
+logger = logging.getLogger("brctl")
+
+
+def list_backup_details(args):  
+  print(f"listing details for {args.timestamp}")
+  timestamp_archives = shared.get_image_metas(args, args.timestamp)
+
+  metas = timestamp_archives[args.timestamp]
+
+  # first we group metas
+  k8s_stacks = {}
+  vm_stacks = {}
+
+  for meta in metas:
+    if meta["type"] == "k8s":
+      if meta["stack"] not in k8s_stacks:
+        k8s_stacks[meta["stack"]] = []
+
+      k8s_stacks[meta["stack"]].append(meta)
+
+    elif meta["type"] in ["lxc", "qemu"]:
+      if meta["stack"] not in vm_stacks:
+        vm_stacks[meta["stack"]] = []
+      
+      vm_stacks[meta["stack"]].append(meta)
+
+    else:
+      raise Exception(f"Invalid meta type found - meta {meta}")
+
+
+  for k8s_stack, k8s_metas in k8s_stacks.items():
+    print(f"  - k8s stack {k8s_stack}:")
+
+    # get stack meta and decode stack namespace secrets
+    stack_meta_db = TinyDB(f"{args.backup_path}stack-meta-db.json")
+    Meta = Query()
+
+    stack_meta = stack_meta_db.get((Meta.timestamp == args.timestamp) & (Meta.stack == k8s_stack) & (Meta.type == "k8s"))
+    
+    namespace_secret_dict = pickle.loads(base64.b64decode(stack_meta["namespace_secret_dict_b64"]))
+
+    namespace_k8s_metas = {}
+
+    # group metas by namespace
+    for meta in k8s_metas:
+      if meta["namespace"] not in namespace_k8s_metas:
+        namespace_k8s_metas[meta["namespace"]] = []
+
+      namespace_k8s_metas[meta["namespace"]].append(meta)
+
+    for namespace, k8s_metas in namespace_k8s_metas.items():
+      print(f"    - namespace {namespace}:")
+      print(f"      - volumes:")
+      for meta in k8s_metas:
+        pvc_name = meta["pvc_name"]
+        pool = meta["pool"]
+        storage_class = meta["storage_class"]
+        print(f"        - {pvc_name}, pool {pool}, storage class {storage_class}")
+      
+      print(f"      - secrets:")
+      for secret in namespace_secret_dict[namespace]:
+        secret_name = secret["metadata"]["name"]
+        print(f"        - {secret_name}")
+
+
+  for stack, vm_metas in vm_stacks.items():
+    print(f"  - vm stack {stack}:")
+    
+    for meta in vm_metas:
+      vmid = meta["vmid"]
+      pool = meta["pool"]
+      image_name = meta["image_name"]
+      type = meta["type"]
+      print(f"    - vmid {vmid}, disk {image_name}, pool {pool}, vm type {type}")
+
+
+def list_backups(args):
+  timestamp_archives = shared.get_image_metas(args)
+
+  print("available backup timestamps (ids):")
+
+  for timestamp in sorted(timestamp_archives):
+    print(f"- timestamp {timestamp}")
+
+  
+
+def restore_vms(args):
+  print(f"restoring {args.timestamp}")
+
+  logger.debug(f"pool mappings {args.pool_mapping}")
+
+  # init proxmoxer
+  if args.proxmox_host and args.proxmox_private_key:
+    proxmox = ProxmoxAPI(
+      args.proxmox_host, user="root", backend='ssh_paramiko', private_key_file=args.proxmox_private_key
+    )
+  else:
+    proxmox = ProxmoxAPI(
+      "localhost", user="root", backend='ssh_paramiko'
+    )
+
+  metas = shared.get_image_metas(args, args.timestamp)[args.timestamp]
+
+  metas_grouped = shared.group_image_metas(metas, ["lxc", "qemu"], "vmid")
+  stack_metas = shared.get_stack_metas(args, args.timestamp, ["lxc", "qemu"], "vmid")
+
+  shared.restore_images(metas_grouped, stack_metas, args, proxmox)
+
+
+# this assumes you first restored the virtual machines
+# and extracted a fitting kubeconfig passing it via --kubeconfig
+def restore_k8s(args):
+  print(f"restoring {args.timestamp}")
+
+  metas = shared.get_image_metas(args, args.timestamp)[args.timestamp]
+
+  metas_grouped = shared.group_image_metas(metas, ["k8s"], "namespace", args.k8s_stack_name)
+
+  stack_meta_db = TinyDB(f"{args.backup_path}stack-meta-db.json")
+  Meta = Query()
+
+  stack_meta = stack_meta_db.get((Meta.timestamp == args.timestamp) & (Meta.stack == args.k8s_stack_name) & (Meta.type == "k8s"))
+  logger.debug(f"stack meta {stack_meta}")
+  namespace_secret_dict = pickle.loads(base64.b64decode(stack_meta["namespace_secret_dict_b64"]))
+
+  # user can manually specify it
+  if args.kubeconfig_new:
+    with open(args.kubeconfig_new, "r") as file:
+      kubeconfig_dict = yaml.safe_load(file)
+  else:
+    # restore into original k8s cluster
+    master_ipv4 = stack_meta["master_ip"]
+    kubeconfig_dict = yaml.safe_load(stack_meta["raw_kubeconfig"])
+
+    # override the connection ip as it is set to localhost on the machines
+    kubeconfig_dict["clusters"][0]["cluster"]["server"] = f"https://{master_ipv4}:6443"
+
+  logger.debug(f"kubeconfig dict {pformat(kubeconfig_dict)}")
+
+  # init kube client
+  loader = KubeConfigLoader(config_dict=kubeconfig_dict)
+  configuration = client.Configuration()
+  loader.load_and_set(configuration)
+
+  # Create a client from this configuration
+  api_client = client.ApiClient(configuration)
+
+  # run the restore
+  shared.restore_pvcs(metas_grouped, namespace_secret_dict, args, api_client)
+
+
+# dynamic backup path function for the --backup-path argument
+def backup_path(value):
+  if value == "":
+    return ""
+  
+  if value.endswith("/"):
+    return value
+  else:
+    return value + "/"
+
+
+# purpose of these tools is disaster recovery into an identical pve + ceph system
+# assumes to be run on a pve system, but can be passed pve host and path to ssh key aswell
+def main():
+  parser = argparse.ArgumentParser(description="CLI for restoring backups.")
+
+  base_parser = argparse.ArgumentParser(add_help=False)
+  base_parser.add_argument("--backup-path", type=backup_path, default=".", help="Path of the mounted backup drive/dir.")
+  base_parser.add_argument("--proxmox-host", type=str, help="Proxmox host, if not run directly on a pve node.")
+  base_parser.add_argument("--proxmox-private-key", type=str, help="Path to pve root private key, for connecting to remote pve.")
+
+  subparsers = parser.add_subparsers(dest="command", required=True)
+
+  list_parser = subparsers.add_parser("list-backups", help="List available backups.", parents=[base_parser])
+  list_parser.set_defaults(func=list_backups)
+
+  list_detail_parser = subparsers.add_parser("backup-details", help="List details of a backup.", parents=[base_parser])
+  list_detail_parser.add_argument("--timestamp", type=str, help="Timestamp of the backup to list details of.", required=True)
+  list_detail_parser.set_defaults(func=list_backup_details)
+
+  restore_parser = subparsers.add_parser("restore-vms", help="Restore vm (qemu/lxc) backups.", parents=[base_parser])
+  restore_parser.add_argument("--timestamp", type=str, help="Timestamp of the backup to restore.", required=True)
+  restore_parser.add_argument("--stack-names", type=str, default="", help="Specific stacks to restore, CSV, acts as a filter.")
+  restore_parser.add_argument("--pool-mapping", action="append", help="Define pool mappings (old to new), for example ssd:nvme.")
+  restore_parser.set_defaults(func=restore_vms)
+
+  k8s_restore_parser = subparsers.add_parser("restore-k8s", help="Restore k8s csi backups. If pvcs with same name exist, test-restore will be appended to pvc name.", parents=[base_parser])
+  k8s_restore_parser.add_argument("--timestamp", type=str, help="Timestamp of the backup to restore.", required=True)
+  k8s_restore_parser.add_argument("--k8s-stack-name", type=str, help="Stack name of k8s stack that will be restored into.", required=True)
+  k8s_restore_parser.add_argument("--kubeconfig-new", type=str, help="Optional kubeconfig for new cluster restores.")
+  k8s_restore_parser.add_argument("--namespaces", type=str, default="", help="Specific namespaces to restore, CSV, acts as a filter. Use with --pool-mapping for controlled migration of pvcs.")
+  k8s_restore_parser.add_argument("--pool-sc-mapping", action="append", help="Define pool storage class mappings (old to new), for example old-pool:new-pool/new-storage-class-name.")
+  k8s_restore_parser.add_argument("--auto-scale", action="store_true", help="When passed deployments and stateful sets will automatically get scaled down and back up again for restore.")
+  k8s_restore_parser.add_argument("--auto-delete", action="store_true", help="When passed existing pvcs in namespace will automatically get deleted before restoring.")
+  k8s_restore_parser.add_argument("--secret-pattern", action="append", help="Define as many times as you need, for example namespace/deployment* (glob style). Will overwrite secret data of matching existing.")
+  k8s_restore_parser.set_defaults(func=restore_k8s)
+
+  args = parser.parse_args()
+  args.func(args)
+
+
+if __name__ == "__main__":
+  main()