py-pve-cloud-backup 0.0.1__py3-none-any.whl

pve_cloud_backup/fetcher/main.py

@@ -0,0 +1,86 @@
+from datetime import datetime
+import logging
+import funcs
+from nextcloud import backup_nextcloud
+from git import backup_git
+from proxmoxer import ProxmoxAPI
+import os
+import yaml
+from kubernetes import client, config
+from pprint import pformat
+import paramiko
+import asyncio
+
+
+logging.basicConfig(level=getattr(logging, os.getenv("LOG_LEVEL", "DEBUG").upper()))
+logger = logging.getLogger("fetcher")
+
+proxmox = ProxmoxAPI(
+  os.getenv("PROXMOXER_HOST"), user=os.getenv("PROXMOXER_USER"), backend='ssh_paramiko', private_key_file='/opt/id_proxmox'
+)
+
+with open("/opt/backup-conf.yaml", "r") as file:
+  backup_config = yaml.safe_load(file)
+
+backup_addr = backup_config["backup_daemon_address"]
+
+# main is prod and always runs in cluster
+config.load_incluster_config()
+v1 = client.CoreV1Api()
+
+
+async def run():
+  timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+
+  # backup to borg file repos
+
+  # defined and not null
+  if backup_config["git_repos"]:
+    await backup_git(backup_addr, timestamp, backup_config["git_repos"])
+  else:
+    logger.info("No git repos to backup provided, skipping")
+
+  # defined and not null
+  if backup_config["nextcloud_files"]:
+    await backup_nextcloud(backup_addr, timestamp, backup_config["nextcloud_files"])
+  else:
+    logger.info("No nextcloud files to backup provided, skipping")
+
+  # backup vms and k8s
+  raw_k8s_meta = None
+  raw_vm_meta = None
+  unique_pools = None
+
+  try:
+    k8s_kubeconfigs = funcs.get_kubernetes_clients(backup_config, proxmox, paramiko.Ed25519Key.from_private_key_file("/opt/id_qemu"))
+    logger.debug(f"k8s_kubeconfigs:\n{pformat(k8s_kubeconfigs)}")
+
+    raw_k8s_meta, k8s_stack_namespace_secrets = funcs.collect_raw_k8s_meta(backup_config, k8s_kubeconfigs)
+    logger.debug(f"k8s_meta:\n{pformat(raw_k8s_meta)}")
+  
+    raw_vm_meta = funcs.collect_raw_vm_meta(proxmox, backup_config)
+    logger.debug(f"vm_meta:\n{pformat(raw_vm_meta)}")
+
+    vm_conf_map = funcs.get_vm_configs(raw_vm_meta, paramiko.RSAKey.from_private_key_file("/opt/id_proxmox"))
+    logger.debug(f"vm_conf_map:\n{pformat(vm_conf_map)}")
+
+    # this simply adds all the images to groups inside of ceph
+    unique_pools = funcs.pool_images(raw_vm_meta, raw_k8s_meta)
+
+    # create group snapshots
+    funcs.snap_and_clone(raw_vm_meta, raw_k8s_meta, timestamp, unique_pools)
+    await funcs.send_backups(raw_vm_meta, raw_k8s_meta, timestamp, backup_addr)
+
+    await funcs.post_image_meta(raw_vm_meta, raw_k8s_meta, timestamp, backup_config, backup_addr)
+    await funcs.post_vm_stack_meta(raw_vm_meta, vm_conf_map, backup_config, backup_addr, timestamp)
+    await funcs.post_k8s_stack_meta(k8s_kubeconfigs, k8s_stack_namespace_secrets, timestamp, backup_addr)
+
+  finally:
+    # we always want to do the cleanup even if something failed
+    funcs.cleanup(raw_vm_meta, raw_k8s_meta, timestamp, unique_pools)
+    
+
+def main():
+  asyncio.run(run())
+
+  

pve_cloud_backup/fetcher/net.py

@@ -0,0 +1,126 @@
+import asyncio
+from enum import Enum
+import zstandard as zstd
+import struct
+import logging
+import pickle
+
+logger = logging.getLogger("fetcher")
+
+class Command(Enum):
+  ARCHIVE = 1
+  IMAGE_META = 2
+  STACK_META = 3
+
+
+async def archive_init(reader, writer, request_dict):
+  # intialize archive command
+  writer.write(struct.pack("B", Command.ARCHIVE.value))
+  await writer.drain()
+  logger.debug("send command")
+
+  # send the archive request dict
+  req_dict_pickled = pickle.dumps(request_dict)
+  writer.write(struct.pack("!I", len(req_dict_pickled)))
+  await writer.drain()
+  writer.write(req_dict_pickled)
+  await writer.drain()
+  logger.debug("send dict")
+
+  # wait for go signal, server needs to aquire write lock
+  # we dont 
+  logger.debug("waiting for go from bdd")
+  signal = await reader.readexactly(1)
+  if signal != b'\x01':
+    logger.error("recieved incorrect go signal")
+    raise Exception("Incorrect go signal!")
+  logger.debug("received go")
+
+
+async def send_cchunk(writer, compressed_chunk):
+  # compress the chunk
+  if compressed_chunk: # only send if something actually got compressed
+    # send size + chunk
+    writer.write(struct.pack("!I", len(compressed_chunk)))
+    await writer.drain()
+    writer.write(compressed_chunk)
+    await writer.drain()
+
+
+async def archive_async(backup_addr, request_dict, chunk_generator):
+  logger.info(request_dict)
+  reader, writer = await asyncio.open_connection(backup_addr, 8888)
+
+  await archive_init(reader, writer, request_dict)
+
+  # initialize the synchronous generator and start reading chunks, compress and send 
+  # compressor = zlib.compressobj(level=1)
+  compressor = zstd.ZstdCompressor(level=1, threads=6).compressobj()
+  async for chunk in chunk_generator():
+    await send_cchunk(writer, compressor.compress(chunk))
+
+  # send rest in compressor, compress doesnt always return a byte array, see bdd.py doc
+  # send size first again
+  await send_cchunk(writer, compressor.flush())
+
+  # send eof to server, signal that we are done
+  logger.debug("sending eof")
+  writer.write(struct.pack("!I", 0))
+  await writer.drain()
+
+  # close the writer here, stdout needs to be closed by caller
+  writer.close()
+  await writer.wait_closed()
+
+
+async def archive(backup_addr, request_dict, chunk_generator):
+  logger.info(request_dict)
+  reader, writer = await asyncio.open_connection(backup_addr, 8888)
+
+  await archive_init(reader, writer, request_dict)
+
+  # initialize the synchronous generator and start reading chunks, compress and send 
+  # compressor = zlib.compressobj(level=1)
+  compressor = zstd.ZstdCompressor(level=1, threads=6).compressobj()
+  for chunk in chunk_generator():
+    await send_cchunk(writer, compressor.compress(chunk))
+
+  # send rest in compressor, compress doesnt always return a byte array, see bdd.py doc
+  # send size first again
+  await send_cchunk(writer, compressor.flush())
+
+  # send eof to server, signal that we are done
+  logger.debug("sending eof")
+  writer.write(struct.pack("!I", 0))
+  await writer.drain()
+
+  # close the writer here, stdout needs to be closed by caller
+  writer.close()
+  await writer.wait_closed()
+
+
+async def meta(backup_addr, cmd, meta_dict):
+  reader, writer = await asyncio.open_connection(backup_addr, 8888)
+  writer.write(struct.pack("B", cmd.value))
+  await writer.drain()
+
+  meta_pickled = pickle.dumps(meta_dict)
+
+  # send size first
+  writer.write(struct.pack("!I", len(meta_pickled)))
+  await writer.drain()
+
+  # now send the dict
+  writer.write(meta_pickled)
+  await writer.drain()
+
+  writer.close()
+  await writer.wait_closed()
+
+
+async def image_meta(backup_addr, meta_dict):
+  await meta(backup_addr, Command.IMAGE_META, meta_dict)
+
+
+async def stack_meta(backup_addr, meta_dict):
+  await meta(backup_addr, Command.STACK_META, meta_dict)

pve_cloud_backup/fetcher/nextcloud.py

@@ -0,0 +1,42 @@
+import requests
+from requests.auth import HTTPBasicAuth
+import os
+import logging
+from net import archive
+
+
+logger = logging.getLogger("fetcher")
+
+password = None
+if os.path.isfile("/opt/nextcloud.pass"):
+  with open("/opt/nextcloud.pass", "r", encoding="utf-8") as file:
+    password = file.read()
+else:
+  logger.info("no nextcloud pass mounted, skipping nextcloud backup.")
+
+username = os.getenv("NEXTCLOUD_USER")
+
+nextcloud_base = os.getenv("NEXTCLOUD_BASE")
+
+
+async def backup_nextcloud(backup_addr, timestamp, nextcloud_files):
+  if password is None:
+    logger.info("no nextcloud pass mounted, skipping nextcloud backup.")
+    return
+
+  for file in nextcloud_files:
+    request_dict = {
+      "borg_archive_type": "nextcloud",
+      "archive_name": file,
+      "timestamp": timestamp,
+      "stdin_name": file
+    }
+
+    def chunk_generator():
+      response = requests.get(f"{nextcloud_base}/remote.php/dav/files/{username}/{file}", auth=HTTPBasicAuth(username, password), stream=True)
+      for chunk in response.iter_content(chunk_size=4 * 1024 * 1024):
+        if chunk:
+          yield chunk
+
+    await archive(backup_addr, request_dict, chunk_generator)
+

py_pve_cloud_backup-0.0.1.dist-info/METADATA

@@ -0,0 +1,12 @@
+Metadata-Version: 2.4
+Name: py-pve-cloud-backup
+Version: 0.0.1
+Author-email: Tobias Huebner <tobias.huebner@vmzberlin.com>
+License-Expression: GPL-3.0-or-later
+Requires-Dist: proxmoxer==2.2.0
+Requires-Dist: kubernetes==34.1.0
+Requires-Dist: PyYAML==6.0.2
+Requires-Dist: paramiko==4.0.0
+Requires-Dist: requests==2.32.5
+Requires-Dist: zstandard==0.25.0
+Requires-Dist: tinydb==4.8.2

py_pve_cloud_backup-0.0.1.dist-info/RECORD

@@ -0,0 +1,13 @@
+pve_cloud_backup/daemon/bdd.py,sha256=iGc1v7RsPQ2Vd-WcthTrFzEjd2jLeI3YnXYXM876U2Y,5592
+pve_cloud_backup/daemon/brctl.py,sha256=BKR2rxMAvfiLxuwfSN8xiZ4eux3TeQlMMbA096jgoKw,8855
+pve_cloud_backup/daemon/shared.py,sha256=Afbbafs0FchUiofmhAKEr01zaE3jxL_nS_KUBViW6Rs,20729
+pve_cloud_backup/fetcher/funcs.py,sha256=IXaDX0_9Fj2GnUIzAl5nDpk-umn27Sr5X3SYZCzeVKU,22199
+pve_cloud_backup/fetcher/git.py,sha256=km_s5SoT-CIZ_RPGXoM9ieNMXXeUmyjLDn9DELh0QZA,1082
+pve_cloud_backup/fetcher/main.py,sha256=viT04NeyWuDmMsm5_XkCRCGszGWmYnx--o9Z5IiZIaE,2889
+pve_cloud_backup/fetcher/net.py,sha256=MVCqdb3ZFI8gbeUB2NrWrkykwuKAFOznH1gqtqyT4OE,3768
+pve_cloud_backup/fetcher/nextcloud.py,sha256=HNb6rM-3t-GOZSJnWppKai1a4Yugx6lh9vuOlA4drdk,1166
+py_pve_cloud_backup-0.0.1.dist-info/METADATA,sha256=SAcgrc7oK3HBnEch46ZWGR91wcgFwGijA2CE5if78eI,380
+py_pve_cloud_backup-0.0.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+py_pve_cloud_backup-0.0.1.dist-info/entry_points.txt,sha256=0QG_j55-wiJS1_vLNsjju0Ly9ycpBSTOhUF-PlWizOM,148
+py_pve_cloud_backup-0.0.1.dist-info/top_level.txt,sha256=w9KT8Yar3s02YhN-DBabObecqlIub36RT6lLZ4iG9dE,17
+py_pve_cloud_backup-0.0.1.dist-info/RECORD,,

py_pve_cloud_backup-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

py_pve_cloud_backup-0.0.1.dist-info/entry_points.txt

@@ -0,0 +1,4 @@
+[console_scripts]
+brctl = pve_cloud_backup.daemon.brctl:main
+daemon = pve_cloud_backup.daemon.bdd:main
+fetcher = pve_cloud_backup.fetcher.main:main

py_pve_cloud_backup-0.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+pve_cloud_backup