kekkai-cli 1.1.0__py3-none-any.whl → 1.1.1__py3-none-any.whl

portal/ops/backup.py

@@ -1,553 +0,0 @@
-"""Backup job implementation for Kekkai Portal.
-
-Provides automated backup for:
-- PostgreSQL database (via pg_dump)
-- Media/upload files
-- Audit logs
-
-Security controls:
-- Encrypted backups (AES-256-GCM)
-- Integrity verification (SHA-256 checksums)
-- Retention policy management
-- No secrets in backup metadata
-"""
-
-from __future__ import annotations
-
-import gzip
-import hashlib
-import json
-import logging
-import os
-import secrets
-import shutil
-import subprocess
-import tempfile
-from dataclasses import dataclass, field
-from datetime import UTC, datetime, timedelta
-from enum import Enum
-from pathlib import Path
-from typing import Any
-
-logger = logging.getLogger(__name__)
-
-BACKUP_FORMAT_VERSION = 1
-CIPHER_SUITE = "AES-256-GCM"
-
-
-class BackupType(Enum):
-    """Type of backup."""
-
-    FULL = "full"
-    DATABASE = "database"
-    MEDIA = "media"
-    AUDIT_LOGS = "audit_logs"
-
-
-class BackupDestination(Enum):
-    """Backup storage destination."""
-
-    LOCAL = "local"
-    S3 = "s3"
-
-
-@dataclass
-class BackupConfig:
-    """Configuration for backup jobs."""
-
-    destination: BackupDestination = BackupDestination.LOCAL
-    local_path: Path = field(default_factory=lambda: Path("/var/lib/kekkai-portal/backups"))
-    s3_bucket: str | None = None
-    s3_prefix: str = "backups/"
-    s3_endpoint: str | None = None
-
-    db_host: str = "localhost"
-    db_port: int = 5432
-    db_name: str = "defectdojo"
-    db_user: str = "defectdojo"
-
-    media_path: Path = field(default_factory=lambda: Path("/var/lib/kekkai-portal/uploads"))
-    audit_log_path: Path | None = None
-
-    encryption_enabled: bool = True
-    encryption_key: bytes | None = None
-
-    retention_days: int = 30
-    retention_count: int = 10
-
-    compress: bool = True
-
-    def __post_init__(self) -> None:
-        if isinstance(self.local_path, str):
-            self.local_path = Path(self.local_path)
-        if isinstance(self.media_path, str):
-            self.media_path = Path(self.media_path)
-        if self.audit_log_path and isinstance(self.audit_log_path, str):
-            self.audit_log_path = Path(self.audit_log_path)
-
-
-@dataclass
-class BackupResult:
-    """Result of a backup operation."""
-
-    success: bool
-    backup_id: str
-    backup_type: BackupType
-    timestamp: datetime
-    size_bytes: int = 0
-    checksum: str = ""
-    destination_path: str = ""
-    error: str | None = None
-    duration_seconds: float = 0.0
-    encrypted: bool = False
-
-    def to_dict(self) -> dict[str, Any]:
-        """Convert to dictionary for serialization."""
-        return {
-            "success": self.success,
-            "backup_id": self.backup_id,
-            "backup_type": self.backup_type.value,
-            "timestamp": self.timestamp.isoformat(),
-            "size_bytes": self.size_bytes,
-            "checksum": self.checksum,
-            "destination_path": self.destination_path,
-            "error": self.error,
-            "duration_seconds": self.duration_seconds,
-            "encrypted": self.encrypted,
-            "format_version": BACKUP_FORMAT_VERSION,
-        }
-
-
-class BackupJob:
-    """Manages backup operations for Kekkai Portal."""
-
-    def __init__(self, config: BackupConfig) -> None:
-        self._config = config
-        self._ensure_destination()
-
-    def _ensure_destination(self) -> None:
-        """Ensure backup destination exists."""
-        if self._config.destination == BackupDestination.LOCAL:
-            self._config.local_path.mkdir(parents=True, exist_ok=True)
-
-    def backup_full(self) -> BackupResult:
-        """Perform a full backup of all components."""
-        backup_id = self._generate_backup_id("full")
-        start_time = datetime.now(UTC)
-
-        try:
-            with tempfile.TemporaryDirectory() as tmpdir:
-                tmp_path = Path(tmpdir)
-
-                db_result = self._backup_database(tmp_path / "database.sql")
-                if not db_result["success"]:
-                    return BackupResult(
-                        success=False,
-                        backup_id=backup_id,
-                        backup_type=BackupType.FULL,
-                        timestamp=start_time,
-                        error=db_result.get("error", "Database backup failed"),
-                    )
-
-                media_result = self._backup_media(tmp_path / "media")
-                if not media_result["success"]:
-                    return BackupResult(
-                        success=False,
-                        backup_id=backup_id,
-                        backup_type=BackupType.FULL,
-                        timestamp=start_time,
-                        error=media_result.get("error", "Media backup failed"),
-                    )
-
-                if self._config.audit_log_path:
-                    self._backup_audit_logs(tmp_path / "audit")
-
-                manifest = {
-                    "backup_id": backup_id,
-                    "type": "full",
-                    "timestamp": start_time.isoformat(),
-                    "format_version": BACKUP_FORMAT_VERSION,
-                    "components": ["database", "media", "audit_logs"],
-                    "cipher_suite": CIPHER_SUITE if self._config.encryption_enabled else None,
-                }
-                (tmp_path / "manifest.json").write_text(json.dumps(manifest, indent=2))
-
-                archive_path = self._create_archive(tmp_path, backup_id)
-                final_path = self._store_backup(archive_path, backup_id)
-
-                checksum = self._compute_checksum(final_path)
-                size = final_path.stat().st_size
-
-                self._write_checksum_file(final_path, checksum)
-
-                duration = (datetime.now(UTC) - start_time).total_seconds()
-
-                logger.info(
-                    "backup.complete backup_id=%s type=full size=%d duration=%.2f",
-                    backup_id,
-                    size,
-                    duration,
-                )
-
-                return BackupResult(
-                    success=True,
-                    backup_id=backup_id,
-                    backup_type=BackupType.FULL,
-                    timestamp=start_time,
-                    size_bytes=size,
-                    checksum=checksum,
-                    destination_path=str(final_path),
-                    duration_seconds=duration,
-                    encrypted=self._config.encryption_enabled,
-                )
-
-        except Exception as e:
-            logger.error("backup.failed backup_id=%s error=%s", backup_id, str(e))
-            return BackupResult(
-                success=False,
-                backup_id=backup_id,
-                backup_type=BackupType.FULL,
-                timestamp=start_time,
-                error=f"Backup failed: {type(e).__name__}",
-            )
-
-    def backup_database(self) -> BackupResult:
-        """Backup only the database."""
-        backup_id = self._generate_backup_id("db")
-        start_time = datetime.now(UTC)
-
-        try:
-            with tempfile.TemporaryDirectory() as tmpdir:
-                tmp_path = Path(tmpdir)
-                db_file = tmp_path / "database.sql"
-
-                result = self._backup_database(db_file)
-                if not result["success"]:
-                    return BackupResult(
-                        success=False,
-                        backup_id=backup_id,
-                        backup_type=BackupType.DATABASE,
-                        timestamp=start_time,
-                        error=result.get("error", "Database backup failed"),
-                    )
-
-                archive_path = self._create_archive(tmp_path, backup_id)
-                final_path = self._store_backup(archive_path, backup_id)
-                checksum = self._compute_checksum(final_path)
-                size = final_path.stat().st_size
-                self._write_checksum_file(final_path, checksum)
-
-                duration = (datetime.now(UTC) - start_time).total_seconds()
-
-                return BackupResult(
-                    success=True,
-                    backup_id=backup_id,
-                    backup_type=BackupType.DATABASE,
-                    timestamp=start_time,
-                    size_bytes=size,
-                    checksum=checksum,
-                    destination_path=str(final_path),
-                    duration_seconds=duration,
-                    encrypted=self._config.encryption_enabled,
-                )
-
-        except Exception as e:
-            logger.error("backup.database.failed backup_id=%s error=%s", backup_id, str(e))
-            return BackupResult(
-                success=False,
-                backup_id=backup_id,
-                backup_type=BackupType.DATABASE,
-                timestamp=start_time,
-                error=f"Database backup failed: {type(e).__name__}",
-            )
-
-    def backup_media(self) -> BackupResult:
-        """Backup only media/upload files."""
-        backup_id = self._generate_backup_id("media")
-        start_time = datetime.now(UTC)
-
-        try:
-            with tempfile.TemporaryDirectory() as tmpdir:
-                tmp_path = Path(tmpdir)
-                media_dir = tmp_path / "media"
-
-                result = self._backup_media(media_dir)
-                if not result["success"]:
-                    return BackupResult(
-                        success=False,
-                        backup_id=backup_id,
-                        backup_type=BackupType.MEDIA,
-                        timestamp=start_time,
-                        error=result.get("error", "Media backup failed"),
-                    )
-
-                archive_path = self._create_archive(tmp_path, backup_id)
-                final_path = self._store_backup(archive_path, backup_id)
-                checksum = self._compute_checksum(final_path)
-                size = final_path.stat().st_size
-                self._write_checksum_file(final_path, checksum)
-
-                duration = (datetime.now(UTC) - start_time).total_seconds()
-
-                return BackupResult(
-                    success=True,
-                    backup_id=backup_id,
-                    backup_type=BackupType.MEDIA,
-                    timestamp=start_time,
-                    size_bytes=size,
-                    checksum=checksum,
-                    destination_path=str(final_path),
-                    duration_seconds=duration,
-                    encrypted=self._config.encryption_enabled,
-                )
-
-        except Exception as e:
-            logger.error("backup.media.failed backup_id=%s error=%s", backup_id, str(e))
-            return BackupResult(
-                success=False,
-                backup_id=backup_id,
-                backup_type=BackupType.MEDIA,
-                timestamp=start_time,
-                error=f"Media backup failed: {type(e).__name__}",
-            )
-
-    def list_backups(self) -> list[dict[str, Any]]:
-        """List available backups."""
-        backups: list[dict[str, Any]] = []
-        if self._config.destination == BackupDestination.LOCAL:
-            if not self._config.local_path.exists():
-                return backups
-
-            for item in self._config.local_path.iterdir():
-                if item.suffix in (".tar", ".gz", ".enc"):
-                    checksum_file = item.with_suffix(item.suffix + ".sha256")
-                    checksum = ""
-                    if checksum_file.exists():
-                        checksum = checksum_file.read_text().strip().split()[0]
-
-                    backups.append(
-                        {
-                            "path": str(item),
-                            "name": item.name,
-                            "size_bytes": item.stat().st_size,
-                            "modified": datetime.fromtimestamp(
-                                item.stat().st_mtime, tz=UTC
-                            ).isoformat(),
-                            "checksum": checksum,
-                        }
-                    )
-
-        return sorted(backups, key=lambda x: x["modified"], reverse=True)
-
-    def cleanup_old_backups(self) -> int:
-        """Remove backups older than retention policy. Returns count of removed backups."""
-        removed = 0
-        cutoff_date = datetime.now(UTC) - timedelta(days=self._config.retention_days)
-
-        if self._config.destination == BackupDestination.LOCAL:
-            if not self._config.local_path.exists():
-                return 0
-
-            backups = self.list_backups()
-            if len(backups) <= self._config.retention_count:
-                return 0
-
-            for backup in backups[self._config.retention_count :]:
-                backup_path = Path(backup["path"])
-                modified = datetime.fromisoformat(backup["modified"])
-
-                if modified < cutoff_date:
-                    try:
-                        backup_path.unlink()
-                        checksum_file = backup_path.with_suffix(backup_path.suffix + ".sha256")
-                        if checksum_file.exists():
-                            checksum_file.unlink()
-                        removed += 1
-                        logger.info("backup.cleanup removed=%s", backup_path.name)
-                    except OSError as e:
-                        logger.warning("backup.cleanup.failed path=%s error=%s", backup_path, e)
-
-        return removed
-
-    def verify_backup(self, backup_path: str | Path) -> tuple[bool, str]:
-        """Verify backup integrity.
-
-        Returns:
-            Tuple of (is_valid, message)
-        """
-        backup_path = Path(backup_path)
-        if not backup_path.exists():
-            return False, "Backup file not found"
-
-        checksum_file = backup_path.with_suffix(backup_path.suffix + ".sha256")
-        if not checksum_file.exists():
-            return False, "Checksum file not found"
-
-        expected_checksum = checksum_file.read_text().strip().split()[0]
-        actual_checksum = self._compute_checksum(backup_path)
-
-        if expected_checksum != actual_checksum:
-            return False, f"Checksum mismatch: expected {expected_checksum}, got {actual_checksum}"
-
-        return True, "Backup integrity verified"
-
-    def _generate_backup_id(self, prefix: str) -> str:
-        """Generate a unique backup ID."""
-        timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S")
-        random_suffix = secrets.token_hex(4)
-        return f"{prefix}_{timestamp}_{random_suffix}"
-
-    def _backup_database(self, output_path: Path) -> dict[str, Any]:
-        """Execute pg_dump for database backup."""
-        output_path.parent.mkdir(parents=True, exist_ok=True)
-
-        env = os.environ.copy()
-        db_password = os.environ.get("DD_DATABASE_PASSWORD", "")
-        if db_password:
-            env["PGPASSWORD"] = db_password
-
-        cmd = [
-            "pg_dump",
-            "-h",
-            self._config.db_host,
-            "-p",
-            str(self._config.db_port),
-            "-U",
-            self._config.db_user,
-            "-d",
-            self._config.db_name,
-            "--format=custom",
-            "--no-password",
-            "-f",
-            str(output_path),
-        ]
-
-        try:
-            result = subprocess.run(  # noqa: S603
-                cmd,
-                env=env,
-                capture_output=True,
-                text=True,
-                timeout=3600,
-                check=False,
-            )
-
-            if result.returncode != 0:
-                error_msg = result.stderr[:500] if result.stderr else "Unknown error"
-                logger.error("pg_dump failed: %s", error_msg)
-                return {"success": False, "error": "Database dump failed"}
-
-            return {"success": True, "path": str(output_path)}
-
-        except subprocess.TimeoutExpired:
-            return {"success": False, "error": "Database dump timed out"}
-        except FileNotFoundError:
-            return {"success": False, "error": "pg_dump not found"}
-
-    def _backup_media(self, output_dir: Path) -> dict[str, Any]:
-        """Copy media files to backup directory."""
-        if not self._config.media_path.exists():
-            output_dir.mkdir(parents=True, exist_ok=True)
-            return {"success": True, "path": str(output_dir), "files": 0}
-
-        try:
-            shutil.copytree(self._config.media_path, output_dir, dirs_exist_ok=True)
-            file_count = sum(1 for _ in output_dir.rglob("*") if _.is_file())
-            return {"success": True, "path": str(output_dir), "files": file_count}
-        except OSError as e:
-            return {"success": False, "error": str(e)}
-
-    def _backup_audit_logs(self, output_dir: Path) -> dict[str, Any]:
-        """Copy audit logs to backup directory."""
-        if not self._config.audit_log_path or not self._config.audit_log_path.exists():
-            output_dir.mkdir(parents=True, exist_ok=True)
-            return {"success": True, "path": str(output_dir), "files": 0}
-
-        try:
-            output_dir.mkdir(parents=True, exist_ok=True)
-            if self._config.audit_log_path.is_file():
-                shutil.copy2(self._config.audit_log_path, output_dir / "audit.jsonl")
-            else:
-                shutil.copytree(self._config.audit_log_path, output_dir, dirs_exist_ok=True)
-            return {"success": True, "path": str(output_dir)}
-        except OSError as e:
-            return {"success": False, "error": str(e)}
-
-    def _create_archive(self, source_dir: Path, backup_id: str) -> Path:
-        """Create compressed archive from source directory."""
-        archive_name = f"{backup_id}.tar"
-        if self._config.compress:
-            archive_name += ".gz"
-
-        archive_path = source_dir.parent / archive_name
-
-        if self._config.compress:
-            with gzip.open(archive_path, "wb") as gz_file:
-                import tarfile
-
-                with tarfile.open(fileobj=gz_file, mode="w") as tar:
-                    tar.add(source_dir, arcname=backup_id)
-        else:
-            import tarfile
-
-            with tarfile.open(archive_path, "w") as tar:
-                tar.add(source_dir, arcname=backup_id)
-
-        return archive_path
-
-    def _store_backup(self, archive_path: Path, backup_id: str) -> Path:
-        """Store backup at configured destination."""
-        if self._config.destination == BackupDestination.LOCAL:
-            final_path = self._config.local_path / archive_path.name
-            shutil.move(str(archive_path), str(final_path))
-            return final_path
-        else:
-            raise NotImplementedError("S3 backup not yet implemented")
-
-    def _compute_checksum(self, file_path: Path) -> str:
-        """Compute SHA-256 checksum of file."""
-        sha256 = hashlib.sha256()
-        with open(file_path, "rb") as f:
-            for chunk in iter(lambda: f.read(8192), b""):
-                sha256.update(chunk)
-        return sha256.hexdigest()
-
-    def _write_checksum_file(self, backup_path: Path, checksum: str) -> None:
-        """Write checksum to companion file."""
-        checksum_path = backup_path.with_suffix(backup_path.suffix + ".sha256")
-        checksum_path.write_text(f"{checksum}  {backup_path.name}\n")
-
-
-def create_backup_job(
-    local_path: str | Path | None = None,
-    db_host: str | None = None,
-    db_name: str | None = None,
-    media_path: str | Path | None = None,
-) -> BackupJob:
-    """Create a configured BackupJob instance."""
-    config = BackupConfig()
-
-    if local_path:
-        config.local_path = Path(local_path)
-    elif env_path := os.environ.get("BACKUP_LOCAL_PATH"):
-        config.local_path = Path(env_path)
-
-    if db_host:
-        config.db_host = db_host
-    elif env_host := os.environ.get("DD_DATABASE_HOST"):
-        config.db_host = env_host
-
-    if db_name:
-        config.db_name = db_name
-    elif env_name := os.environ.get("DD_DATABASE_NAME"):
-        config.db_name = env_name
-
-    if media_path:
-        config.media_path = Path(media_path)
-    elif env_media := os.environ.get("PORTAL_UPLOAD_DIR"):
-        config.media_path = Path(env_media)
-
-    if env_audit := os.environ.get("PORTAL_AUDIT_DIR"):
-        config.audit_log_path = Path(env_audit)
-
-    return BackupJob(config)