fpu-barometer-admin 0.3.0__py3-none-any.whl

fpu_barometer_admin/cli/deploy.py

@@ -0,0 +1,719 @@
+"""Azure deployment helpers for the FPU admin CLI."""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import subprocess
+import sys
+import time
+import uuid
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Sequence
+
+import requests
+
+from fpu_barometer_admin.connectors.static_sources import STATIC_SOURCE_FILES
+
+
+@dataclass(frozen=True)
+class AzureDeployConfig:
+    resource_group: str = "fpu-research-rg"
+    location: str = "westeurope"
+    environment: str = "prod"
+    storage_container: str = "datasets"
+    project_root: Path | None = None
+    python_version: str = "3.10"
+
+
+@dataclass(frozen=True)
+class AzureDeploymentResult:
+    resource_group: str
+    location: str
+    environment: str
+    storage_account: str
+    storage_container: str
+    function_app: str
+    function_app_url: str
+    config_path: Path
+
+
+@dataclass(frozen=True)
+class AzureCodeRedeployConfig:
+    resource_group: str | None = None
+    function_app: str | None = None
+    project_root: Path | None = None
+    scheduler_cron: str = "0 15 2 * * *"
+
+
+@dataclass(frozen=True)
+class AzureCodeRedeployResult:
+    resource_group: str
+    function_app: str
+    function_app_url: str
+    scheduler_cron: str
+    config_path: Path
+
+
+def generate_unique_name(
+    base_name: str, max_length: int = 60, *, allow_dash: bool = True
+) -> str:
+    """Generate an Azure-safe unique resource name."""
+
+    suffix = uuid.uuid4().hex[:8]
+    normalized = base_name.lower()
+    if not allow_dash:
+        normalized = "".join(
+            character for character in normalized if character.isalnum()
+        )
+        separator = ""
+    else:
+        normalized = "".join(
+            character
+            for character in normalized
+            if character.isalnum() or character == "-"
+        ).strip("-")
+        separator = "-"
+    available_length = max_length - len(separator) - len(suffix)
+    return f"{normalized[:available_length].rstrip('-')}{separator}{suffix}"
+
+
+def check_prerequisites() -> dict[str, bool]:
+    """Check local tools needed to deploy to Azure."""
+
+    return {
+        "azure_cli": _command_available(["az", "--version"]),
+        "func_tools": _command_available(["func", "--version"]),
+        "python_build": _command_available(["python", "-m", "build", "--version"]),
+    }
+
+
+def check_azure_login() -> bool:
+    """Return whether Azure CLI has an active login."""
+
+    try:
+        _run_command(["az", "account", "show"], "checking Azure CLI login", timeout=30)
+    except RuntimeError:
+        return False
+    return True
+
+
+def wait_for_function_app(
+    function_app_name: str, resource_group: str, max_wait_seconds: int = 60
+) -> bool:
+    """Wait until Azure reports the Function App as running and reachable."""
+
+    print("Waiting for Function App to become ready...")
+    started = time.time()
+    while time.time() - started < max_wait_seconds:
+        try:
+            result = _run_command(
+                [
+                    "az",
+                    "functionapp",
+                    "show",
+                    "--name",
+                    function_app_name,
+                    "--resource-group",
+                    resource_group,
+                    "--query",
+                    "state",
+                    "--output",
+                    "tsv",
+                ],
+                "checking function app state",
+                timeout=30,
+            )
+            if result.stdout.strip() == "Running":
+                response = requests.head(
+                    f"https://{function_app_name}.azurewebsites.net", timeout=5
+                )
+                if response.status_code in {200, 401, 403, 404}:
+                    return True
+        except Exception:
+            pass
+        time.sleep(1)
+    return False
+
+
+def stage_function_app(wheel_path: Path | Sequence[Path], staging_dir: Path) -> Path:
+    """Create a deployable Azure Functions root around built package wheels."""
+
+    wheel_paths = [wheel_path] if isinstance(wheel_path, Path) else list(wheel_path)
+    if not wheel_paths:
+        raise RuntimeError("No wheels supplied for Function App staging")
+    if staging_dir.exists():
+        shutil.rmtree(staging_dir)
+    staging_dir.mkdir(parents=True)
+
+    for wheel in wheel_paths:
+        shutil.copy2(wheel, staging_dir / wheel.name)
+    (staging_dir / "function_app.py").write_text(
+        "from fpu_barometer_admin.handlers.function_app import app\n", encoding="utf-8"
+    )
+    (staging_dir / "host.json").write_text(
+        json.dumps(
+            {
+                "version": "2.0",
+                "extensionBundle": {
+                    "id": "Microsoft.Azure.Functions.ExtensionBundle",
+                    "version": "[4.*, 5.0.0)",
+                },
+            },
+            indent=2,
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+    requirement_lines = [f"./{wheel.name}" for wheel in wheel_paths]
+    requirement_lines[-1] = f"{requirement_lines[-1]}[admin]"
+    (staging_dir / "requirements.txt").write_text(
+        "\n".join(requirement_lines) + "\n", encoding="utf-8"
+    )
+    return staging_dir
+
+
+def deploy_azure(config: AzureDeployConfig) -> AzureDeploymentResult:
+    """Deploy Barometer's Azure Functions API and Blob storage."""
+
+    _assert_prerequisites()
+    if not check_azure_login():
+        raise RuntimeError("Not logged into Azure CLI. Run 'az login' first.")
+
+    project_root = _resolve_project_root(config.project_root)
+    storage_account = generate_unique_name("barometerdata", 24, allow_dash=False)
+    function_app = generate_unique_name(f"barometer-api-{config.environment}", 60)
+    function_app_url = f"https://{function_app}.azurewebsites.net"
+
+    print("Starting Barometer Azure deployment")
+    print(f"Resource group: {config.resource_group}")
+    print(f"Location: {config.location}")
+    print(f"Storage account: {storage_account}")
+    print(f"Function app: {function_app}")
+
+    _run_command(
+        [
+            "az",
+            "group",
+            "create",
+            "--name",
+            config.resource_group,
+            "--location",
+            config.location,
+            "--output",
+            "table",
+        ],
+        "creating resource group",
+    )
+    _run_command(
+        [
+            "az",
+            "storage",
+            "account",
+            "create",
+            "--name",
+            storage_account,
+            "--resource-group",
+            config.resource_group,
+            "--location",
+            config.location,
+            "--sku",
+            "Standard_LRS",
+            "--kind",
+            "StorageV2",
+            "--access-tier",
+            "Hot",
+            "--allow-blob-public-access",
+            "false",
+            "--enable-hierarchical-namespace",
+            "true",
+            "--output",
+            "table",
+        ],
+        "creating storage account",
+    )
+    connection_string = _run_command(
+        [
+            "az",
+            "storage",
+            "account",
+            "show-connection-string",
+            "--resource-group",
+            config.resource_group,
+            "--name",
+            storage_account,
+            "--query",
+            "connectionString",
+            "--output",
+            "tsv",
+        ],
+        "getting storage connection string",
+    ).stdout.strip()
+    _run_command(
+        [
+            "az",
+            "storage",
+            "container",
+            "create",
+            "--name",
+            config.storage_container,
+            "--connection-string",
+            connection_string,
+            "--public-access",
+            "off",
+            "--output",
+            "none",
+        ],
+        f"creating storage container {config.storage_container}",
+    )
+    upload_static_source_files(
+        project_root,
+        connection_string=connection_string,
+        container_name=config.storage_container,
+    )
+    _run_command(
+        [
+            "az",
+            "functionapp",
+            "create",
+            "--name",
+            function_app,
+            "--resource-group",
+            config.resource_group,
+            "--consumption-plan-location",
+            config.location,
+            "--storage-account",
+            storage_account,
+            "--runtime",
+            "python",
+            "--runtime-version",
+            config.python_version,
+            "--os-type",
+            "Linux",
+            "--functions-version",
+            "4",
+        ],
+        "creating function app",
+    )
+    _run_command(
+        [
+            "az",
+            "functionapp",
+            "config",
+            "appsettings",
+            "set",
+            "--name",
+            function_app,
+            "--resource-group",
+            config.resource_group,
+            "--settings",
+            f"FPU_STORAGE_BACKEND=azure",
+            f"FPU_AZURE_STORAGE_CONNECTION_STRING={connection_string}",
+            f"FPU_AZURE_STORAGE_CONTAINER={config.storage_container}",
+            "FPU_SCHEDULER_CRON=0 15 2 * * *",
+            "FUNCTIONS_WORKER_RUNTIME=python",
+            "FUNCTIONS_EXTENSION_VERSION=~4",
+            "AzureWebJobsFeatureFlags=EnableWorkerIndexing",
+            "FUNCTIONS_WORKER_PROCESS_COUNT=1",
+            "WEBSITE_MAX_DYNAMIC_APPLICATION_SCALE_OUT=1",
+            "--output",
+            "table",
+        ],
+        "configuring function app",
+    )
+    if not wait_for_function_app(function_app, config.resource_group):
+        print("Function App readiness check timed out; continuing with deployment.")
+
+    deploy_workspace = project_root / ".scratch" / "deploy"
+    try:
+        wheel_paths = build_wheels(project_root)
+        staging_dir = stage_function_app(
+            wheel_paths, deploy_workspace / "azure-functions"
+        )
+        _run_command(
+            [
+                "func",
+                "azure",
+                "functionapp",
+                "publish",
+                function_app,
+                "--python",
+                "--build",
+                "remote",
+            ],
+            "publishing function app",
+            timeout=900,
+            cwd=staging_dir,
+        )
+    except Exception:
+        shutil.rmtree(deploy_workspace, ignore_errors=True)
+        raise
+
+    result = AzureDeploymentResult(
+        resource_group=config.resource_group,
+        location=config.location,
+        environment=config.environment,
+        storage_account=storage_account,
+        storage_container=config.storage_container,
+        function_app=function_app,
+        function_app_url=function_app_url,
+        config_path=_write_deployment_config(
+            project_root,
+            config=config,
+            storage_account=storage_account,
+            function_app=function_app,
+            function_app_url=function_app_url,
+        ),
+    )
+    print("Deployment complete")
+    print(f"API: {result.function_app_url}")
+    return result
+
+
+def redeploy_azure_code(config: AzureCodeRedeployConfig) -> AzureCodeRedeployResult:
+    """Redeploy code to an existing Function App without touching storage resources."""
+
+    _assert_prerequisites()
+    if not check_azure_login():
+        raise RuntimeError("Not logged into Azure CLI. Run 'az login' first.")
+
+    project_root = _resolve_project_root(config.project_root)
+    deployment_config = {}
+    if not (config.resource_group and config.function_app):
+        deployment_config = _read_deployment_config(project_root)
+    resource_group = config.resource_group or deployment_config.get("resourceGroup")
+    function_app = config.function_app or deployment_config.get("functionApp")
+    if not resource_group or not function_app:
+        raise RuntimeError(
+            "Redeploy needs a resource group and Function App. Pass --resource-group "
+            "and --function-app, or keep .scratch/deploy/azure-deployment.json."
+        )
+    function_app_url = (
+        f"https://{function_app}.azurewebsites.net"
+        if config.function_app
+        else deployment_config.get(
+            "functionAppUrl", f"https://{function_app}.azurewebsites.net"
+        )
+    )
+
+    print("Starting Barometer Azure code redeploy")
+    print(f"Resource group: {resource_group}")
+    print(f"Function app: {function_app}")
+    print("Storage resources: preserved")
+
+    _run_command(
+        [
+            "az",
+            "functionapp",
+            "config",
+            "appsettings",
+            "set",
+            "--name",
+            function_app,
+            "--resource-group",
+            resource_group,
+            "--settings",
+            f"FPU_SCHEDULER_CRON={config.scheduler_cron}",
+            "--output",
+            "table",
+        ],
+        "configuring scheduler app setting",
+    )
+
+    redeploy_workspace = project_root / ".scratch" / "redeploy"
+    try:
+        wheel_paths = build_wheels(project_root)
+        staging_dir = stage_function_app(
+            wheel_paths, redeploy_workspace / "azure-functions"
+        )
+        _run_command(
+            [
+                "func",
+                "azure",
+                "functionapp",
+                "publish",
+                function_app,
+                "--python",
+                "--build",
+                "remote",
+            ],
+            "publishing function app",
+            timeout=900,
+            cwd=staging_dir,
+        )
+    except Exception:
+        shutil.rmtree(redeploy_workspace, ignore_errors=True)
+        raise
+
+    result = AzureCodeRedeployResult(
+        resource_group=resource_group,
+        function_app=function_app,
+        function_app_url=function_app_url,
+        scheduler_cron=config.scheduler_cron,
+        config_path=project_root / ".scratch" / "deploy" / "azure-deployment.json",
+    )
+    print("Code redeploy complete")
+    print(f"API: {result.function_app_url}")
+    return result
+
+
+def upload_static_source_files(
+    project_root: Path,
+    *,
+    connection_string: str,
+    container_name: str,
+) -> None:
+    """Upload deploy-time static Connector source files to logical storage."""
+
+    for source_file in STATIC_SOURCE_FILES:
+        source_path = project_root / source_file.local_path
+        if not source_path.exists():
+            raise RuntimeError(f"Static source file not found: {source_path}")
+        _run_command(
+            [
+                "az",
+                "storage",
+                "blob",
+                "upload",
+                "--container-name",
+                container_name,
+                "--name",
+                source_file.logical_path,
+                "--file",
+                str(source_path),
+                "--connection-string",
+                connection_string,
+                "--overwrite",
+                "true",
+                "--output",
+                "none",
+            ],
+            f"uploading static source {source_file.dataset}",
+        )
+
+
+def build_wheel(project_root: Path) -> Path:
+    """Build and return the primary package wheel path."""
+
+    return build_wheels(project_root)[-1]
+
+
+def build_wheels(project_root: Path) -> list[Path]:
+    """Build deployment package wheels into the workspace dist directory."""
+
+    dist = project_root / "dist"
+    if dist.exists():
+        shutil.rmtree(dist)
+    dist.mkdir(parents=True, exist_ok=True)
+    for package_root in _deployment_package_roots(project_root):
+        _run_command(
+            [
+                "python",
+                "-m",
+                "build",
+                "--wheel",
+                "--outdir",
+                str(dist),
+                str(package_root),
+            ],
+            f"building package wheel for {package_root.name}",
+            timeout=180,
+            cwd=project_root,
+        )
+    wheels = sorted(dist.glob("*.whl"))
+    if not wheels:
+        raise RuntimeError("Build completed but no wheel was produced")
+    return wheels
+
+
+def _deployment_package_roots(project_root: Path) -> list[Path]:
+    workspace_packages = [project_root / "packages" / "fpu-barometer", project_root / "packages" / "fpu-barometer-admin"]
+    if all((package_root / "pyproject.toml").exists() for package_root in workspace_packages):
+        return workspace_packages
+    return [project_root]
+
+
+def _read_deployment_config(project_root: Path) -> dict[str, Any]:
+    config_path = project_root / ".scratch" / "deploy" / "azure-deployment.json"
+    if not config_path.exists():
+        return {}
+    return json.loads(config_path.read_text(encoding="utf-8"))
+
+
+def _write_deployment_config(
+    project_root: Path,
+    *,
+    config: AzureDeployConfig,
+    storage_account: str,
+    function_app: str,
+    function_app_url: str,
+) -> Path:
+    config_dir = project_root / ".scratch" / "deploy"
+    config_dir.mkdir(parents=True, exist_ok=True)
+    config_path = config_dir / "azure-deployment.json"
+    config_path.write_text(
+        json.dumps(
+            {
+                "resourceGroup": config.resource_group,
+                "location": config.location,
+                "environment": config.environment,
+                "storageAccount": storage_account,
+                "storageContainer": config.storage_container,
+                "functionApp": function_app,
+                "functionAppUrl": function_app_url,
+                "deployedAt": time.strftime("%Y-%m-%d %H:%M:%S"),
+            },
+            indent=2,
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+    return config_path
+
+
+def _assert_prerequisites() -> None:
+    prereqs = check_prerequisites()
+    missing = [name for name, available in prereqs.items() if not available]
+    if missing:
+        install_hints = {
+            "azure_cli": "Install Azure CLI and run 'az login'.",
+            "func_tools": "Install Azure Functions Core Tools v4.",
+            "python_build": "Install Python build support with 'python -m pip install build'.",
+        }
+        hints = " ".join(install_hints[name] for name in missing)
+        raise RuntimeError(
+            f"Missing deployment prerequisites: {', '.join(missing)}. {hints}"
+        )
+
+
+def _command_available(args: Sequence[str]) -> bool:
+    try:
+        result = subprocess.run(
+            _resolve_command_args(args),
+            capture_output=True,
+            text=True,
+            timeout=30,
+            shell=False,
+        )
+    except (FileNotFoundError, subprocess.TimeoutExpired):
+        return False
+    return result.returncode == 0
+
+
+def _run_command(
+    args: Sequence[str],
+    description: str,
+    *,
+    timeout: int = 300,
+    cwd: Path | None = None,
+) -> subprocess.CompletedProcess[str]:
+    resolved_args = _resolve_command_args(args)
+    result = subprocess.run(
+        resolved_args,
+        capture_output=True,
+        text=True,
+        timeout=timeout,
+        shell=False,
+        cwd=cwd,
+    )
+    if result.returncode != 0:
+        raise RuntimeError(
+            f"Failed while {description}.\n"
+            f"Command: {' '.join(_redact_args(resolved_args))}\n"
+            f"stdout:\n{_redact_text(result.stdout)}\n\n"
+            f"stderr:\n{_redact_text(result.stderr)}"
+        )
+    return result
+
+
+def _resolve_command_args(args: Sequence[str]) -> list[str]:
+    resolved = list(args)
+    if resolved:
+        resolved[0] = _resolve_executable(str(resolved[0]))
+    return resolved
+
+
+def _resolve_executable(executable: str) -> str:
+    if executable == "python":
+        return sys.executable
+    if os.path.dirname(executable):
+        return executable
+    path = shutil.which(executable)
+    if path:
+        return path
+    if os.name == "nt" and not Path(executable).suffix:
+        for suffix in (".cmd", ".exe", ".bat"):
+            path = shutil.which(f"{executable}{suffix}")
+            if path:
+                return path
+    return executable
+
+
+def _redact_text(value: str) -> str:
+    redacted = value
+    markers = ("AccountKey=", "SharedAccessSignature=", "sig=")
+    for marker in markers:
+        start = redacted.find(marker)
+        while start != -1:
+            value_start = start + len(marker)
+            value_end = len(redacted)
+            for separator in (";", " ", "\n", "&"):
+                separator_index = redacted.find(separator, value_start)
+                if separator_index != -1:
+                    value_end = min(value_end, separator_index)
+            redacted = redacted[:value_start] + "<redacted>" + redacted[value_end:]
+            start = redacted.find(marker, value_start + len("<redacted>"))
+    return redacted
+
+
+def _redact_args(args: Sequence[str]) -> list[str]:
+    redacted: list[str] = []
+    secret_markers = (
+        "CONNECTION_STRING=",
+        "AccountKey=",
+        "SharedAccessSignature=",
+        "sig=",
+    )
+    for arg in args:
+        value = str(arg)
+        if any(marker in value for marker in secret_markers):
+            if "=" in value:
+                key = value.split("=", 1)[0]
+                redacted.append(f"{key}=<redacted>")
+            else:
+                redacted.append("<redacted>")
+        else:
+            redacted.append(value)
+    return redacted
+
+
+def _resolve_project_root(project_root: Path | None) -> Path:
+    if project_root is not None:
+        root = Path(project_root)
+        if not (root / "pyproject.toml").exists():
+            raise RuntimeError(f"Project root does not contain pyproject.toml: {root}")
+        return root
+    for candidate in [Path.cwd(), *Path.cwd().parents]:
+        if (candidate / "pyproject.toml").exists():
+            return candidate
+    raise RuntimeError("Could not find project root containing pyproject.toml")
+
+
+__all__ = [
+    "AzureCodeRedeployConfig",
+    "AzureCodeRedeployResult",
+    "AzureDeployConfig",
+    "AzureDeploymentResult",
+    "build_wheel",
+    "build_wheels",
+    "check_azure_login",
+    "check_prerequisites",
+    "deploy_azure",
+    "redeploy_azure_code",
+    "generate_unique_name",
+    "stage_function_app",
+    "upload_static_source_files",
+    "wait_for_function_app",
+]