mini-runbot 0.2.0__py3-none-any.whl

mini_runbot/adapters/runtime/compose.py

@@ -0,0 +1,323 @@
+from __future__ import annotations
+
+import json
+import subprocess
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+from jinja2 import (
+    Environment,
+    FileSystemLoader,
+    PackageLoader,
+    StrictUndefined,
+    select_autoescape,
+)
+
+from mini_runbot.config import Settings
+from mini_runbot.domain.errors import RuntimeOperationError
+from mini_runbot.domain.models import Build
+from mini_runbot.ports.services import CommandResult, RuntimeInspection
+
+
+class DockerComposeRuntimeService:
+    def __init__(self, settings: Settings, template_root: Path | None = None) -> None:
+        self.settings = settings
+        loader = (
+            FileSystemLoader(template_root)
+            if template_root is not None
+            else PackageLoader("mini_runbot", "templates")
+        )
+        self.environment = Environment(
+            loader=loader,
+            undefined=StrictUndefined,
+            autoescape=select_autoescape(default=False),
+            keep_trailing_newline=True,
+        )
+
+    def prepare(self, build_id: str, workspace_path: Path) -> None:
+        self._validated_workspace(build_id, workspace_path)
+        (workspace_path / "logs").mkdir(parents=True, exist_ok=True)
+        (workspace_path / "runtime").mkdir(parents=True, exist_ok=True)
+
+    def render(self, build: Build) -> CommandResult:
+        started = time.monotonic()
+        if build.host_port is None:
+            raise RuntimeOperationError("Build has no allocated host port")
+        if any(not repository.checkout_path for repository in build.repositories):
+            raise RuntimeOperationError("All repositories must be checked out before rendering")
+        repositories = sorted(
+            build.repositories, key=lambda item: (item.addons_priority, item.name)
+        )
+        addons_path = ",".join(
+            ["/usr/lib/python3/dist-packages/odoo/addons"]
+            + [f"/mnt/addons/{item.name}" for item in repositories]
+        )
+        rendered = self.environment.get_template("compose.yaml.j2").render(
+            compose_project_name=build.compose_project_name,
+            database_name=build.database_name,
+            host_port=build.host_port,
+            odoo_image=self.settings.odoo_image,
+            postgres_image=self.settings.postgres_image,
+            repositories=repositories,
+            addons_path=addons_path,
+        )
+        path = self._compose_path(build)
+        path.write_text(rendered, encoding="utf-8")
+        log_path = build.workspace_path / "logs" / "render.log"
+        log_path.write_text(f"Rendered {path.name}\n", encoding="utf-8")
+        return self._result(started, log_path, "Compose configuration rendered")
+
+    def validate(self, build: Build) -> CommandResult:
+        return self._run(build, "compose_validate", ["config", "--quiet"])
+
+    def start_database(self, build: Build) -> CommandResult:
+        return self._run(build, "database", ["up", "-d", "--wait", "db"])
+
+    def install_modules(self, build: Build) -> CommandResult:
+        arguments = [
+            "run",
+            "--rm",
+            "odoo",
+            "odoo",
+            "-d",
+            build.database_name,
+            "-i",
+            ",".join(build.modules),
+        ]
+        if not self.settings.load_demo_data:
+            arguments.append("--without-demo=all")
+        arguments.extend(
+            [
+                "--stop-after-init",
+                f"--addons-path={self._addons_path(build)}",
+            ]
+        )
+        return self._run(
+            build,
+            "install",
+            arguments,
+        )
+
+    def test_modules(self, build: Build) -> CommandResult:
+        tags = ",".join(f"/{module}" for module in build.modules)
+        return self._run(
+            build,
+            "test",
+            [
+                "run",
+                "--rm",
+                "odoo",
+                "odoo",
+                "-d",
+                build.database_name,
+                "--test-enable",
+                f"--test-tags={tags}",
+                "--stop-after-init",
+                f"--addons-path={self._addons_path(build)}",
+            ],
+        )
+
+    def start_server(self, build: Build) -> CommandResult:
+        return self._run(build, "start", ["up", "-d", "odoo"])
+
+    def wait_healthy(self, build: Build) -> CommandResult:
+        if not build.preview_url:
+            raise RuntimeOperationError("Build has no preview URL")
+        started = time.monotonic()
+        deadline = started + self.settings.health_timeout_seconds
+        last_error = "no response"
+        while time.monotonic() < deadline:
+            try:
+                with urllib.request.urlopen(build.preview_url, timeout=5) as response:
+                    if response.status < 500:
+                        log_path = build.workspace_path / "logs" / "healthcheck.log"
+                        log_path.write_text(
+                            f"HTTP {response.status} from {build.preview_url}\n", encoding="utf-8"
+                        )
+                        return self._result(started, log_path, "Odoo HTTP healthcheck passed")
+            except (OSError, urllib.error.URLError) as exc:
+                last_error = str(exc)
+            time.sleep(1)
+        raise RuntimeOperationError(f"Odoo healthcheck timed out: {last_error}")
+
+    def inspect(self, build: Build) -> RuntimeInspection:
+        compose_path = self._compose_path(build)
+        if not compose_path.is_file():
+            return RuntimeInspection(False, False, (), "Compose configuration is absent")
+        command = [
+            "docker",
+            "compose",
+            "--project-name",
+            build.compose_project_name,
+            "--file",
+            str(compose_path),
+            "ps",
+            "--all",
+            "--format",
+            "json",
+        ]
+        try:
+            result = subprocess.run(
+                command,
+                cwd=build.workspace_path,
+                capture_output=True,
+                text=True,
+                timeout=self.settings.command_timeout_seconds,
+                check=False,
+            )
+        except (OSError, subprocess.TimeoutExpired) as exc:
+            raise RuntimeOperationError(
+                f"Docker runtime inspection could not complete: {exc}"
+            ) from exc
+        log_path = build.workspace_path / "logs" / "runtime_inspect.log"
+        log_path.write_text(f"{result.stdout}{result.stderr}"[-1_000_000:], encoding="utf-8")
+        if result.returncode != 0:
+            raise RuntimeOperationError(f"Docker runtime inspection failed; see {log_path}")
+        entries = self._parse_compose_ps(result.stdout)
+        services = tuple(
+            sorted(
+                str(item.get("Service"))
+                for item in entries
+                if item.get("Service")
+            )
+        )
+        running_services = {
+            str(item.get("Service"))
+            for item in entries
+            if str(item.get("State", "")).lower() == "running"
+        }
+        db_entries = [item for item in entries if item.get("Service") == "db"]
+        database_healthy = all(
+            str(item.get("Health", "")).lower() in {"", "healthy"}
+            for item in db_entries
+        )
+        expected = {"db", "odoo"}
+        running = expected.issubset(running_services) and database_healthy
+        summary = (
+            "Docker runtime is running"
+            if running
+            else f"Docker runtime incomplete; services={','.join(services) or 'none'}"
+        )
+        return RuntimeInspection(bool(entries), running, services, summary)
+
+    def destroy(self, build_id: str, workspace_path: Path) -> None:
+        workspace = self._validated_workspace(build_id, workspace_path)
+        compose_path = workspace / "runtime" / "compose.yaml"
+        if compose_path.exists():
+            result = subprocess.run(
+                [
+                    "docker",
+                    "compose",
+                    "--project-name",
+                    build_id.replace("-", "_"),
+                    "--file",
+                    str(compose_path),
+                    "down",
+                    "--volumes",
+                    "--remove-orphans",
+                ],
+                cwd=workspace,
+                capture_output=True,
+                text=True,
+                timeout=self.settings.command_timeout_seconds,
+                check=False,
+            )
+            log_path = workspace / "logs" / "destroy.log"
+            log_path.write_text(f"{result.stdout}{result.stderr}"[-1_000_000:], encoding="utf-8")
+            if result.returncode != 0:
+                raise RuntimeOperationError(f"Compose destruction failed; see {log_path}")
+
+    def _run(self, build: Build, stage: str, arguments: list[str]) -> CommandResult:
+        compose_path = self._compose_path(build)
+        command = [
+            "docker",
+            "compose",
+            "--project-name",
+            build.compose_project_name,
+            "--file",
+            str(compose_path),
+            *arguments,
+        ]
+        started = time.monotonic()
+        try:
+            result = subprocess.run(
+                command,
+                cwd=build.workspace_path,
+                capture_output=True,
+                text=True,
+                timeout=self.settings.command_timeout_seconds,
+                check=False,
+            )
+        except (OSError, subprocess.TimeoutExpired) as exc:
+            raise RuntimeOperationError(f"{stage} command could not complete: {exc}") from exc
+        log_path = build.workspace_path / "logs" / f"{stage}.log"
+        log_path.write_text(f"{result.stdout}{result.stderr}"[-1_000_000:], encoding="utf-8")
+        if result.returncode != 0:
+            raise RuntimeOperationError(
+                f"{stage} failed with exit code {result.returncode}",
+                exit_code=result.returncode,
+                log_path=str(log_path),
+            )
+        return CommandResult(
+            exit_code=result.returncode,
+            duration_seconds=time.monotonic() - started,
+            log_path=str(log_path),
+            summary=f"{stage} completed",
+        )
+
+    def _compose_path(self, build: Build) -> Path:
+        self._validated_workspace(build.id, build.workspace_path)
+        return build.workspace_path / "runtime" / "compose.yaml"
+
+    @staticmethod
+    def _addons_path(build: Build) -> str:
+        repositories = sorted(
+            build.repositories, key=lambda item: (item.addons_priority, item.name)
+        )
+        return ",".join(
+            ["/usr/lib/python3/dist-packages/odoo/addons"]
+            + [f"/mnt/addons/{item.name}" for item in repositories]
+        )
+
+    def _validated_workspace(self, build_id: str, workspace_path: Path) -> Path:
+        root = self.settings.builds_root.resolve()
+        actual = workspace_path.resolve()
+        if actual != (root / build_id).resolve() or actual.parent != root:
+            raise RuntimeOperationError(f"Workspace is outside configured root: {actual}")
+        return actual
+
+    @staticmethod
+    def _result(started: float, log_path: Path, summary: str) -> CommandResult:
+        return CommandResult(
+            exit_code=0,
+            duration_seconds=time.monotonic() - started,
+            log_path=str(log_path),
+            summary=summary,
+        )
+
+    @staticmethod
+    def _parse_compose_ps(output: str) -> list[dict[str, object]]:
+        content = output.strip()
+        if not content:
+            return []
+        try:
+            parsed = json.loads(content)
+            if isinstance(parsed, list):
+                return [item for item in parsed if isinstance(item, dict)]
+            if isinstance(parsed, dict):
+                return [parsed]
+        except json.JSONDecodeError:
+            entries: list[dict[str, object]] = []
+            for line in content.splitlines():
+                try:
+                    item = json.loads(line)
+                except json.JSONDecodeError as exc:
+                    raise RuntimeOperationError(
+                        "Docker runtime inspection returned invalid JSON"
+                    ) from exc
+                if isinstance(item, dict):
+                    entries.append(item)
+            return entries
+        raise RuntimeOperationError("Docker runtime inspection returned invalid JSON")

mini_runbot/adapters/runtime/local.py

@@ -0,0 +1,30 @@
+import shutil
+from pathlib import Path
+
+from mini_runbot.domain.errors import UnsafePathError
+
+
+class LocalRuntimeService:
+    """Controlled Phase 0 runtime that owns only a per-build runtime directory."""
+
+    def __init__(self, builds_root: Path) -> None:
+        self.builds_root = builds_root.resolve()
+
+    def _validate_workspace(self, build_id: str, workspace_path: Path) -> Path:
+        expected = (self.builds_root / build_id).resolve()
+        actual = workspace_path.resolve()
+        if actual != expected or actual.parent != self.builds_root:
+            raise UnsafePathError(f"Workspace is outside the configured build root: {actual}")
+        return actual
+
+    def prepare(self, build_id: str, workspace_path: Path) -> None:
+        workspace = self._validate_workspace(build_id, workspace_path)
+        (workspace / "logs").mkdir(parents=True, exist_ok=True)
+        (workspace / "runtime").mkdir(parents=True, exist_ok=True)
+
+    def destroy(self, build_id: str, workspace_path: Path) -> None:
+        workspace = self._validate_workspace(build_id, workspace_path)
+        runtime_path = workspace / "runtime"
+        if runtime_path.exists():
+            shutil.rmtree(runtime_path)
+

mini_runbot/adapters/runtime/ports.py

@@ -0,0 +1,49 @@
+import socket
+from threading import Lock
+
+from mini_runbot.domain.errors import RuntimeOperationError
+from mini_runbot.ports.repositories import PortLeaseStore
+
+
+class SocketPortAllocator:
+    def __init__(
+        self, start: int, end: int, lease_store: PortLeaseStore | None = None
+    ) -> None:
+        if not 1024 <= start <= end <= 65535:
+            raise ValueError("Port range must be between 1024 and 65535")
+        self.start = start
+        self.end = end
+        self.lease_store = lease_store
+        self._allocated: dict[int, str] = {}
+        self._lock = Lock()
+
+    def allocate(self, build_id: str, excluded: set[int] | None = None) -> int:
+        excluded = excluded or set()
+        with self._lock:
+            for port in range(self.start, self.end + 1):
+                if port in self._allocated or port in excluded:
+                    continue
+                with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as candidate:
+                    try:
+                        candidate.bind(("127.0.0.1", port))
+                    except OSError:
+                        continue
+                    if self.lease_store and not self.lease_store.try_acquire_port(
+                        build_id, port
+                    ):
+                        continue
+                    self._allocated[port] = build_id
+                    return port
+        raise RuntimeOperationError("No preview ports are available in the configured range")
+
+    def release(self, build_id: str, port: int) -> None:
+        with self._lock:
+            if self._allocated.get(port) == build_id:
+                self._allocated.pop(port)
+            if self.lease_store:
+                self.lease_store.release_port(build_id, port)
+
+    def reconcile(self) -> list[int]:
+        if not self.lease_store:
+            return []
+        return self.lease_store.reconcile_port_leases()

mini_runbot/api/app.py

@@ -0,0 +1,147 @@
+from contextlib import asynccontextmanager
+from functools import lru_cache
+from pathlib import Path
+from typing import Annotated
+
+from fastapi import Depends, FastAPI, HTTPException, Query, status
+from fastapi.responses import FileResponse
+from fastapi.staticfiles import StaticFiles
+
+from mini_runbot.api.schemas import BuildLogsResponse, BuildResponse, PublicConfigResponse
+from mini_runbot.application.build_manager import BuildManager
+from mini_runbot.application.executor import LocalBuildExecutor
+from mini_runbot.application.scheduler import CleanupScheduler
+from mini_runbot.bootstrap import create_manager
+from mini_runbot.config import Settings
+from mini_runbot.domain.errors import BuildNotFoundError, MiniRunbotError
+from mini_runbot.domain.validation import CreateBuildRequest
+
+
+@asynccontextmanager
+async def lifespan(application: FastAPI):
+    settings = Settings.from_environment()
+    scheduler: CleanupScheduler | None = None
+    if settings.cleanup_interval_seconds > 0:
+        scheduler = CleanupScheduler(get_manager(), settings.cleanup_interval_seconds)
+        scheduler.start()
+    application.state.cleanup_scheduler = scheduler
+    try:
+        yield
+    finally:
+        if scheduler is not None:
+            scheduler.stop()
+
+
+app = FastAPI(title="Mini-Runbot", version="0.2.0", lifespan=lifespan)
+web_root = Path(__file__).resolve().parents[1] / "web"
+app.mount("/assets", StaticFiles(directory=web_root), name="assets")
+
+
+@lru_cache(maxsize=1)
+def get_manager() -> BuildManager:
+    return create_manager(docker=True)
+
+
+ManagerDependency = Annotated[BuildManager, Depends(get_manager)]
+
+
+@lru_cache(maxsize=1)
+def get_executor() -> LocalBuildExecutor:
+    settings = Settings.from_environment()
+    return LocalBuildExecutor(get_manager(), max_workers=settings.max_concurrent_builds)
+
+
+ExecutorDependency = Annotated[LocalBuildExecutor, Depends(get_executor)]
+
+
+@app.get("/", include_in_schema=False)
+def dashboard() -> FileResponse:
+    return FileResponse(web_root / "index.html")
+
+
+@app.get("/app-config", response_model=PublicConfigResponse)
+def public_config() -> PublicConfigResponse:
+    settings = Settings.from_environment()
+    return PublicConfigResponse(
+        repositories=[
+            {
+                "alias": alias,
+                "default_ref": repository.default_ref,
+                "allow_request_ref": repository.allow_request_ref,
+                "addons_priority": repository.addons_priority,
+            }
+            for alias, repository in sorted(settings.repositories.items())
+        ],
+        max_concurrent_builds=settings.max_concurrent_builds,
+    )
+
+
+@app.post("/builds", response_model=BuildResponse, status_code=status.HTTP_202_ACCEPTED)
+def create_build(
+    request: CreateBuildRequest,
+    manager: ManagerDependency,
+    executor: ExecutorDependency,
+) -> BuildResponse:
+    if isinstance(executor, LocalBuildExecutor) and not manager.execution_ready:
+        raise HTTPException(
+            status_code=503,
+            detail="No repositories are configured for build execution",
+        )
+    try:
+        build = manager.create(request)
+    except MiniRunbotError as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+    executor.submit(build.id)
+    return BuildResponse.from_domain(build)
+
+
+@app.get("/builds", response_model=list[BuildResponse])
+def list_builds(manager: ManagerDependency) -> list[BuildResponse]:
+    return [BuildResponse.from_domain(build) for build in manager.list()]
+
+
+@app.get("/builds/{build_id}", response_model=BuildResponse)
+def get_build(build_id: str, manager: ManagerDependency) -> BuildResponse:
+    try:
+        return BuildResponse.from_domain(manager.get(build_id))
+    except BuildNotFoundError as exc:
+        raise HTTPException(status_code=404, detail=str(exc)) from exc
+
+
+@app.delete("/builds/{build_id}", response_model=BuildResponse)
+def destroy_build(
+    build_id: str,
+    manager: ManagerDependency,
+    executor: ExecutorDependency,
+) -> BuildResponse:
+    try:
+        manager.get(build_id)
+        if executor.is_active(build_id):
+            raise HTTPException(
+                status_code=status.HTTP_409_CONFLICT,
+                detail=(
+                    "Build execution is still active; wait for it to finish before destroying it"
+                ),
+            )
+        return BuildResponse.from_domain(manager.destroy(build_id))
+    except BuildNotFoundError as exc:
+        raise HTTPException(status_code=404, detail=str(exc)) from exc
+
+
+@app.get("/builds/{build_id}/logs", response_model=BuildLogsResponse)
+def get_build_logs(
+    build_id: str,
+    manager: ManagerDependency,
+    stage_name: Annotated[str | None, Query(alias="stage")] = None,
+) -> BuildLogsResponse:
+    limit = 100_000
+    try:
+        content = manager.read_logs(build_id, stage_name, max_bytes=limit)
+    except BuildNotFoundError as exc:
+        raise HTTPException(status_code=404, detail=str(exc)) from exc
+    return BuildLogsResponse(
+        build_id=build_id,
+        stage=stage_name,
+        content=content,
+        truncated_to_bytes=limit,
+    )

mini_runbot/api/schemas.py

@@ -0,0 +1,73 @@
+from datetime import datetime
+
+from pydantic import BaseModel, ConfigDict
+
+from mini_runbot.domain.enums import BuildStatus, StageStatus
+from mini_runbot.domain.models import Build
+
+
+class RepositoryResponse(BaseModel):
+    model_config = ConfigDict(from_attributes=True)
+
+    name: str
+    requested_ref: str
+    commit_sha: str | None
+    checkout_path: str | None
+    addons_priority: int
+
+
+class StageResponse(BaseModel):
+    model_config = ConfigDict(from_attributes=True)
+
+    name: str
+    status: StageStatus
+    started_at: datetime | None
+    finished_at: datetime | None
+    duration_seconds: float | None
+    exit_code: int | None
+    log_path: str | None
+    summary: str | None
+
+
+class BuildResponse(BaseModel):
+    model_config = ConfigDict(from_attributes=True)
+
+    id: str
+    status: BuildStatus
+    requested_ref: str
+    repositories: list[RepositoryResponse]
+    modules: list[str]
+    created_at: datetime
+    started_at: datetime | None
+    finished_at: datetime | None
+    expires_at: datetime
+    host_port: int | None
+    preview_url: str | None
+    failure_stage: str | None
+    failure_message: str | None
+    stages: list[StageResponse]
+    version: int
+
+    @classmethod
+    def from_domain(cls, build: Build) -> "BuildResponse":
+        return cls.model_validate(build)
+
+
+class BuildLogsResponse(BaseModel):
+    build_id: str
+    stage: str | None
+    content: str
+    truncated_to_bytes: int
+
+
+class RepositoryOptionResponse(BaseModel):
+    alias: str
+    default_ref: str
+    allow_request_ref: bool
+    addons_priority: int
+
+
+class PublicConfigResponse(BaseModel):
+    repositories: list[RepositoryOptionResponse]
+    default_ttl_seconds: int = 14_400
+    max_concurrent_builds: int