FastAPI-fastkit 1.3.0__py3-none-any.whl → 1.4.1__py3-none-any.whl

fastapi_fastkit/backend/inspection/core.py

@@ -0,0 +1,351 @@
+# --------------------------------------------------------------------------
+# The TemplateInspector orchestrator.
+#
+# The inspector generates a real project from the template (copy + convert +
+# metadata injection), runs every check against that generated project, and
+# cleans the temporary directory up afterwards. Individual checks live in
+# sibling modules; this class only sequences them and owns the lifecycle.
+#
+# @author bnbong
+# --------------------------------------------------------------------------
+import os
+import shutil
+import time
+from pathlib import Path
+from types import SimpleNamespace
+from typing import Any, Callable, Dict, List, Optional, Tuple
+
+from fastapi_fastkit.backend.scaffolder import ProjectScaffolder, ScaffoldOptions
+from fastapi_fastkit.core.settings import settings
+from fastapi_fastkit.utils.logging import debug_log
+
+from . import checks, consistency, freshness, lint, smoke, strategies
+from .context import InspectionContext, InspectionOptions
+from .docker import DockerCompose
+from .fsutils import force_cleanup_directory
+from .report import build_report, print_report
+
+#: Metadata used to generate the throwaway project that gets inspected.
+DUMMY_METADATA = {
+    "project_name": "test-template",
+    "author": "Template Inspector",
+    "author_email": "inspector@fastapi-fastkit.dev",
+    "description": "Test project for template inspection",
+}
+
+
+class TemplateInspector:
+    """Validate a FastAPI template by generating and exercising a project.
+
+    Uses the context manager protocol so the temporary project directory is
+    always removed, even when a check raises.
+    """
+
+    def __init__(
+        self,
+        template_path: str,
+        temp_base_dir: Optional[str] = None,
+        options: Optional[InspectionOptions] = None,
+    ):
+        template_name = Path(template_path).name
+        base_dir = temp_base_dir or os.path.dirname(os.path.dirname(__file__))
+        self.ctx = InspectionContext(
+            template_path=Path(template_path),
+            temp_dir=os.path.join(base_dir, f"temp_{template_name}"),
+            options=options or InspectionOptions(),
+        )
+        self._cleanup_needed = False
+
+    # -- state exposed for callers and tests -------------------------------
+    @property
+    def template_path(self) -> Path:
+        """The on-disk template directory being inspected."""
+        return self.ctx.template_path
+
+    @property
+    def temp_dir(self) -> str:
+        """The generated project directory used for the inspection."""
+        return self.ctx.temp_dir
+
+    @property
+    def errors(self) -> List[str]:
+        """Fatal findings accumulated so far."""
+        return self.ctx.errors
+
+    @property
+    def warnings(self) -> List[str]:
+        """Non-fatal findings accumulated so far."""
+        return self.ctx.warnings
+
+    @property
+    def template_config(self) -> Optional[Dict[str, Any]]:
+        """Parsed ``template-config.yml`` contents, when the template ships one."""
+        return self.ctx.template_config
+
+    # -- lifecycle ---------------------------------------------------------
+    def __enter__(self) -> "TemplateInspector":
+        """Create the temporary directory and generate a project from the template."""
+        try:
+            if os.path.exists(self.ctx.temp_dir):
+                debug_log(
+                    f"Cleaning up existing temp directory: {self.ctx.temp_dir}", "info"
+                )
+                try:
+                    shutil.rmtree(self.ctx.temp_dir)
+                except OSError as e:
+                    debug_log(
+                        f"Failed to cleanup existing temp directory: {e}", "warning"
+                    )
+
+            os.makedirs(self.ctx.temp_dir, exist_ok=True)
+            # Mark for cleanup immediately: a failure below must not leak the
+            # directory that was just created.
+            self._cleanup_needed = True
+            self._generate_project()
+
+            self.ctx.template_config = self._load_template_config()
+            debug_log(f"Created temporary directory at {self.ctx.temp_dir}", "debug")
+            return self
+        except Exception as e:
+            debug_log(f"Failed to setup template inspector: {e}", "error")
+            self._cleanup()
+            raise
+
+    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        """Remove the generated project directory."""
+        self._cleanup()
+
+    def _cleanup(self) -> None:
+        """Tear down Docker services and delete the temporary directory."""
+        if not (self._cleanup_needed and os.path.exists(self.ctx.temp_dir)):
+            return
+
+        try:
+            if self._cleanup_docker_services():
+                # Give the daemon a moment to release bind-mounted files.
+                time.sleep(3)
+            force_cleanup_directory(self.ctx.temp_dir)
+        except Exception as e:
+            debug_log(
+                f"Warning: Unexpected error during cleanup of "
+                f"{self.ctx.temp_dir}: {e}",
+                "warning",
+            )
+            try:
+                force_cleanup_directory(self.ctx.temp_dir)
+            except Exception:
+                pass
+        finally:
+            self._cleanup_needed = False
+
+    def _cleanup_docker_services(self) -> bool:
+        """Stop any Docker Compose stack started for this template.
+
+        Returns ``True`` when a teardown was actually attempted - templates
+        without a compose file skip the (slow) docker invocations entirely.
+        """
+        compose_files = [
+            name
+            for name in ("docker-compose.yml", "docker-compose.yaml", "compose.yml")
+            if os.path.exists(self.ctx.temp_path(name))
+        ]
+        if not compose_files:
+            return False
+
+        try:
+            DockerCompose(self.ctx.temp_dir, compose_files[0]).cleanup()
+            return True
+        except Exception as e:
+            debug_log(f"Failed to cleanup Docker services: {e}", "warning")
+            return False
+
+    def _generate_project(self) -> None:
+        """Produce the project to inspect through the real ``startdemo`` path.
+
+        Running the actual ``ProjectScaffolder`` (rather than re-implementing
+        copy + inject here) is the point: whatever generation does for a user
+        - placeholder substitution, the ``[tool.fastapi-fastkit]`` metadata
+        block, the template-only files it filters out - is exactly what the
+        checks then see. Virtualenv creation and dependency installation are
+        switched off, so no package manager is ever invoked.
+
+        A generation failure is propagated: there is nothing meaningful to
+        inspect when the template cannot even be deployed, and ``__enter__``
+        already removes the half-written directory.
+        """
+        template_path = self.ctx.template_path.resolve()
+        project_dir = Path(self.ctx.temp_dir).resolve()
+
+        scaffold_settings = SimpleNamespace(
+            FASTKIT_TEMPLATE_ROOT=str(template_path.parent),
+            USER_WORKSPACE=str(project_dir.parent),
+            PACKAGE_MANAGER_CONFIG=settings.PACKAGE_MANAGER_CONFIG,
+        )
+        options = ScaffoldOptions(
+            project_name=project_dir.name,
+            author=str(DUMMY_METADATA["author"]),
+            author_email=str(DUMMY_METADATA["author_email"]),
+            description=str(DUMMY_METADATA["description"]),
+            package_manager="uv",
+            template=template_path.name,
+            create_project_folder=True,
+            with_venv=False,
+            with_install=False,
+        )
+
+        ProjectScaffolder(scaffold_settings, options).run()
+
+    def _load_template_config(self) -> Optional[Dict[str, Any]]:
+        """Load ``template-config.yml`` from the template, if it ships one.
+
+        The file is read from the template directory rather than the generated
+        project: it is inspector-only metadata and is deliberately not copied
+        into a user's project by the transducer.
+        """
+        config_file = str(self.ctx.template_path / "template-config.yml-tpl")
+        if not os.path.exists(config_file):
+            # Legacy layout: some templates shipped the config without the
+            # ``-tpl`` marker, in which case it still lands in the copy.
+            config_file = self.ctx.temp_path("template-config.yml")
+        if not os.path.exists(config_file):
+            debug_log(
+                "No template-config.yml found, using default testing strategy", "info"
+            )
+            return None
+
+        try:
+            import yaml
+        except ImportError:  # pragma: no cover - PyYAML is an optional extra
+            debug_log(
+                "PyYAML is not installed; ignoring template-config.yml", "warning"
+            )
+            return None
+
+        try:
+            with open(config_file, "r", encoding="utf-8") as f:
+                config = yaml.safe_load(f)
+        except (yaml.YAMLError, OSError, UnicodeDecodeError) as e:
+            debug_log(f"Failed to load template configuration: {e}", "warning")
+            return None
+
+        if not isinstance(config, dict):
+            debug_log("Invalid template configuration format", "warning")
+            return None
+
+        debug_log(
+            f"Loaded template configuration: {config.get('name', 'Unknown')}", "info"
+        )
+        return config
+
+    # -- checks ------------------------------------------------------------
+    def inspect_template(self) -> bool:
+        """Run every inspection step, stopping at the first failure.
+
+        :return: True if template is valid, False otherwise
+        """
+        check_list: List[Tuple[str, Callable[[], bool]]] = [
+            ("File Structure", self._check_file_structure),
+            ("File Extensions", self._check_file_extensions),
+            ("Test Suite Presence", self._check_tests_present),
+            ("Junk Files", self._check_no_junk_files),
+            ("Dependencies", self._check_dependencies),
+            ("Configuration Consistency", self._check_configuration_consistency),
+            ("FastAPI Implementation", self._check_fastapi_implementation),
+            ("Placeholder Substitution", self._check_no_placeholder_residue),
+            # Compile Check is a pure static check: it runs before the (much
+            # heavier) Template Tests so a syntax error is reported without
+            # waiting for an environment, and so it never observes files a
+            # containerised test run left behind.
+            ("Compile Check", self._check_compileall),
+            ("Template Tests", self._test_template),
+            ("Type Check", self._check_mypy),
+            ("Smoke Test", self._check_smoke_test),
+            ("Dependency Freshness", self._check_dependency_freshness),
+        ]
+
+        for check_name, check_func in check_list:
+            debug_log(f"Running check: {check_name}", "info")
+            if not check_func():
+                debug_log(f"Check failed: {check_name}", "error")
+                return False
+            debug_log(f"Check passed: {check_name}", "info")
+
+        return True
+
+    def _check_file_structure(self) -> bool:
+        return checks.check_file_structure(self.ctx)
+
+    def _check_file_extensions(self) -> bool:
+        return checks.check_file_extensions(self.ctx)
+
+    def _check_dependencies(self) -> bool:
+        return checks.check_dependencies(self.ctx)
+
+    def _check_fastapi_implementation(self) -> bool:
+        return checks.check_fastapi_implementation(self.ctx)
+
+    def _check_tests_present(self) -> bool:
+        return checks.check_tests_present(self.ctx)
+
+    def _check_no_junk_files(self) -> bool:
+        return checks.check_no_junk_files(self.ctx)
+
+    def _check_no_placeholder_residue(self) -> bool:
+        return checks.check_no_placeholder_residue(self.ctx)
+
+    def _check_configuration_consistency(self) -> bool:
+        return consistency.check_configuration_consistency(self.ctx)
+
+    def _check_compileall(self) -> bool:
+        return lint.check_compileall(self.ctx)
+
+    def _check_mypy(self) -> bool:
+        return lint.check_mypy(self.ctx)
+
+    def _check_smoke_test(self) -> bool:
+        return smoke.check_smoke_test(self.ctx)
+
+    def _check_dependency_freshness(self) -> bool:
+        return freshness.check_dependency_freshness(self.ctx)
+
+    def _test_template(self) -> bool:
+        return strategies.run_template_tests(self.ctx)
+
+    def get_report(self) -> Dict[str, Any]:
+        """
+        Get inspection report with errors and warnings.
+
+        :return: Dictionary containing inspection results
+        """
+        return build_report(self.ctx)
+
+
+def inspect_fastapi_template(
+    template_path: str,
+    temp_base_dir: Optional[str] = None,
+    options: Optional[InspectionOptions] = None,
+) -> Dict[str, Any]:
+    """
+    Convenience function to inspect a FastAPI template.
+
+    :param template_path: Path to the template to inspect
+    :param temp_base_dir: Base directory for temporary files (defaults to backend directory)
+    :param options: Optional switches (offline mode, smoke test, mypy, ...)
+    :return: Inspection report dictionary
+    """
+    template_name = Path(template_path).name
+    debug_log(
+        f"Starting template inspection for {template_name} at {template_path}", "info"
+    )
+
+    with TemplateInspector(template_path, temp_base_dir, options) as inspector:
+        is_valid = inspector.inspect_template()
+        report = inspector.get_report()
+        print_report(inspector.ctx, is_valid)
+
+    debug_log(
+        f"Template inspection completed for {template_name}. Valid: {is_valid}, "
+        f"Errors: {len(report['errors'])}, Warnings: {len(report['warnings'])}",
+        "info",
+    )
+    return report

fastapi_fastkit/backend/inspection/docker.py

@@ -0,0 +1,320 @@
+# --------------------------------------------------------------------------
+# Docker Compose orchestration for templates that declare requires_docker.
+#
+# ``DockerCompose`` owns every ``docker``/``docker-compose`` invocation so the
+# test strategies stay free of process plumbing.
+#
+# @author bnbong
+# --------------------------------------------------------------------------
+import json
+import os
+import subprocess
+import time
+from typing import Any, Dict, List, Optional, Sequence
+
+from fastapi_fastkit.utils.logging import debug_log
+
+COMPOSE_COMMAND = "docker-compose"
+#: Fallback for hosts that only ship the ``docker compose`` CLI plugin.
+COMPOSE_PLUGIN_COMMAND = ["docker", "compose"]
+#: Resolved compose invocation, switched by :meth:`DockerCompose.is_available`.
+_compose_prefix: List[str] = [COMPOSE_COMMAND]
+#: Throwaway image used to give bind-mounted files back to the host user.
+RECLAIM_IMAGE = "alpine:3.20"
+SHORT_TIMEOUT = 10
+STATUS_TIMEOUT = 30
+CLEANUP_TIMEOUT = 60
+TEST_TIMEOUT = 300
+
+
+class DockerCompose:
+    """Thin wrapper around the docker-compose CLI for one project directory."""
+
+    def __init__(self, project_dir: str, compose_file: str = "docker-compose.yml"):
+        self.project_dir = project_dir
+        self.compose_file = compose_file
+
+    def _run(
+        self, args: Sequence[str], timeout: int
+    ) -> Optional[subprocess.CompletedProcess[str]]:
+        """Run a command in the project directory, returning ``None`` on failure."""
+        try:
+            return subprocess.run(
+                list(args),
+                cwd=self.project_dir,
+                capture_output=True,
+                text=True,
+                timeout=timeout,
+            )
+        except (subprocess.TimeoutExpired, OSError) as e:
+            debug_log(f"Command {' '.join(args)} failed: {e}", "warning")
+            return None
+
+    @staticmethod
+    def compose_command() -> List[str]:
+        """Return the compose invocation resolved by :meth:`is_available`."""
+        return list(_compose_prefix)
+
+    def _compose(
+        self, args: Sequence[str], timeout: int
+    ) -> Optional[subprocess.CompletedProcess[str]]:
+        """Run a docker-compose subcommand against the configured compose file."""
+        return self._run(
+            [*self.compose_command(), "-f", self.compose_file, *args], timeout=timeout
+        )
+
+    @staticmethod
+    def _probe(command: Sequence[str]) -> bool:
+        """Return whether ``command`` exits successfully."""
+        try:
+            result = subprocess.run(
+                list(command), capture_output=True, text=True, timeout=SHORT_TIMEOUT
+            )
+        except (subprocess.TimeoutExpired, OSError):
+            return False
+        return result.returncode == 0
+
+    @staticmethod
+    def is_available() -> bool:
+        """Check that docker responds and a compose implementation exists.
+
+        ``docker-compose`` stays the preferred command; hosts that only ship
+        the ``docker compose`` CLI plugin fall back to it.
+        """
+        global _compose_prefix
+
+        if not DockerCompose._probe(["docker", "--version"]):
+            return False
+        if DockerCompose._probe([COMPOSE_COMMAND, "--version"]):
+            _compose_prefix = [COMPOSE_COMMAND]
+            return True
+        if DockerCompose._probe([*COMPOSE_PLUGIN_COMMAND, "version"]):
+            debug_log("Falling back to the 'docker compose' CLI plugin", "info")
+            _compose_prefix = list(COMPOSE_PLUGIN_COMMAND)
+            return True
+        return False
+
+    def _services(self, timeout: int) -> List[Dict[str, Any]]:
+        """Return the parsed ``docker-compose ps --format json`` entries."""
+        result = self._compose(["ps", "--format", "json"], timeout=timeout)
+        if result is None or result.returncode != 0:
+            return []
+
+        # Compose sometimes interleaves ``time="..." level=warning`` log lines
+        # with the JSON payload, so only JSON-looking lines are considered.
+        payload = "\n".join(
+            line
+            for line in result.stdout.splitlines()
+            if line.strip().startswith(("[", "{"))
+        ).strip()
+        if not payload:
+            return []
+
+        # Compose < v2.21 prints a single JSON array, newer versions print one
+        # JSON object per line (NDJSON).
+        try:
+            document = json.loads(payload)
+        except json.JSONDecodeError:
+            return DockerCompose._parse_json_lines(payload)
+
+        if isinstance(document, dict):
+            return [document]
+        if isinstance(document, list):
+            return [entry for entry in document if isinstance(entry, dict)]
+        return []
+
+    @staticmethod
+    def _parse_json_lines(payload: str) -> List[Dict[str, Any]]:
+        """Parse NDJSON output, skipping malformed or non-object lines."""
+        services: List[Dict[str, Any]] = []
+        for line in payload.split("\n"):
+            if not line.strip():
+                continue
+            try:
+                entry = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            if isinstance(entry, dict):
+                services.append(entry)
+        return services
+
+    def containers_running(self) -> bool:
+        """Check whether every container of the project is already up."""
+        result = self._compose(["ps", "-q"], timeout=STATUS_TIMEOUT)
+        if result is None or result.returncode != 0:
+            return False
+
+        container_ids = [line.strip() for line in result.stdout.strip().split("\n")]
+        container_ids = [cid for cid in container_ids if cid]
+        if not container_ids:
+            return False
+
+        for container_id in container_ids:
+            status = self._run(
+                ["docker", "inspect", "-f", "{{.State.Running}}", container_id],
+                timeout=SHORT_TIMEOUT,
+            )
+            if status is None or status.returncode != 0:
+                return False
+            if status.stdout.strip() != "true":
+                return False
+        return True
+
+    def up(self, timeout: int) -> subprocess.CompletedProcess[str]:
+        """Build and start the compose services in the background."""
+        result = self._compose(["up", "-d", "--build"], timeout=timeout)
+        if result is None:
+            raise subprocess.TimeoutExpired(" ".join(self.compose_command()), timeout)
+        return result
+
+    def wait_until_healthy(self, timeout: int) -> None:
+        """Block until every service reports ``running`` or ``timeout`` elapses."""
+        debug_log("Waiting for services to be healthy...", "info")
+        deadline = time.time() + timeout
+
+        while time.time() < deadline:
+            services = self._services(timeout=SHORT_TIMEOUT)
+            if services and all(
+                service.get("State") == "running" for service in services
+            ):
+                debug_log("All services are running", "info")
+                # Give the application a moment to finish booting.
+                time.sleep(10)
+                return
+
+            debug_log("Services not ready yet, waiting...", "info")
+            time.sleep(5)
+
+        debug_log(
+            f"Services did not become healthy within {timeout} seconds", "warning"
+        )
+
+    def verify_services_running(self) -> Optional[str]:
+        """Return ``None`` when db and app are up, else a descriptive error."""
+        services = self._services(timeout=STATUS_TIMEOUT)
+        if not services:
+            return "Failed to check service status"
+
+        db_running = False
+        app_running = False
+        for service in services:
+            name = str(service.get("Name", ""))
+            state = str(service.get("State", ""))
+            debug_log(f"Service {name}: {state}", "info")
+            if "db" in name and state == "running":
+                db_running = True
+            elif "app" in name and state == "running":
+                app_running = True
+
+        if not db_running:
+            return "Database service is not running"
+        if not app_running:
+            logs = self._compose(
+                ["logs", "--tail", "50", "app"], timeout=STATUS_TIMEOUT
+            )
+            if logs is not None and logs.returncode == 0:
+                return (
+                    "Application service is not running. " f"Logs: {logs.stdout[-500:]}"
+                )
+            return "Application service is not running"
+
+        debug_log("All required services are running", "info")
+        return None
+
+    def published_port(self, service_hint: str = "app") -> Optional[int]:
+        """Return the host port the app container publishes, if any.
+
+        ``docker-compose ps --format json`` reports a ``Publishers`` list per
+        service; an entry with a non-zero ``PublishedPort`` is a port bound on
+        the host, which is what a smoke test can reach. Services whose name
+        contains ``service_hint`` are preferred, so a database that happens to
+        publish a port is never mistaken for the application.
+        """
+        services = self._services(timeout=STATUS_TIMEOUT)
+        if not services:
+            return None
+
+        def _matches(service: Dict[str, Any]) -> bool:
+            name = f"{service.get('Service', '')} {service.get('Name', '')}"
+            return service_hint in name
+
+        for candidate in (
+            [entry for entry in services if _matches(entry)],
+            services,
+        ):
+            for service in candidate:
+                port = self._first_published_port(service)
+                if port is not None:
+                    return port
+        return None
+
+    @staticmethod
+    def _first_published_port(service: Dict[str, Any]) -> Optional[int]:
+        """Extract the first host-bound port from one ``ps`` entry."""
+        publishers = service.get("Publishers")
+        if not isinstance(publishers, list):
+            return None
+        for publisher in publishers:
+            if not isinstance(publisher, dict):
+                continue
+            try:
+                port = int(publisher.get("PublishedPort", 0))
+            except (TypeError, ValueError):
+                continue
+            if port > 0:
+                return port
+        return None
+
+    def exec_tests(
+        self, use_test_script: bool
+    ) -> Optional[subprocess.CompletedProcess[str]]:
+        """Run the template's tests inside the app container."""
+        if use_test_script:
+            command = ["exec", "-T", "app", "bash", "scripts/test.sh"]
+        else:
+            command = ["exec", "-T", "app", "python", "-m", "pytest", "tests/", "-v"]
+        return self._compose(command, timeout=TEST_TIMEOUT)
+
+    def reclaim_bind_mount_ownership(self) -> None:
+        """Give the bind-mounted project directory back to the host user.
+
+        Containers run as root, so a test run inside the stack leaves
+        root-owned artefacts (``__pycache__``, ``.pytest_cache``, ...) in the
+        mounted project. Later host-side steps and the temp directory cleanup
+        cannot touch those and fail with a PermissionError, so the ownership
+        is reset from inside a throwaway container. Best effort: a failure
+        here is never worth failing an inspection over.
+        """
+        if not hasattr(os, "getuid"):  # pragma: no cover - Windows hosts
+            return
+
+        uid, gid = os.getuid(), os.getgid()
+        if uid == 0:
+            return
+
+        debug_log("Reclaiming ownership of bind-mounted project files", "info")
+        self._run(
+            [
+                "docker",
+                "run",
+                "--rm",
+                "-v",
+                f"{self.project_dir}:/mnt",
+                RECLAIM_IMAGE,
+                "chown",
+                "-R",
+                f"{uid}:{gid}",
+                "/mnt",
+            ],
+            timeout=CLEANUP_TIMEOUT,
+        )
+
+    def cleanup(self) -> None:
+        """Tear down services and volumes, ignoring any failure."""
+        debug_log("Cleaning up Docker services", "info")
+        self.reclaim_bind_mount_ownership()
+        self._run(
+            [*self.compose_command(), "down", "-v", "--remove-orphans"],
+            timeout=CLEANUP_TIMEOUT,
+        )
+        self._run(["docker", "system", "prune", "-f"], timeout=STATUS_TIMEOUT)