pytest-api-cov 0.1.0__py3-none-any.whl

pytest_api_cov/plugin.py

@@ -0,0 +1,270 @@
+"""pytest plugin for API coverage tracking."""
+
+import importlib
+import importlib.util
+import logging
+import os
+from typing import Any, Optional
+
+import pytest
+
+from .config import get_pytest_api_cov_report_config
+from .frameworks import get_framework_adapter
+from .models import SessionData
+from .pytest_flags import add_pytest_api_cov_flags
+from .report import generate_pytest_api_cov_report
+
+logger = logging.getLogger(__name__)
+
+
+def is_supported_framework(app: Any) -> bool:
+    """Check if the app is a supported framework (Flask or FastAPI)."""
+    if app is None:
+        return False
+
+    app_type = type(app).__name__
+    module_name = getattr(type(app), "__module__", "").split(".")[0]
+
+    return (module_name == "flask" and app_type == "Flask") or (module_name == "fastapi" and app_type == "FastAPI")
+
+
+def auto_discover_app() -> Optional[Any]:
+    """Automatically discover Flask/FastAPI apps in common locations."""
+    logger.debug("> Auto-discovering app in common locations...")
+
+    common_patterns = [
+        ("app.py", ["app", "application", "main"]),
+        ("main.py", ["app", "application", "main"]),
+        ("server.py", ["app", "application", "server"]),
+        ("wsgi.py", ["app", "application"]),
+        ("asgi.py", ["app", "application"]),
+    ]
+
+    for filename, attr_names in common_patterns:
+        if os.path.exists(filename):
+            logger.debug(f"> Found {filename}, checking for app variables...")
+            try:
+                module_name = filename[:-3]  # .py extension
+                spec = importlib.util.spec_from_file_location(module_name, filename)
+                if spec and spec.loader:
+                    module = importlib.util.module_from_spec(spec)
+                    spec.loader.exec_module(module)
+
+                    for attr_name in attr_names:
+                        if hasattr(module, attr_name):
+                            app = getattr(module, attr_name)
+                            if is_supported_framework(app):
+                                logger.info(
+                                    f"✅ Auto-discovered {type(app).__name__} app in {filename} as '{attr_name}'"
+                                )
+                                return app
+                            else:
+                                logger.debug(f"> Found '{attr_name}' in {filename} but it's not a supported framework")
+
+            except Exception as e:
+                logger.debug(f"> Could not import {filename}: {e}")
+                continue
+
+    logger.debug("> No app auto-discovered")
+    return None
+
+
+def get_helpful_error_message() -> str:
+    """Generate a helpful error message for setup guidance."""
+    return """
+🚫 No API app found!
+
+Quick Setup Options:
+
+Option 1 - Auto-discovery (Recommended):
+  Place your FastAPI/Flask app in one of these files:
+  • app.py (with variable named 'app', 'application', or 'main')
+  • main.py (with variable named 'app', 'application', or 'main')
+  • server.py (with variable named 'app', 'application', or 'server')
+
+  Example app.py:
+    from fastapi import FastAPI
+    app = FastAPI()  # <- Plugin will auto-discover this
+
+Option 2 - Manual fixture:
+  Create conftest.py with:
+
+    import pytest
+    from your_module import your_app
+
+    @pytest.fixture
+    def app():
+        return your_app
+
+Then run: pytest --api-cov-report
+
+Need help? Run: pytest-api-cov init (for setup wizard)
+"""
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+    """Add API coverage flags to the pytest parser."""
+    add_pytest_api_cov_flags(parser)
+
+
+def pytest_configure(config: pytest.Config) -> None:
+    """Configure the pytest session and logging."""
+    if config.getoption("--api-cov-report"):
+        verbosity = config.option.verbose
+
+        if verbosity >= 2:  # -vv or more
+            log_level = logging.DEBUG
+        elif verbosity >= 1:  # -v
+            log_level = logging.INFO
+        else:
+            log_level = logging.WARNING
+
+        logger.setLevel(log_level)
+
+        if not logger.handlers:
+            handler = logging.StreamHandler()
+            handler.setLevel(log_level)
+            formatter = logging.Formatter("%(message)s")
+            handler.setFormatter(formatter)
+            logger.addHandler(handler)
+
+        logger.info("Initializing API coverage plugin...")
+
+    if config.pluginmanager.hasplugin("xdist"):
+        config.pluginmanager.register(DeferXdistPlugin(), "defer_xdist_plugin")
+
+
+def pytest_sessionstart(session: pytest.Session) -> None:
+    """Initialize the call recorder at the start of the session."""
+    if session.config.getoption("--api-cov-report"):
+        session.api_coverage_data = SessionData()  # type: ignore[attr-defined]
+
+
+@pytest.fixture
+def client(request: pytest.FixtureRequest) -> Any:
+    """
+    Smart auto-discovering test client that records API calls for coverage.
+
+    Tries to find an 'app' fixture first, then auto-discovers apps in common locations.
+    """
+    session = request.node.session
+
+    if not session.config.getoption("--api-cov-report"):
+        pytest.skip("API coverage not enabled. Use --api-cov-report flag.")
+
+    app = None
+    try:
+        app = request.getfixturevalue("app")
+        logger.debug("> Found 'app' fixture")
+    except pytest.FixtureLookupError:
+        logger.debug("> No 'app' fixture found, trying auto-discovery...")
+        app = auto_discover_app()
+
+    if app is None:
+        helpful_msg = get_helpful_error_message()
+        print(helpful_msg)
+        pytest.skip("No API app found. See error message above for setup guidance.")
+
+    if not is_supported_framework(app):
+        pytest.skip(f"Unsupported framework: {type(app).__name__}. pytest-api-coverage supports Flask and FastAPI.")
+
+    try:
+        adapter = get_framework_adapter(app)
+    except TypeError as e:
+        pytest.skip(f"Framework detection failed: {e}")
+
+    coverage_data = getattr(session, "api_coverage_data", None)
+    if coverage_data is None:
+        pytest.skip("API coverage data not initialized. This should not happen.")
+
+    if not coverage_data.discovered_endpoints.endpoints:
+        try:
+            endpoints = adapter.get_endpoints()
+            framework_name = type(app).__name__
+            for endpoint in endpoints:
+                coverage_data.add_discovered_endpoint(endpoint, f"{framework_name.lower()}_adapter")
+            logger.info(f"> pytest-api-coverage: Discovered {len(endpoints)} endpoints.")
+            logger.debug(f"> Discovered endpoints: {endpoints}")
+        except Exception as e:
+            logger.warning(f"> pytest-api-coverage: Could not discover endpoints. Error: {e}")
+
+    client = adapter.get_tracked_client(coverage_data.recorder, request.node.name)
+    yield client
+
+
+def pytest_sessionfinish(session: pytest.Session) -> None:
+    """Generate the API coverage report at the end of the session."""
+    if session.config.getoption("--api-cov-report"):
+        coverage_data = getattr(session, "api_coverage_data", None)
+        if coverage_data is None:
+            logger.warning("> No API coverage data found. Plugin may not have been properly initialized.")
+            return
+
+        logger.debug(f"> pytest-api-coverage: Generating report for {len(coverage_data.recorder)} recorded endpoints.")
+        if hasattr(session.config, "workeroutput"):
+            serializable_recorder = coverage_data.recorder.to_serializable()
+            session.config.workeroutput["api_call_recorder"] = serializable_recorder
+            session.config.workeroutput["discovered_endpoints"] = coverage_data.discovered_endpoints.endpoints
+            logger.debug("> Sent API call data and discovered endpoints to master process")
+        else:
+            logger.debug("> No workeroutput found, generating report for master data.")
+
+            worker_recorder_data = getattr(session.config, "worker_api_call_recorder", {})
+            worker_endpoints = getattr(session.config, "worker_discovered_endpoints", [])
+
+            # Merge worker data into session data
+            if worker_recorder_data or worker_endpoints:
+                coverage_data.merge_worker_data(worker_recorder_data, worker_endpoints)
+                logger.debug(f"> Merged worker data: {len(worker_recorder_data)} endpoints")
+
+            logger.debug(f"> Final merged data: {len(coverage_data.recorder)} recorded endpoints")
+            logger.debug(f"> Using discovered endpoints: {coverage_data.discovered_endpoints.endpoints}")
+
+            api_cov_config = get_pytest_api_cov_report_config(session.config)
+            status = generate_pytest_api_cov_report(
+                api_cov_config=api_cov_config,
+                called_data=coverage_data.recorder.calls,
+                discovered_endpoints=coverage_data.discovered_endpoints.endpoints,
+            )
+            if session.exitstatus == 0:
+                session.exitstatus = status
+
+        if hasattr(session, "api_coverage_data"):
+            delattr(session, "api_coverage_data")
+
+        if hasattr(session.config, "worker_api_call_recorder"):
+            delattr(session.config, "worker_api_call_recorder")
+
+
+class DeferXdistPlugin:
+    """Simple class to defer pytest-xdist hook until we know it is installed."""
+
+    def pytest_testnodedown(self, node: Any) -> None:
+        """Collect API call data from each worker as they finish."""
+        logger.debug("> pytest-api-coverage: Worker node down.")
+        worker_data = node.workeroutput.get("api_call_recorder", {})
+        discovered_endpoints = node.workeroutput.get("discovered_endpoints", [])
+        logger.debug(f"> Worker data: {worker_data}")
+        logger.debug(f"> Worker discovered endpoints: {discovered_endpoints}")
+
+        # Merge API call data
+        if worker_data:
+            logger.debug("> Worker data found, merging with current data.")
+            current = getattr(node.config, "worker_api_call_recorder", {})
+            logger.debug(f"> Current data before merge: {current}")
+
+            # Merge the worker data into current
+            for endpoint, calls in worker_data.items():
+                if endpoint not in current:
+                    current[endpoint] = set()
+                elif not isinstance(current[endpoint], set):
+                    current[endpoint] = set(current[endpoint])
+                current[endpoint].update(calls)
+                logger.debug(f"> Updated endpoint {endpoint} with calls: {calls}")
+
+            node.config.worker_api_call_recorder = current
+            logger.debug(f"> Updated current data: {current}")
+
+        if discovered_endpoints and not getattr(node.config, "worker_discovered_endpoints", []):
+            node.config.worker_discovered_endpoints = discovered_endpoints
+            logger.debug(f"> Set discovered endpoints from worker: {discovered_endpoints}")

pytest_api_cov/pytest_flags.py

@@ -0,0 +1,68 @@
+"""pytest flag configuration for API coverage."""
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+if TYPE_CHECKING:
+    import pytest
+
+
+def add_pytest_api_cov_flags(parser: pytest.Parser) -> None:
+    """Add API coverage flags to the parser."""
+    parser.addoption(
+        "--api-cov-report",
+        action="store_true",
+        default=False,
+        help="Generate API coverage report.",
+    )
+    parser.addoption(
+        "--api-cov-fail-under",
+        action="store",
+        type=float,
+        default=None,
+        help="Fail if API coverage is below this percentage.",
+    )
+    parser.addoption(
+        "--api-cov-show-uncovered-endpoints",
+        action="store_true",
+        default=True,
+        help="Show uncovered endpoints in the console report.",
+    )
+    parser.addoption(
+        "--api-cov-show-covered-endpoints",
+        action="store_true",
+        default=False,
+        help="Show covered endpoints in the console report.",
+    )
+    parser.addoption(
+        "--api-cov-show-excluded-endpoints",
+        action="store_true",
+        default=False,
+        help="Show excluded endpoints in the console report.",
+    )
+    parser.addoption(
+        "--api-cov-exclusion-patterns",
+        action="append",
+        default=[],
+        help="Patterns for endpoints to exclude from coverage.",
+    )
+    parser.addoption(
+        "--api-cov-report-path",
+        action="store",
+        type=str,
+        default=None,
+        help="Path to save the API coverage report.",
+    )
+    parser.addoption(
+        "--api-cov-force-sugar",
+        action="store_true",
+        default=False,
+        help="Force use of API coverage sugar in console report.",
+    )
+    parser.addoption(
+        "--api-cov-force-sugar-disabled",
+        action="store_true",
+        default=False,
+        help="Disable use of API coverage sugar in console report.",
+    )

pytest_api_cov/report.py

@@ -0,0 +1,185 @@
+"""API coverage report generation."""
+
+import json
+import re
+from pathlib import Path
+from re import Pattern
+from typing import Any, Dict, List, Set, Tuple
+
+from rich.console import Console
+
+from .config import ApiCoverageReportConfig
+
+
+def endpoint_to_regex(endpoint: str) -> Pattern[str]:
+    """Create a regex pattern from an endpoint by replacing dynamic segments."""
+    placeholder = "___PLACEHOLDER___"
+    temp_endpoint = re.escape(re.sub(r"<[^>]+>|\{[^}]+\}", placeholder, endpoint))
+    return re.compile("^" + temp_endpoint.replace(placeholder, "(.+)") + "$")
+
+
+def contains_escape_characters(endpoint: str) -> bool:
+    """Escape special characters in the endpoint string."""
+    return ("<" in endpoint and ">" in endpoint) or ("{" in endpoint and "}" in endpoint)
+
+
+def categorise_endpoints(
+    endpoints: List[str],
+    called_data: Dict[str, Set[str]],
+    exclusion_patterns: List[str],
+) -> Tuple[List[str], List[str], List[str]]:
+    """Categorise endpoints into covered, uncovered, and excluded.
+
+    Exclusion patterns support simple wildcard matching:
+    - Use * for wildcard (matches any characters)
+    - All other characters are matched literally
+    - Examples: "/admin/*", "/health", "/docs/*"
+    """
+    covered, uncovered, excluded = [], [], []
+
+    compiled_patterns = (
+        [re.compile("^" + re.escape(pattern).replace(r"\*", ".*") + "$") for pattern in exclusion_patterns]
+        if exclusion_patterns
+        else None
+    )
+
+    for endpoint in endpoints:
+        if compiled_patterns and any(p.match(endpoint) for p in compiled_patterns):
+            excluded.append(endpoint)
+            continue
+        elif contains_escape_characters(endpoint):
+            pattern = endpoint_to_regex(endpoint)
+            is_covered = any(pattern.match(ep) for ep in called_data)
+        else:
+            is_covered = endpoint in called_data
+        covered.append(endpoint) if is_covered else uncovered.append(endpoint)
+    return covered, uncovered, excluded
+
+
+def print_endpoints(
+    console: Console,
+    label: str,
+    endpoints: List[str],
+    symbol: str,
+    style: str,
+) -> None:
+    """Prints a list of endpoints to the console with a label and style."""
+    if endpoints:
+        console.print(f"[{style}]{label}[/]:")
+        for endpoint in endpoints:
+            console.print(f"  {symbol} [{style}]{endpoint}[/]")
+
+
+def compute_coverage(covered_count: int, uncovered_count: int) -> float:
+    """Compute API coverage percentage."""
+    total = covered_count + uncovered_count
+    return round(100 * covered_count / total, 2) if total > 0 else 0.0
+
+
+def prepare_endpoint_detail(endpoints: List[str], called_data: Dict[str, Set[str]]) -> List[Dict[str, Any]]:
+    """Prepare endpoint details by mapping each endpoint to its callers."""
+    details = []
+    for endpoint in endpoints:
+        if contains_escape_characters(endpoint):
+            pattern = endpoint_to_regex(endpoint)
+            callers = set()
+            for call in called_data:
+                if pattern.match(call):
+                    callers.update(called_data[call])
+        else:
+            callers = called_data.get(endpoint, set())
+        details.append({"endpoint": endpoint, "callers": sorted(list(callers))})
+    return sorted(details, key=lambda x: len(x["callers"]))
+
+
+def write_report_file(report_data: Dict[str, Any], report_path: str) -> None:
+    """Write the report data to a JSON file."""
+    path = Path(report_path).resolve()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(path, "w") as f:
+        json.dump(report_data, f, indent=2)
+
+
+def generate_pytest_api_cov_report(
+    api_cov_config: ApiCoverageReportConfig,
+    called_data: Dict[str, Set[str]],
+    discovered_endpoints: List[str],
+) -> int:
+    """Generate and print the API coverage report, returning an exit status."""
+
+    console = Console()
+
+    if not discovered_endpoints:
+        console.print("\n[bold red]No endpoints discovered. Please check your test setup.[/bold red]")
+        return 0
+
+    separator = "=" * 20
+    console.print(f"\n\n[bold blue]{separator} API Coverage Report {separator}[/bold blue]")
+
+    covered, uncovered, excluded = categorise_endpoints(
+        discovered_endpoints,
+        called_data,
+        api_cov_config.exclusion_patterns,
+    )
+
+    if api_cov_config.show_uncovered_endpoints:
+        print_endpoints(
+            console,
+            "Uncovered Endpoints",
+            uncovered,
+            "❌" if api_cov_config.force_sugar else "[X]",
+            "red",
+        )
+
+    if api_cov_config.show_covered_endpoints:
+        print_endpoints(
+            console,
+            "Covered Endpoints",
+            covered,
+            "✅" if api_cov_config.force_sugar else "[.]",
+            "green",
+        )
+
+    if api_cov_config.show_excluded_endpoints:
+        print_endpoints(
+            console,
+            label="Excluded Endpoints",
+            endpoints=excluded,
+            symbol="🚫" if api_cov_config.force_sugar else "[-]",
+            style="grey50",
+        )
+
+    coverage = compute_coverage(len(covered), len(uncovered))
+    status = 0
+
+    if api_cov_config.fail_under is None:
+        console.print(f"\n[bold green]Total API Coverage: {coverage}%[/bold green]")
+    elif coverage < api_cov_config.fail_under:
+        console.print(
+            f"\n[bold red]FAIL: Required coverage of {api_cov_config.fail_under}% not met. "
+            f"Actual coverage: {coverage}%[/bold red]"
+        )
+        status = 1
+    else:
+        console.print(
+            f"\n[bold green]SUCCESS: Coverage of {coverage}% meets requirement of "
+            f"{api_cov_config.fail_under}%[/bold green]"
+        )
+
+    if api_cov_config.report_path:
+        detail = prepare_endpoint_detail(covered + uncovered, called_data)
+        final_report = {
+            "status": status,
+            "coverage": coverage,
+            "required_coverage": api_cov_config.fail_under,
+            "total_endpoints": len(covered) + len(uncovered),
+            "covered_count": len(covered),
+            "uncovered_count": len(uncovered),
+            "excluded_count": len(excluded),
+            "detail": detail,
+        }
+        write_report_file(final_report, api_cov_config.report_path)
+        console.print(f"\n[grey50]JSON report saved to {api_cov_config.report_path}[/grey50]")
+
+    console.print(f"[bold blue]{'=' * (42 + len(' API Coverage Report '))}[/bold blue]\n")
+    return status