auto-api-test 0.1.0__py3-none-any.whl

auto_api_test/plugin.py

@@ -0,0 +1,193 @@
+"""pytest plugin: collects YAML test files and registers them as test items."""
+
+from pathlib import Path
+
+import pytest
+
+from .parser import load_yaml, get_config, get_tests
+from .runner import Runner, TestResult
+from .reporter import generate_report, collect_failures
+from .issue import create_issues_from_failures
+
+# Module-level caches
+_yaml_cache: dict[str, dict] = {}
+_file_configs: dict[str, dict] = {}
+
+
+def pytest_collect_file(parent, file_path):
+    """Collect .yaml and .yml files as test modules."""
+    if file_path.suffix in (".yaml", ".yml"):
+        return YamlTestFile.from_parent(parent, path=file_path)
+
+
+class YamlTestFile(pytest.File):
+    """A YAML file containing API test cases."""
+
+    def collect(self):
+        try:
+            data = load_yaml(self.path)
+            _yaml_cache[str(self.path)] = data
+        except Exception as e:
+            item = YamlTestItem.from_parent(self, name="parse_error")
+            item._error = str(e)
+            item._test_data = None
+            item._config = {}
+            item._index = 0
+            yield item
+            return
+
+        config = get_config(data)
+        _file_configs[str(self.path)] = config
+        tests = get_tests(data)
+        for i, test in enumerate(tests):
+            item = YamlTestItem.from_parent(self, name=test["name"])
+            item._error = None
+            item._test_data = test
+            item._config = config
+            item._index = i
+            yield item
+
+
+class YamlTestItem(pytest.Item):
+    """A single YAML API test case."""
+
+    def __init__(self, name, parent):
+        super().__init__(name, parent)
+        self._error: str | None = None
+        self._test_data: dict | None = None
+        self._config: dict = {}
+        self._index: int = 0
+        self._result: TestResult | None = None
+
+    def runtest(self):
+        if self._error:
+            raise YamlTestError(self._error)
+
+        runner = self._get_or_create_runner()
+        result = runner.run_tests([self._test_data])[0]
+        self._result = result
+
+        if not hasattr(self.session.config, "_atr_results"):
+            self.session.config._atr_results = []
+        self.session.config._atr_results.append(result)
+
+        if result.is_error:
+            raise YamlTestError(result.error)
+        if not result.passed:
+            raise YamlTestAssertionError(result.failures)
+
+    def _get_or_create_runner(self):
+        """Get or create a Runner shared across tests in the same file."""
+        cache_key = f"_yaml_runner_{self.fspath}"
+        runner = getattr(self.session, cache_key, None)
+        if runner is None:
+            config = dict(self._config)
+            opt = getattr(self.session.config, "option", None)
+            if opt and getattr(opt, "base_url", None):
+                config["base_url"] = opt.base_url
+            runner = Runner(config)
+            setattr(self.session, cache_key, runner)
+        return runner
+
+    def repr_failure(self, excinfo):
+        if isinstance(excinfo.value, YamlTestError):
+            return f"ERROR: {excinfo.value}"
+        if isinstance(excinfo.value, YamlTestAssertionError):
+            lines = ["Assertion failures:"]
+            for f in excinfo.value.failures:
+                lines.append(f"  - {f}")
+            return "\n".join(lines)
+        return super().repr_failure(excinfo)
+
+    def reportinfo(self):
+        return self.fspath, None, self.name
+
+
+class YamlTestError(Exception):
+    """Raised when a test encounters an error (network, parse, etc.)."""
+
+
+class YamlTestAssertionError(Exception):
+    """Raised when test assertions fail."""
+
+    def __init__(self, failures: list[str]):
+        self.failures = failures
+        super().__init__("\n".join(failures))
+
+
+def pytest_addoption(parser):
+    """Add custom CLI options for YAML API testing."""
+    parser.addoption("--report", default=None, help="Markdown report output path")
+    parser.addoption("--base-url", default=None, help="Override base_url for all tests")
+    parser.addoption("--env-file", default=None, help="Load environment variables from file")
+
+
+def pytest_configure(config):
+    """Load env-file if specified."""
+    env_file = config.getoption("env_file", None)
+    if env_file:
+        from dotenv import load_dotenv
+        load_dotenv(env_file)
+
+
+def pytest_sessionfinish(session, exitstatus):
+    """Generate report and optionally create issues after all tests complete."""
+    results = getattr(session.config, "_atr_results", [])
+    if not results:
+        return
+
+    # Determine report path: CLI override > YAML file parent dir / report.md
+    report_path = session.config.getoption("report", None)
+    if not report_path:
+        for item in session.items:
+            if isinstance(item, YamlTestItem):
+                report_path = str(item.path.parent / "report.md")
+                break
+    if not report_path:
+        return
+
+    report_content = generate_report(results, report_path)
+
+    # Auto-create issues if configured in any YAML file
+    _maybe_create_issues(session, results)
+
+
+def _maybe_create_issues(session, results: list[TestResult]):
+    """Create git platform issues for failed tests if configured."""
+    failures = collect_failures(results)
+    if not failures:
+        return
+
+    for item in session.items:
+        if not isinstance(item, YamlTestItem):
+            continue
+
+        config = _file_configs.get(str(item.path), {})
+        issue_cfg = config.get("issue", {})
+        if not issue_cfg.get("enabled"):
+            continue
+
+        platform = issue_cfg.get("platform")
+        repo = issue_cfg.get("repo")
+        if not platform or not repo:
+            continue
+
+        base_url = issue_cfg.get("base_url", "")
+        labels = issue_cfg.get("labels")
+
+        # Only create issues for failures from this file
+        file_failures = [
+            f for f in failures
+            if any(r.name == f["name"] and not r.passed for r in results if r.name == f["name"])
+        ]
+        if not file_failures:
+            continue
+
+        try:
+            created = create_issues_from_failures(
+                file_failures, platform, repo, base_url, labels
+            )
+            for issue in created:
+                print(f"  Created issue: {issue.get('url', issue)}")
+        except Exception as e:
+            print(f"  Warning: failed to create issues: {e}")

auto_api_test/reporter.py

@@ -0,0 +1,115 @@
+"""Markdown test report generator."""
+
+import datetime
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .runner import TestResult
+
+
+def generate_report(results: list["TestResult"], output_path: str | Path) -> str:
+    """Generate a Markdown test report from test results.
+
+    Args:
+        results: List of test results.
+        output_path: Absolute path for the report file.
+    """
+    total = len(results)
+    passed = sum(1 for r in results if r.passed)
+    failed = total - passed
+    errors = sum(1 for r in results if r.is_error)
+    pass_rate = f"{passed / total * 100:.1f}%" if total else "0%"
+    total_duration = sum(r.duration_ms for r in results)
+
+    now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+    lines = [
+        "# API Test Report",
+        "",
+        f"**Generated:** {now}",
+        "",
+        "## Summary",
+        "",
+        "| Metric | Value |",
+        "|--------|-------|",
+        f"| Total | {total} |",
+        f"| Passed | {passed} |",
+        f"| Failed | {failed - errors} |",
+        f"| Errors | {errors} |",
+        f"| Pass Rate | {pass_rate} |",
+        f"| Total Duration | {total_duration:.2f} ms |",
+        "",
+        "## Test Results",
+        "",
+        "| # | Name | Method | URL | Status | Duration | Result |",
+        "|---|------|--------|-----|--------|----------|--------|",
+    ]
+
+    for i, r in enumerate(results, 1):
+        status = str(r.status_code) if r.status_code else "-"
+        result_str = "✅ PASS" if r.passed else ("❌ ERROR" if r.is_error else "❌ FAIL")
+        url_short = r.request_url
+        if len(url_short) > 60:
+            url_short = url_short[:57] + "..."
+        lines.append(
+            f"| {i} | {r.name} | {r.request_method} | {url_short} | {status} | {r.duration_ms:.2f} ms | {result_str} |"
+        )
+
+    # Failure details
+    failed_results = [r for r in results if not r.passed]
+    if failed_results:
+        lines.extend(["", "## Failure Details", ""])
+        for r in failed_results:
+            lines.append(f"### {r.name}")
+            lines.append("")
+            if r.is_error:
+                lines.append(f"**Error:** `{r.error}`")
+            else:
+                for f in r.failures:
+                    lines.append(f"- {f}")
+            if r.response_body:
+                lines.extend([
+                    "",
+                    "<details><summary>Response Body</summary>",
+                    "",
+                    "```json",
+                    r.response_body[:1000],
+                    "```",
+                    "",
+                    "</details>",
+                ])
+            lines.append("")
+
+    report = "\n".join(lines)
+
+    output_path = Path(output_path)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text(report, encoding="utf-8")
+
+    return report
+
+
+def collect_failures(results: list["TestResult"]) -> list[dict]:
+    """Extract failed/error tests as structured dicts for issue creation."""
+    failures = []
+    for r in results:
+        if r.passed:
+            continue
+        entry = {
+            "name": r.name,
+            "method": r.request_method,
+            "url": r.request_url,
+            "status_code": r.status_code,
+            "duration_ms": r.duration_ms,
+        }
+        if r.is_error:
+            entry["type"] = "ERROR"
+            entry["detail"] = r.error
+        else:
+            entry["type"] = "FAIL"
+            entry["detail"] = "\n".join(f"- {f}" for f in r.failures)
+        if r.response_body:
+            entry["response_body"] = r.response_body[:500]
+        failures.append(entry)
+    return failures

auto_api_test/runner.py

@@ -0,0 +1,164 @@
+"""HTTP request runner with variable context management."""
+
+import json as json_module
+import re
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+import requests
+
+from .assertions import assert_headers, assert_json, assert_status_code
+from .parser import resolve_vars
+
+
+@dataclass
+class TestResult:
+    """Result of a single test case execution."""
+    name: str
+    passed: bool
+    status_code: int = 0
+    duration_ms: float = 0.0
+    request_url: str = ""
+    request_method: str = ""
+    response_body: str = ""
+    failures: list[str] = field(default_factory=list)
+    error: str = ""
+    is_error: bool = False
+    response_json: Any = None
+
+
+class Runner:
+    """Executes API test cases sequentially with variable context."""
+
+    def __init__(self, config: dict):
+        self.base_url = config.get("base_url", "").rstrip("/")
+        self.timeout = config.get("timeout", 30)
+        self.default_headers = config.get("headers", {})
+        self.session = requests.Session()
+        self.session.headers.update(self.default_headers)
+        self.variables: dict[str, str] = {}
+
+    def run_tests(self, tests: list[dict]) -> list[TestResult]:
+        """Run all test cases in order, maintaining variable context."""
+        results = []
+        for test in tests:
+            result = self._run_single(test)
+            results.append(result)
+            if not result.is_error:
+                self._extract_vars(test, result)
+        return results
+
+    def _run_single(self, test: dict) -> TestResult:
+        """Execute a single test case."""
+        name = test["name"]
+        req_spec = test.get("request", {})
+        expect = test.get("expect", {})
+
+        req_spec = resolve_vars(req_spec, self.variables)
+        expect = resolve_vars(expect, self.variables)
+
+        method = req_spec.get("method", "GET").upper()
+        url = req_spec.get("url", "")
+        if self.base_url and not url.startswith(("http://", "https://")):
+            url = f"{self.base_url}{url}"
+
+        headers = req_spec.get("headers", {})
+        params = req_spec.get("params", {})
+        json_body = req_spec.get("json")
+        body = req_spec.get("body")
+
+        start = time.monotonic()
+        try:
+            resp = self.session.request(
+                method=method,
+                url=url,
+                headers=headers,
+                params=params,
+                json=json_body,
+                data=body,
+                timeout=self.timeout,
+            )
+        except requests.RequestException as e:
+            duration = (time.monotonic() - start) * 1000
+            return TestResult(
+                name=name,
+                passed=False,
+                request_url=url,
+                request_method=method,
+                duration_ms=round(duration, 2),
+                error=str(e),
+                is_error=True,
+            )
+
+        duration = (time.monotonic() - start) * 1000
+        failures = []
+
+        if "status_code" in expect:
+            msg = assert_status_code(expect["status_code"], resp.status_code)
+            if msg:
+                failures.append(msg)
+
+        resp_json = None
+        if "json" in expect:
+            try:
+                resp_json = resp.json()
+                failures.extend(assert_json(expect["json"], resp_json))
+            except ValueError:
+                failures.append("Response body is not valid JSON")
+
+        if "headers" in expect:
+            failures.extend(assert_headers(expect["headers"], dict(resp.headers)))
+
+        try:
+            resp_body = resp.text[:2000]
+        except Exception:
+            resp_body = ""
+
+        return TestResult(
+            name=name,
+            passed=len(failures) == 0,
+            status_code=resp.status_code,
+            duration_ms=round(duration, 2),
+            request_url=url,
+            request_method=method,
+            response_body=resp_body,
+            failures=failures,
+            response_json=resp_json,
+        )
+
+    def _resolve_extract_path(self, data: Any, path: str) -> Any:
+        """Resolve a dotted path to extract a value from JSON data."""
+        parts = path.split(".")
+        current = data
+        for part in parts:
+            if isinstance(current, dict) and part in current:
+                current = current[part]
+            elif isinstance(current, list) and part.isdigit():
+                idx = int(part)
+                if idx < len(current):
+                    current = current[idx]
+                else:
+                    return None
+            else:
+                return None
+        return current
+
+    def _extract_vars(self, test: dict, result: TestResult) -> None:
+        """Extract variables from response for subsequent tests."""
+        extract = test.get("extract", {})
+        if not extract:
+            return
+
+        resp_json = result.response_json
+        if resp_json is None:
+            try:
+                resp_json = json_module.loads(result.response_body)
+            except (json_module.JSONDecodeError, TypeError):
+                return
+
+        for var_name, json_path in extract.items():
+            json_path = resolve_vars(json_path, self.variables)
+            value = self._resolve_extract_path(resp_json, json_path)
+            if value is not None:
+                self.variables[var_name] = str(value)