pytest-xspec 0.5.0__py3-none-any.whl

pytest_xspec/collector.py

@@ -0,0 +1,23 @@
+"""Validate requirement associations before pytest filters collected tests."""
+
+from collections.abc import Collection
+
+import pytest
+
+
+def requirement_ids(item: pytest.Item, inventory: Collection[str]) -> tuple[str, ...]:
+    identifiers: dict[str, None] = {}
+
+    for marker in item.iter_markers("xspec"):
+        if len(marker.args) != 1 or marker.kwargs or not isinstance(marker.args[0], str):
+            raise pytest.UsageError(
+                f"{item.nodeid}: xspec requires exactly one positional string ID and no keywords"
+            )
+
+        identifier = marker.args[0]
+        if identifier not in inventory:
+            raise pytest.UsageError(f"{item.nodeid}: unknown xspec requirement ID {identifier!r}")
+
+        identifiers[identifier] = None
+
+    return tuple(identifiers)

pytest_xspec/config.py

@@ -0,0 +1,60 @@
+"""Read requirement inventories from the two supported project files."""
+
+import configparser
+import sys
+from importlib import import_module
+
+import pytest
+
+
+def load_requirements(config: pytest.Config) -> list[str]:
+    inventories: dict[str, object] = {}
+    pyproject = config.rootpath / "pyproject.toml"
+    ini = config.rootpath / "pytest.ini"
+
+    try:
+        if pyproject.is_file():
+            # pytest supplies tomli on Python 3.10; newer Python supplies tomllib.
+            toml = import_module("tomllib" if sys.version_info >= (3, 11) else "tomli")
+            with pyproject.open("rb") as stream:
+                section = toml.load(stream).get("tool", {}).get("xspec", {})
+
+            if not isinstance(section, dict):
+                raise pytest.UsageError("pyproject.toml: xspec must be a table")
+
+            if "requirements" in section:
+                inventories[pyproject.name] = section["requirements"]
+
+        if ini.is_file():
+            parser = configparser.ConfigParser(interpolation=None)
+            parser.read(ini, encoding="utf-8")
+
+            if parser.has_option("pytest", "xspec"):
+                inventories[ini.name] = [
+                    line.strip()
+                    for line in parser.get("pytest", "xspec").splitlines()
+                    if line.strip()
+                ]
+    except (OSError, ValueError, configparser.Error) as error:
+        raise pytest.UsageError(f"Cannot read xspec requirements: {error}") from error
+
+    if len(inventories) > 1:
+        raise pytest.UsageError("Declare requirements in only one of pyproject.toml and pytest.ini")
+
+    if not inventories:
+        return []
+
+    filename, requirements = next(iter(inventories.items()))
+    if not isinstance(requirements, list) or any(
+        not isinstance(requirement, str) for requirement in requirements
+    ):
+        raise pytest.UsageError(f"{filename}: requirements must be a list of string IDs")
+
+    seen: set[str] = set()
+    for requirement in requirements:
+        if requirement in seen:
+            raise pytest.UsageError(f"{filename}: duplicate requirements ID {requirement!r}")
+
+        seen.add(requirement)
+
+    return requirements

pytest_xspec/outcomes.py

@@ -0,0 +1,38 @@
+"""Translate execution phases into requirement verification outcomes."""
+
+from collections.abc import Iterable
+
+PRECEDENCE = {
+    status: rank
+    for rank, status in enumerate(
+        ("PASSED", "DESELECTED", "SKIPPED", "XFAILED", "XPASSED", "FAILED", "ERROR")
+    )
+}
+
+
+def phase_outcome(
+    when: str | None, outcome: str, *, wasxfail: bool = False, longrepr: object = None
+) -> str | None:
+    if wasxfail and outcome == "skipped":
+        return "XFAILED"
+
+    if outcome == "failed":
+        if when != "call":
+            return "ERROR"
+
+        if isinstance(longrepr, str) and longrepr.startswith("[XPASS(strict)] "):
+            return "XPASSED"
+
+        return "FAILED"
+
+    if outcome == "skipped":
+        return "SKIPPED"
+
+    if when == "call":
+        return "XPASSED" if wasxfail else "PASSED"
+
+    return None
+
+
+def aggregate(outcomes: Iterable[str]) -> str:
+    return max(outcomes, key=PRECEDENCE.__getitem__, default="MISSING")

pytest_xspec/plugin.py

@@ -0,0 +1,94 @@
+"""Pytest entry point for executable requirement verification."""
+
+from collections.abc import Generator, Sequence
+from pathlib import Path
+
+import pytest
+from _pytest.terminal import TerminalReporter
+
+from pytest_xspec.collector import requirement_ids
+from pytest_xspec.config import load_requirements
+from pytest_xspec.outcomes import aggregate, phase_outcome
+from pytest_xspec.reporting import build_report, write_json, write_terminal
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+    group = parser.getgroup("xspec")
+    group.addoption("--xspec-report", action="store_true", help="Print requirement verification")
+    group.addoption("--xspec-strict", action="store_true", help="Fail on unverified requirements")
+    group.addoption("--xspec-json", metavar="PATH", help="Write requirement verification as JSON")
+
+    parser.addini("xspec", "Requirement IDs, one per line (pytest.ini only)", "linelist")
+
+
+def pytest_configure(config: pytest.Config) -> None:
+    config.addinivalue_line("markers", "xspec(id): associate a test with a declared requirement")
+    config.pluginmanager.register(RequirementPlugin(config), "xspec-results")
+
+
+class RequirementPlugin:
+    def __init__(self, config: pytest.Config) -> None:
+        self.config = config
+        self.requirements: dict[str, list[str]] = {
+            identifier: [] for identifier in load_requirements(config)
+        }
+        self.outcomes: dict[str, str | None] = {}
+        self.collection_errors = 0
+
+    @pytest.hookimpl(wrapper=True, tryfirst=True)
+    def pytest_collection_modifyitems(self, items: list[pytest.Item]) -> Generator[None]:
+        for item in items:
+            identifiers = requirement_ids(item, self.requirements)
+
+            if identifiers and item.nodeid not in self.outcomes:
+                self.outcomes[item.nodeid] = None
+
+                for identifier in identifiers:
+                    self.requirements[identifier].append(item.nodeid)
+
+        yield
+
+    def pytest_deselected(self, items: Sequence[pytest.Item]) -> None:
+        for item in items:
+            if item.nodeid in self.outcomes:
+                self.outcomes[item.nodeid] = "DESELECTED"
+
+    def pytest_collectreport(self, report: pytest.CollectReport) -> None:
+        if report.failed:
+            self.collection_errors += 1
+
+    def pytest_runtest_logreport(self, report: pytest.TestReport) -> None:
+        if report.nodeid in self.outcomes:
+            outcome = phase_outcome(
+                report.when,
+                report.outcome,
+                wasxfail=hasattr(report, "wasxfail"),
+                longrepr=report.longrepr,
+            )
+
+            if outcome is not None:
+                previous = self.outcomes[report.nodeid]
+                self.outcomes[report.nodeid] = aggregate(
+                    [previous, outcome] if previous is not None else [outcome]
+                )
+
+    def pytest_sessionfinish(self, session: pytest.Session) -> None:
+        report = build_report(self.requirements, self.outcomes, self.collection_errors)
+
+        path = self.config.getoption("xspec_json")
+        if path:
+            write_json(report, Path(path))
+
+        if (
+            self.config.getoption("xspec_strict")
+            and not report["summary"]["verified"]
+            and session.exitstatus in (pytest.ExitCode.OK, pytest.ExitCode.NO_TESTS_COLLECTED)
+        ):
+            session.exitstatus = pytest.ExitCode.TESTS_FAILED
+
+    def pytest_terminal_summary(self, terminalreporter: TerminalReporter) -> None:
+        if self.config.getoption("xspec_report") or self.config.getoption("xspec_strict"):
+            write_terminal(
+                build_report(self.requirements, self.outcomes, self.collection_errors),
+                terminalreporter,
+            )

pytest_xspec/reporting.py

@@ -0,0 +1,103 @@
+"""Build and render the public requirement report."""
+
+import json
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from typing import TypedDict
+
+import pytest
+from _pytest.terminal import TerminalReporter
+
+from pytest_xspec.outcomes import aggregate
+
+
+class TestResult(TypedDict):
+    nodeid: str
+    outcome: str
+
+
+class RequirementResult(TypedDict):
+    status: str
+    tests: list[TestResult]
+
+
+class Summary(TypedDict):
+    requirements: int
+    passed: int
+    unverified: int
+    coverage: float
+    verified: bool
+    collection_errors: int
+
+
+class Report(TypedDict):
+    requirements: dict[str, RequirementResult]
+    summary: Summary
+
+
+def build_report(
+    requirements: Mapping[str, Sequence[str]],
+    outcomes: Mapping[str, str | None],
+    collection_errors: int,
+) -> Report:
+    results: dict[str, RequirementResult] = {}
+
+    for identifier, nodeids in requirements.items():
+        tests: list[TestResult] = [
+            {"nodeid": nodeid, "outcome": outcomes[nodeid] or "SKIPPED"} for nodeid in nodeids
+        ]
+        results[identifier] = {
+            "status": aggregate(test["outcome"] for test in tests),
+            "tests": tests,
+        }
+
+    count = len(results)
+    passed = sum(result["status"] == "PASSED" for result in results.values())
+
+    return {
+        "requirements": results,
+        "summary": {
+            "requirements": count,
+            "passed": passed,
+            "unverified": count - passed,
+            "coverage": passed / count * 100 if count else 0.0,
+            "verified": passed == count and collection_errors == 0,
+            "collection_errors": collection_errors,
+        },
+    }
+
+
+def write_json(report: Report, path: Path) -> None:
+    try:
+        path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    except OSError as error:
+        raise pytest.UsageError(f"Cannot write xspec JSON report {path}: {error}") from error
+
+
+def write_terminal(report: Report, terminal: TerminalReporter) -> None:
+    red = not report["summary"]["verified"]
+    terminal.write_sep("=", "xspec", red=red, green=not red)
+
+    width = max([len("Requirement"), *(len(identifier) for identifier in report["requirements"])])
+    terminal.write_line(f"{'Requirement':<{width}}  {'Status':<10}  Tests", red=red, green=not red)
+
+    for identifier, result in report["requirements"].items():
+        terminal.write_line(
+            f"{identifier:<{width}}  {result['status']:<10}  {len(result['tests'])}",
+            red=red,
+            green=not red,
+        )
+
+    for label, field in (
+        ("Requirements", "requirements"),
+        ("Passed", "passed"),
+        ("Unverified", "unverified"),
+    ):
+        terminal.write_line(f"{label}: {report['summary'][field]}", red=red, green=not red)
+
+    terminal.write_line(f"Coverage: {report['summary']['coverage']:.1f}%", red=red, green=not red)
+
+    if report["summary"]["collection_errors"]:
+        terminal.write_line(
+            f"Collection errors: {report['summary']['collection_errors']}", red=red, green=not red
+        )

pytest_xspec-0.5.0.dist-info/METADATA

@@ -0,0 +1,205 @@
+Metadata-Version: 2.5
+Name: pytest-xspec
+Version: 0.5.0
+Summary: Executable acceptance criteria for pytest
+Project-URL: Repository, https://github.com/Thyholt/pytest-xspec
+Project-URL: Issues, https://github.com/Thyholt/pytest-xspec/issues
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.10
+Requires-Dist: pytest>=8
+Description-Content-Type: text/markdown
+
+# pytest-xspec
+
+[![PyPI version][pypi-version-badge]][pypi]
+[![Python versions][python-versions-badge]][pypi]
+[![Tests][tests-badge]][tests]
+[![Code coverage][codecov-badge]][codecov]
+
+**Executable specifications with pytest.**
+
+Link specifications to your existing pytest tests, organize your test suite your way, and see
+which specifications your test suite verifies. Detect missing tests and enforce verification in CI.
+
+## Usage
+
+Install `pytest-xspec` into your pytest environment. The plugin is discovered
+automatically. Python 3.10+ and pytest 8+ are supported.
+
+```sh
+pip install pytest-xspec
+```
+
+### Declare specifications
+
+Declare specification IDs under `requirements` in `pyproject.toml`.
+
+```toml
+[tool.xspec]
+requirements = ["REQ-001", "REQ-002"]
+```
+
+Alternatively, use one ID per line in `pytest.ini`.
+
+```ini
+[pytest]
+xspec =
+    REQ-001
+    REQ-002
+```
+
+Declare specifications in one of these files. Using both is an error, and other
+files do not supply specifications. IDs are opaque strings. Duplicates and marked
+IDs that have not been declared are rejected. No inventory, or an empty inventory,
+means zero specifications.
+
+### Mark tests
+
+Associate tests with specifications, using one marker per specification.
+
+```python
+import pytest
+
+
+@pytest.mark.xspec("REQ-001")
+@pytest.mark.xspec("REQ-002")
+def test_shared_behavior():
+    assert 1 + 1 == 2
+```
+
+### Run pytest
+
+Plain `pytest` checks specification configuration and markers without printing
+a specification report. Add `--xspec-report` to see the report, or `--xspec-strict`
+to also fail the run when a specification is unverified.
+
+```sh
+pytest --xspec-report             # Print the verification report
+pytest --xspec-json=report.json   # Export statuses and individual test outcomes
+pytest --xspec-strict             # Report and fail on unverified specifications
+pytest -m xspec --xspec-strict    # Run only marked specification tests
+```
+
+The report is green when all specifications are verified and red when verification fails.
+The first report shows the example above; the second shows a specification without tests.
+
+![Terminal reports showing verified specifications in green and a missing specification in red.](https://raw.githubusercontent.com/Thyholt/pytest-xspec/master/docs/images/terminal-reports.png)
+
+### Read specification statuses
+
+Reports label specifications as requirements.
+
+A specification is verified when every test marked with its ID passes. One passing
+test is not enough if another test for the same specification fails or skips.
+
+When those tests have different results, the report uses the first matching
+status in this table, reading from top to bottom.
+
+| Status | Meaning |
+| --- | --- |
+| `ERROR` | A test's setup or cleanup failed. |
+| `FAILED` | A test failed. |
+| `XPASSED` | A test marked as an expected failure passed instead. |
+| `XFAILED` | A test failed as expected. |
+| `SKIPPED` | A test was skipped or never ran. |
+| `DESELECTED` | A test was excluded by a filter such as `-k` or `-m`. |
+| `PASSED` | Every test for the specification passed. |
+
+For example, if one test passes and another skips, the specification is `SKIPPED`
+and remains unverified. If one test fails and another skips, it is `FAILED`.
+
+### Missing and unfinished tests
+
+Pytest first finds tests, then runs them. If it finds no marked tests for a
+declared specification, that specification is `MISSING`. Tests that were found but
+never ran are `SKIPPED`, including runs that only find tests or stop early.
+If pytest encounters an error while finding tests, such as a broken import,
+the verification check cannot succeed.
+
+### Tests without markers
+
+Tests without a specification marker do not change specification statuses. They
+can still make pytest fail. In [the duck example][acceptance-only],
+the specification passes, but the unmarked cheese test fails the pytest run.
+
+### Check coverage
+
+Coverage is the percentage of declared specifications that are verified. With
+no declared specifications and no errors while finding tests, the verification
+check succeeds and coverage is 0.0%.
+
+Errors while finding tests make verification fail, even at 100% coverage. The report follows
+pytest's color settings, so `--color=no` disables color.
+
+For a complete report, let pytest find the whole test suite. With `-k` or `-m`,
+the plugin still knows about the tests excluded by the filter and reports marked
+ones as `DESELECTED`. If you run only one folder, it cannot see tests in other
+folders.
+
+## Runnable examples
+
+Start with [the pizza][pizza], then try the other
+examples to see what makes a specification pass or remain unverified.
+
+Run the commands below from the repository root. Dependencies are installed automatically.
+
+| Example | With `--xspec-strict` | Exit code |
+| --- | --- | --- |
+| [Quickstart][quickstart] - Two slices of pizza | `PASSED`, 100% coverage | 0 |
+| [Missing specification][missing-requirement] - A lost sock | `MISSING`, 50% coverage | 1 |
+| [Partial verification][partial-verification] - Cat skips a meeting | `SKIPPED`, 0% coverage | 1 |
+| [Acceptance only][acceptance-only] - Duck versus cheese | `PASSED` with `-m xspec` | 0 |
+
+### Run an example
+
+Run the `Quickstart` example, then replace `quickstart` with another example folder.
+
+```sh
+uv run --locked pytest examples/quickstart --xspec-strict
+```
+
+For `acceptance_only`, add `-m xspec` to run only the marked test.
+Without that filter, the unmarked test makes pytest fail.
+
+### Inspect the JSON report
+
+Export the example results to see the specification status and individual test outcomes.
+
+```sh
+uv run --locked pytest examples/quickstart --xspec-json=examples/quickstart/report.json
+```
+
+Open [examples/quickstart/report.json][report] after running the command.
+
+### Verify all examples
+
+Check that every example behaves as described.
+
+```sh
+uv run --locked pytest examples/test_examples.py -n 0
+```
+
+All five checks should pass, including checks for the examples that deliberately
+fail. The examples are also verified in CI.
+
+## Contributing
+
+See [CONTRIBUTING.md][contributing] for setup, tests and pull requests.
+
+[acceptance-only]: https://github.com/Thyholt/pytest-xspec/tree/master/examples/acceptance_only
+[pizza]: https://github.com/Thyholt/pytest-xspec/blob/master/examples/quickstart/test_behavior.py
+[quickstart]: https://github.com/Thyholt/pytest-xspec/tree/master/examples/quickstart
+[missing-requirement]: https://github.com/Thyholt/pytest-xspec/tree/master/examples/missing_requirement
+[partial-verification]: https://github.com/Thyholt/pytest-xspec/tree/master/examples/partial_verification
+[report]: https://github.com/Thyholt/pytest-xspec/blob/master/examples/quickstart/report.json
+[contributing]: https://github.com/Thyholt/pytest-xspec/blob/master/CONTRIBUTING.md
+
+[pypi]: https://pypi.org/project/pytest-xspec/
+[pypi-version-badge]: https://img.shields.io/pypi/v/pytest-xspec
+[python-versions-badge]: https://img.shields.io/badge/python-3.10%2B-blue
+[tests]: https://github.com/Thyholt/pytest-xspec/actions/workflows/check.yml
+[tests-badge]: https://img.shields.io/github/actions/workflow/status/Thyholt/pytest-xspec/check.yml?branch=master&label=tests
+
+[codecov]: https://app.codecov.io/gh/Thyholt/pytest-xspec
+[codecov-badge]: https://codecov.io/gh/Thyholt/pytest-xspec/branch/master/graph/badge.svg

pytest_xspec-0.5.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+pytest_xspec/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pytest_xspec/collector.py,sha256=4qSrsbvJ78bqggYYG4Sbq7bXp2wc0DjRs_50W2KE76c,794
+pytest_xspec/config.py,sha256=KFRbd4GRj20SjDv9d5lGtq0fsLUdiON44OEyVpxdhfc,2181
+pytest_xspec/outcomes.py,sha256=K4EpjJMq16J9ffDSUKceNf1U5I1Bx2e8Ngi3iL_R1GQ,952
+pytest_xspec/plugin.py,sha256=9UYo-qN-VS5vtTIbeND9bBCZ3JyCnoJwGU7GAIc7V4Y,3770
+pytest_xspec/reporting.py,sha256=T4v4i8bCxzG6srsXCW3sv1jvARdvpn1C7dZHXGm8C3k,3096
+pytest_xspec-0.5.0.dist-info/METADATA,sha256=8l014jjgVJ2-XByMd0J7vtAJO2jfSkWeU3aJsMGcgSg,7687
+pytest_xspec-0.5.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+pytest_xspec-0.5.0.dist-info/entry_points.txt,sha256=TCQsg6OwG8dKvjg3eeazOa9fY5GgKp1ONpBbMtoCZiI,39
+pytest_xspec-0.5.0.dist-info/licenses/LICENSE,sha256=UKFrzi3aP48KpaWL0SxbfrYo85PrVDvEqw3oShDk6xg,1082
+pytest_xspec-0.5.0.dist-info/RECORD,,

pytest_xspec-0.5.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

pytest_xspec-0.5.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[pytest11]
+xspec = pytest_xspec.plugin

pytest_xspec-0.5.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Andreas Thyholt Henriksen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.