pytest-openapi 0.1.1.dev202601050151__py3-none-any.whl

pytest_openapi/openapi.py

@@ -0,0 +1,259 @@
+"""OpenAPI specification validation and parsing."""
+
+import sys
+
+import requests
+
+
+def check_request_body_has_example(method, path, operation):
+    """Check if a POST/PUT/DELETE operation has a request body example
+    or schema.
+
+    Args:
+        method: HTTP method (post, put, delete)
+        path: API endpoint path
+        operation: OpenAPI operation object
+
+    Returns:
+        str or None: Error message if example AND schema are missing, None otherwise
+    """
+    request_body = operation.get("requestBody", {})
+    content = request_body.get("content", {})
+
+    has_example_or_schema = False
+    for media_type, media_obj in content.items():
+        if (
+            "example" in media_obj
+            or "examples" in media_obj
+            or "schema" in media_obj
+        ):
+            has_example_or_schema = True
+            break
+
+    if content and not has_example_or_schema:
+        return f"  - {method.upper()} {path}: Missing request body example or schema"
+
+    return None
+
+
+def check_operation_has_responses(method, path, operation):
+    """Check if an operation has response definitions.
+
+    Args:
+        method: HTTP method (get, post, put, delete)
+        path: API endpoint path
+        operation: OpenAPI operation object
+
+    Returns:
+        str or None: Error message if responses are missing, None otherwise
+    """
+    responses = operation.get("responses", {})
+    if not responses:
+        return f"  - {method.upper()} {path}: Missing response definitions"
+
+    return None
+
+
+def check_response_has_example(method, path, status_code, response_obj):
+    """Check if a response definition has an example or schema.
+
+    Args:
+        method: HTTP method (get, post, put, delete)
+        path: API endpoint path
+        status_code: HTTP status code
+        response_obj: OpenAPI response object
+
+    Returns:
+        str or None: Error message if example AND schema are missing, None otherwise
+    """
+    content = response_obj.get("content", {})
+
+    has_example_or_schema = False
+    for media_type, media_obj in content.items():
+        if (
+            "example" in media_obj
+            or "examples" in media_obj
+            or "schema" in media_obj
+        ):
+            has_example_or_schema = True
+            break
+
+    if content and not has_example_or_schema:
+        return f"  - {method.upper()} {path}: Missing response example or schema for status {status_code}"
+
+    return None
+
+
+def check_schema_descriptions(schema, path_prefix=""):
+    """Recursively check that all fields in a schema have descriptions.
+
+    Args:
+        schema: OpenAPI schema object
+        path_prefix: Current path in the schema (for error messages)
+
+    Returns:
+        list: List of error messages for fields missing descriptions
+    """
+    errors = []
+
+    if not isinstance(schema, dict):
+        return errors
+
+    # Check properties
+    properties = schema.get("properties", {})
+    for prop_name, prop_schema in properties.items():
+        prop_path = f"{path_prefix}.{prop_name}" if path_prefix else prop_name
+
+        # Check if this property has a description
+        if "description" not in prop_schema:
+            errors.append(f"Field '{prop_path}' is missing a description")
+
+        # Recursively check nested objects
+        if prop_schema.get("type") == "object":
+            nested_errors = check_schema_descriptions(prop_schema, prop_path)
+            errors.extend(nested_errors)
+
+        # Check items in arrays
+        elif prop_schema.get("type") == "array":
+            items = prop_schema.get("items", {})
+            if items.get("type") == "object":
+                nested_errors = check_schema_descriptions(
+                    items, f"{prop_path}[]"
+                )
+                errors.extend(nested_errors)
+
+    return errors
+
+
+def check_endpoint_schema_descriptions(method, path, operation):
+    """Check that all schema fields have descriptions for an endpoint.
+
+    Args:
+        method: HTTP method (get, post, put, delete)
+        path: API endpoint path
+        operation: OpenAPI operation object
+
+    Returns:
+        list: List of error messages for missing descriptions
+    """
+    errors = []
+
+    # Check request body schema
+    if method in ["post", "put", "delete"]:
+        request_body = operation.get("requestBody", {})
+        content = request_body.get("content", {})
+
+        for media_type, media_obj in content.items():
+            schema = media_obj.get("schema", {})
+            if schema:
+                schema_errors = check_schema_descriptions(schema)
+                for error in schema_errors:
+                    errors.append(
+                        f"  - {method.upper()} {path} request body: {error}"
+                    )
+
+    # Check response schemas
+    responses = operation.get("responses", {})
+    for status_code, response_obj in responses.items():
+        content = response_obj.get("content", {})
+
+        for media_type, media_obj in content.items():
+            schema = media_obj.get("schema", {})
+            if schema:
+                schema_errors = check_schema_descriptions(schema)
+                for error in schema_errors:
+                    errors.append(
+                        f"  - {method.upper()} {path} response {status_code}: {error}"
+                    )
+
+    return errors
+
+
+def validate_openapi_spec(base_url):
+    """Validate that the OpenAPI spec is available and meets
+    requirements.
+
+    Checks:
+    1. /openapi.json endpoint is accessible
+    2. All GET/POST/PUT/DELETE endpoints have request examples (where applicable)
+    3. All endpoints have response schemas
+
+    Args:
+        base_url: Base URL of the API server
+
+    Raises:
+        SystemExit: If validation fails
+    """
+    openapi_url = f"{base_url}/openapi.json"
+
+    # Check 1: Fetch OpenAPI spec
+    try:
+        response = requests.get(openapi_url, timeout=10)
+        response.raise_for_status()
+        spec = response.json()
+    except requests.exceptions.RequestException as e:
+        print(f"\n❌ ERROR: Could not fetch OpenAPI spec from {openapi_url}")
+        print(f"   Reason: {e}")
+        sys.exit(1)
+    except ValueError as e:
+        print(f"\n❌ ERROR: Invalid JSON in OpenAPI spec from {openapi_url}")
+        print(f"   Reason: {e}")
+        sys.exit(1)
+    # TODO: Add JsonDecodeError handling
+
+    # Validate the spec structure
+    if "paths" not in spec:
+        print("\n❌ ERROR: OpenAPI spec missing 'paths' key")
+        sys.exit(1)
+
+    # Check 2 & 3: Validate endpoints
+    errors = []
+    paths = spec.get("paths", {})
+
+    for path, path_item in paths.items():
+        for method in ["get", "post", "put", "delete"]:
+            if method not in path_item:
+                continue
+
+            operation = path_item[method]
+
+            # Check request body examples for POST/PUT/DELETE
+            if method in ["post", "put", "delete"]:
+                error = check_request_body_has_example(method, path, operation)
+                if error:
+                    errors.append(error)
+
+            # Check response schemas for all methods
+            error = check_operation_has_responses(method, path, operation)
+            if error:
+                errors.append(error)
+                continue
+
+            responses = operation.get("responses", {})
+            for status_code, response_obj in responses.items():
+                error = check_response_has_example(
+                    method, path, status_code, response_obj
+                )
+                if error:
+                    errors.append(error)
+
+            # Check schema field descriptions
+            description_errors = check_endpoint_schema_descriptions(
+                method, path, operation
+            )
+            errors.extend(description_errors)
+
+    # Report errors
+    if errors:
+        print("\n❌ ERROR: OpenAPI spec validation failed")
+        print("\nThe following endpoints have validation errors:")
+        for error in errors:
+            print(error)
+        print(
+            "\npytest-openapi requires proper schemas with descriptions for all fields, "
+            "and either examples or complete schemas for all endpoints."
+        )
+        sys.exit(1)
+
+    print(f"\n✅ OpenAPI spec validated successfully from {openapi_url}")
+    print(f"   Found {len(paths)} path(s)")

pytest_openapi/plugin.py

@@ -0,0 +1,92 @@
+"""Pytest plugin for OpenAPI contract testing."""
+
+
+def pytest_addoption(parser):
+    """Add --openapi CLI option."""
+    group = parser.getgroup("openapi")
+    group.addoption(
+        "--openapi",
+        action="store",
+        metavar="BASE_URL",
+        help="Run OpenAPI contract tests against the specified base URL",
+    )
+
+
+def pytest_configure(config):
+    """Configure pytest with OpenAPI marker."""
+    config.addinivalue_line(
+        "markers",
+        "openapi: OpenAPI contract tests",
+    )
+
+    # If --openapi flag is provided, validate and test the OpenAPI spec
+    base_url = config.getoption("--openapi")
+    if base_url:
+        import pytest
+        import requests
+
+        from .contract import (
+            get_test_report,
+            test_delete_endpoint,
+            test_get_endpoint,
+            test_post_endpoint,
+            test_put_endpoint,
+        )
+        from .openapi import validate_openapi_spec
+
+        # Run validation checks
+        validate_openapi_spec(base_url)
+
+        # Fetch the OpenAPI spec
+        try:
+            response = requests.get(f"{base_url}/openapi.json", timeout=10)
+            response.raise_for_status()
+            spec = response.json()
+        except Exception as e:
+            pytest.exit(f"\n❌ Failed to fetch OpenAPI spec: {e}", returncode=1)
+
+        # Run contract tests on all endpoints
+        paths = spec.get("paths", {})
+        errors = []
+
+        for path, path_item in paths.items():
+            for method in ["get", "post", "put", "delete"]:
+                if method not in path_item:
+                    continue
+
+                operation = path_item[method]
+
+                # Select appropriate test function
+                if method == "get":
+                    success, error = test_get_endpoint(
+                        base_url, path, operation
+                    )
+                elif method == "post":
+                    success, error = test_post_endpoint(
+                        base_url, path, operation
+                    )
+                elif method == "put":
+                    success, error = test_put_endpoint(
+                        base_url, path, operation
+                    )
+                elif method == "delete":
+                    success, error = test_delete_endpoint(
+                        base_url, path, operation
+                    )
+
+                if not success:
+                    errors.append(f"  {method.upper()} {path}: {error}")
+
+        # Generate and display human-readable report
+        print("\n")
+        print(get_test_report())
+
+        # Report results and exit
+        if errors:
+            print("\n❌ Contract tests failed:")
+            for error in errors:
+                print(error)
+            pytest.exit("Contract tests failed", returncode=1)
+        else:
+            print("\n✅ All contract tests passed!")
+            # Don't call sys.exit(0) - let pytest finish normally

pytest_openapi-0.1.1.dev202601050151.dist-info/METADATA

@@ -0,0 +1,182 @@
+Metadata-Version: 2.4
+Name: pytest-openapi
+Version: 0.1.1.dev202601050151
+Author-email: Sinan Ozel <coding@sinan.slmail.me>
+License: MIT License
+        
+        Copyright (c) 2025 Sinan Ozel
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Homepage, https://github.com/sinan-ozel/pytest-openapi
+Project-URL: Issues, https://github.com/sinan-ozel/pytest-openapi/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pytest>=7.0.0
+Requires-Dist: requests>=2.31.0
+Requires-Dist: exrex>=0.11.0
+Provides-Extra: test
+Requires-Dist: pytest>=7.0.0; extra == "test"
+Requires-Dist: pytest-depends>=1.0.1; extra == "test"
+Requires-Dist: pytest-mock>=3.14.0; extra == "test"
+Requires-Dist: httpx>=0.28.1; extra == "test"
+Provides-Extra: dev
+Requires-Dist: isort>=5.12.0; extra == "dev"
+Requires-Dist: ruff>=0.12.11; extra == "dev"
+Requires-Dist: black>=24.0.0; extra == "dev"
+Requires-Dist: docformatter>=1.7.5; extra == "dev"
+Provides-Extra: docs
+Requires-Dist: mkdocs-material>=9.0.0; extra == "docs"
+Requires-Dist: mkdocstrings[python]>=0.24.0; extra == "docs"
+Provides-Extra: publish
+Requires-Dist: packaging>=25.0; extra == "publish"
+Dynamic: license-file
+
+![Tests & Lint](https://github.com/sinan-ozel/pytest-openapi/actions/workflows/ci.yaml/badge.svg?branch=main)
+![PyPI](https://img.shields.io/pypi/v/pytest-openapi.svg)
+![Downloads](https://static.pepy.tech/badge/pytest-openapi)
+![Monthly Downloads](https://static.pepy.tech/badge/pytest-openapi/month)
+[![Documentation](https://img.shields.io/badge/docs-mkdocs-blue)](https://sinan-ozel.github.io/pytest-openapi/)
+![License](https://img.shields.io/github/license/sinan-ozel/pytest-openapi.svg)
+![Python](https://img.shields.io/badge/Python-3.11%2B-blue)
+
+# 🧪 OpenAPI Contract Tester
+
+An opinionated, lightweight **black-box contract tester** against a **live API** using its OpenAPI specification as the source of truth.
+
+This tool validates OpenAPI quality, generates test cases from schemas, and verifies that real HTTP responses match the contract.
+This "certifies" that the documentation is complete with descriptions, example, and schema, and that the endpoint behaves as the documentation suggests.
+
+## Why?
+
+This package tries to simulate the frustrations of API users, as consumers.
+With the rise of "agents", this type of documentation-code match became even
+more important, because LLMs really have trouble choosing tools or using them
+properly when they do not work as intended.
+
+## ✨ What it does
+
+### ▶️ Quick Example
+
+![Swagger POST endpoint /email](swagger-screenshot-1.png)
+
+```bash
+pytest --openapi=http://localhost:8000
+```
+
+```
+Test #10 ✅
+POST /email
+Requested:
+  {
+    "body": "Lorem ipsum dolor sit amet",
+    "from": "Lorem ipsum dolor sit amet",
+    "subject": "Lorem ipsum dolor sit amet",
+    "to": "Test!@#$%^&*()_+-=[]{}|;:<>?,./`~"
+  }
+
+Expected 201
+  {
+    "body": "Hi Bob, how are you?",
+    "from": "alice@example.com",
+    "id": 1,
+    "subject": "Hello",
+    "to": "bob@example.com"
+  }
+
+Actual 201
+  {
+    "body": "Lorem ipsum dolor sit amet",
+    "from": "Lorem ipsum dolor sit amet",
+    "id": 10,
+    "subject": "Lorem ipsum dolor sit amet",
+    "to": "Test!@#$%^&*()_+-=[]{}|;:<>?,./`~"
+  }
+
+```
+Generates multiple QA tests.
+
+✔️ Validates OpenAPI request/response definitions
+✔️ Enforces schema field descriptions
+✔️ Generates test cases from schemas, checks response codes and types in the response
+✔️ Tests the exanples
+✔️ Tests **GET / POST / PUT / DELETE** endpoints
+✔️ Compares live responses against examples
+✔️ Produces a readable test report
+
+
+# ▶️ Detailed Example
+
+## Install
+```bash
+pip install pytest-openapi
+```
+
+## Run
+
+Say that you have a service running at port `8000` on `localhost`. Then, run:
+
+```bash
+pytest --openapi=http://localhost:8000
+```
+
+## Server
+See here an example server - `email-server`: [tests/test_servers/email_server/server.py](tests/test_servers/email_server/server.py)
+
+## Resulting Tests
+
+[tests/test_servers/email_server/email_test_output.txt](tests/test_servers/email_server/email_test_output.txt)
+
+# Future Plans / TODO
+
+This is a work in progress.
+- [ ] A check that the example matches the schema
+- [ ] Ask that 400 responses be in the documentation.
+- [ ] A check for regexp and email formats.
+- [ ] Extra checks from 200 or 201 messages with missing keys to see 400 messages.
+- [ ] Option to turn off the description requirement.
+
+## In Consideration
+- [ ] Use LLM-as-a-judge to assess the error messages and check their spelling.
+
+# Contributing
+Contributions are welcome!
+
+The only requirement is 🐳 Docker.
+
+Test are containerized, run them using the VS Code task `test`. If you don't want to use VS Code, the command is `docker compose -f ./tests/docker-compose.yaml --project-directory ./tests up --build --abort-on-container-exit --exit-code-from test`. Run this before making a PR, please.
+
+There is also a development environment for VS Code, if you need it. On this environment, you can run the task `run-mock-server` to run one of the [mock servers](tests/test_servers) and see the output.
+
+You can add your own mock server, and then add integration tests. Just follow the same pattern as every test to make a call - `subprocess.run('pytest', '--openapi=http://your-server:8000`.
+
+Please reformat and lint before making a PR. The VS Task is `lint`, and if you don't want to use VS Code, the command is: `docker compose -f ./lint/docker-compose.yaml --project-directory ./lint up --build --abort-on-container-exit --exit-code-from linter`. Run this before making a PR, please.
+
+If you add a functionality, please add to the the documentation.
+
+Please submit a pull request or open an issue for any bugs or feature requests.
+
+The moment your PR is merged, you get a dev release. You can then set up the version number to use your changes.
+
+# License
+This project is licensed under the MIT License. See the [LICENSE](LICENSE) file for details.
+

pytest_openapi-0.1.1.dev202601050151.dist-info/RECORD

@@ -0,0 +1,11 @@
+pytest_openapi/__init__.py,sha256=mi83Zkryg5sd8L5o1QhMUJtIerdG3p6UWysB0tY7fOk,38
+pytest_openapi/case_generator.py,sha256=vi_PtPgAhHliO6po2SYOmZNJFpyWGi0rYjg-q-kz6bc,13851
+pytest_openapi/contract.py,sha256=Le_EdJgfgYR4MCa7N7FYAI4eNbhVuPHpdC5gPMyZV0k,27246
+pytest_openapi/openapi.py,sha256=dlGw_8GHI_qrQa4Y82BFgKRzrfAat878fCZkXbmeR3E,8328
+pytest_openapi/plugin.py,sha256=vrvOQm2yOKdDl3eFvyVfkLMBGkQurqwqAyTvEBQyvPo,2954
+pytest_openapi-0.1.1.dev202601050151.dist-info/licenses/LICENSE,sha256=uCWQbz0H2dwv5L630wv0BEcZOs9hWrOj1lR7mPnEnV8,1067
+pytest_openapi-0.1.1.dev202601050151.dist-info/METADATA,sha256=spp5jzHq0fCNVt4UIudRYh4QlFkpvwqAfmVF96NNCNo,7249
+pytest_openapi-0.1.1.dev202601050151.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pytest_openapi-0.1.1.dev202601050151.dist-info/entry_points.txt,sha256=PGXqI13KlKF_FMiu_Bfwg7IsiLP8kjZ85W5cYxAvVZk,50
+pytest_openapi-0.1.1.dev202601050151.dist-info/top_level.txt,sha256=uxNtTL00ENrH7IA6TwaBpMKzllTkGIQ7Z2WTbkEmn3M,15
+pytest_openapi-0.1.1.dev202601050151.dist-info/RECORD,,

pytest_openapi-0.1.1.dev202601050151.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

pytest_openapi-0.1.1.dev202601050151.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[pytest11]
+pytest_openapi = pytest_openapi.plugin

pytest_openapi-0.1.1.dev202601050151.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Sinan Ozel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pytest_openapi-0.1.1.dev202601050151.dist-info/top_level.txt

@@ -0,0 +1 @@
+pytest_openapi