pytest-openapi 0.1.0__tar.gz

pytest_openapi-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Sinan Ozel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pytest_openapi-0.1.0/PKG-INFO

@@ -0,0 +1,174 @@
+Metadata-Version: 2.4
+Name: pytest-openapi
+Version: 0.1.0
+Author-email: Sinan Ozel <coding@sinan.slmail.me>
+License: MIT License
+        
+        Copyright (c) 2025 Sinan Ozel
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Homepage, https://github.com/sinan-ozel/pytest-openapi
+Project-URL: Issues, https://github.com/sinan-ozel/pytest-openapi/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pytest>=7.0.0
+Requires-Dist: requests>=2.31.0
+Requires-Dist: exrex>=0.11.0
+Provides-Extra: test
+Requires-Dist: pytest>=7.0.0; extra == "test"
+Requires-Dist: pytest-depends>=1.0.1; extra == "test"
+Requires-Dist: pytest-mock>=3.14.0; extra == "test"
+Requires-Dist: httpx>=0.28.1; extra == "test"
+Provides-Extra: dev
+Requires-Dist: isort>=5.12.0; extra == "dev"
+Requires-Dist: ruff>=0.12.11; extra == "dev"
+Requires-Dist: black>=24.0.0; extra == "dev"
+Requires-Dist: docformatter>=1.7.5; extra == "dev"
+Provides-Extra: docs
+Requires-Dist: mkdocs-material>=9.0.0; extra == "docs"
+Requires-Dist: mkdocstrings[python]>=0.24.0; extra == "docs"
+Provides-Extra: publish
+Requires-Dist: packaging>=25.0; extra == "publish"
+Dynamic: license-file
+
+# 🧪 OpenAPI Contract Tester
+
+An opinionated, lightweight **black-box contract tester** against a **live API** using its OpenAPI specification as the source of truth.
+
+This tool validates OpenAPI quality, generates test cases from schemas, and verifies that real HTTP responses match the contract.
+This "certifies" that the documentation is complete with descriptions, example, and schema, and that the endpoint behaves as the documentation suggests.
+
+## Why?
+
+This package tries to simulate the frustrations of API users, as consumers.
+With the rise of "agents", this type of documentation-code match became even
+more important, because LLMs really have trouble choosing tools or using them
+properly when they do not work as intended.
+
+## ✨ What it does
+
+### ▶️ Quick Example
+
+![Swagger POST endpoint /email](swagger-screenshot-1.png)
+
+```bash
+pytest --openapi=http://localhost:8000
+```
+
+```
+Test #10 ✅
+POST /email
+Requested:
+  {
+    "body": "Lorem ipsum dolor sit amet",
+    "from": "Lorem ipsum dolor sit amet",
+    "subject": "Lorem ipsum dolor sit amet",
+    "to": "Test!@#$%^&*()_+-=[]{}|;:<>?,./`~"
+  }
+
+Expected 201
+  {
+    "body": "Hi Bob, how are you?",
+    "from": "alice@example.com",
+    "id": 1,
+    "subject": "Hello",
+    "to": "bob@example.com"
+  }
+
+Actual 201
+  {
+    "body": "Lorem ipsum dolor sit amet",
+    "from": "Lorem ipsum dolor sit amet",
+    "id": 10,
+    "subject": "Lorem ipsum dolor sit amet",
+    "to": "Test!@#$%^&*()_+-=[]{}|;:<>?,./`~"
+  }
+
+```
+Generates multiple QA tests.
+
+✔️ Validates OpenAPI request/response definitions
+✔️ Enforces schema field descriptions
+✔️ Generates test cases from schemas, checks response codes and types in the response
+✔️ Tests the exanples
+✔️ Tests **GET / POST / PUT / DELETE** endpoints
+✔️ Compares live responses against examples
+✔️ Produces a readable test report
+
+
+# ▶️ Detailed Example
+
+## Install
+```bash
+pip install pytest-openapi
+```
+
+## Run
+
+Say that you have a service running at port `8000` on `localhost`. Then, run:
+
+```bash
+pytest --openapi=http://localhost:8000
+```
+
+## Server
+See here an example server - `email-server`: [tests/test_servers/email_server/server.py](tests/test_servers/email_server/server.py)
+
+## Resulting Tests
+
+[tests/test_servers/email_server/email_test_output.txt](tests/test_servers/email_server/email_test_output.txt)
+
+# Future Plans / TODO
+
+This is a work in progress.
+- [ ] A check that the example matches the schema
+- [ ] Ask that 400 responses be in the documentation.
+- [ ] A check for regexp and email formats.
+- [ ] Extra checks from 200 or 201 messages with missing keys to see 400 messages.
+- [ ] Option to turn off the description requirement.
+
+## In Consideration
+- [ ] Use LLM-as-a-judge to assess the error messages and check their spelling.
+
+# Contributing
+Contributions are welcome!
+
+The only requirement is 🐳 Docker.
+
+Test are containerized, run them using the VS Code task `test`. If you don't want to use VS Code, the command is `docker compose -f ./tests/docker-compose.yaml --project-directory ./tests up --build --abort-on-container-exit --exit-code-from test`. Run this before making a PR, please.
+
+There is also a development environment for VS Code, if you need it. On this environment, you can run the task `run-mock-server` to run one of the [mock servers](tests/test_servers) and see the output.
+
+You can add your own mock server, and then add integration tests. Just follow the same pattern as every test to make a call - `subprocess.run('pytest', '--openapi=http://your-server:8000`.
+
+Please reformat and lint before making a PR. The VS Task is `lint`, and if you don't want to use VS Code, the command is: `docker compose -f ./lint/docker-compose.yaml --project-directory ./lint up --build --abort-on-container-exit --exit-code-from linter`. Run this before making a PR, please.
+
+If you add a functionality, please add to the the documentation.
+
+Please submit a pull request or open an issue for any bugs or feature requests.
+
+The moment your PR is merged, you get a dev release. You can then set up the version number to use your changes.
+
+# License
+This project is licensed under the MIT License. See the [LICENSE](LICENSE) file for details.
+

pytest_openapi-0.1.0/README.md

@@ -0,0 +1,121 @@
+# 🧪 OpenAPI Contract Tester
+
+An opinionated, lightweight **black-box contract tester** against a **live API** using its OpenAPI specification as the source of truth.
+
+This tool validates OpenAPI quality, generates test cases from schemas, and verifies that real HTTP responses match the contract.
+This "certifies" that the documentation is complete with descriptions, example, and schema, and that the endpoint behaves as the documentation suggests.
+
+## Why?
+
+This package tries to simulate the frustrations of API users, as consumers.
+With the rise of "agents", this type of documentation-code match became even
+more important, because LLMs really have trouble choosing tools or using them
+properly when they do not work as intended.
+
+## ✨ What it does
+
+### ▶️ Quick Example
+
+![Swagger POST endpoint /email](swagger-screenshot-1.png)
+
+```bash
+pytest --openapi=http://localhost:8000
+```
+
+```
+Test #10 ✅
+POST /email
+Requested:
+  {
+    "body": "Lorem ipsum dolor sit amet",
+    "from": "Lorem ipsum dolor sit amet",
+    "subject": "Lorem ipsum dolor sit amet",
+    "to": "Test!@#$%^&*()_+-=[]{}|;:<>?,./`~"
+  }
+
+Expected 201
+  {
+    "body": "Hi Bob, how are you?",
+    "from": "alice@example.com",
+    "id": 1,
+    "subject": "Hello",
+    "to": "bob@example.com"
+  }
+
+Actual 201
+  {
+    "body": "Lorem ipsum dolor sit amet",
+    "from": "Lorem ipsum dolor sit amet",
+    "id": 10,
+    "subject": "Lorem ipsum dolor sit amet",
+    "to": "Test!@#$%^&*()_+-=[]{}|;:<>?,./`~"
+  }
+
+```
+Generates multiple QA tests.
+
+✔️ Validates OpenAPI request/response definitions
+✔️ Enforces schema field descriptions
+✔️ Generates test cases from schemas, checks response codes and types in the response
+✔️ Tests the exanples
+✔️ Tests **GET / POST / PUT / DELETE** endpoints
+✔️ Compares live responses against examples
+✔️ Produces a readable test report
+
+
+# ▶️ Detailed Example
+
+## Install
+```bash
+pip install pytest-openapi
+```
+
+## Run
+
+Say that you have a service running at port `8000` on `localhost`. Then, run:
+
+```bash
+pytest --openapi=http://localhost:8000
+```
+
+## Server
+See here an example server - `email-server`: [tests/test_servers/email_server/server.py](tests/test_servers/email_server/server.py)
+
+## Resulting Tests
+
+[tests/test_servers/email_server/email_test_output.txt](tests/test_servers/email_server/email_test_output.txt)
+
+# Future Plans / TODO
+
+This is a work in progress.
+- [ ] A check that the example matches the schema
+- [ ] Ask that 400 responses be in the documentation.
+- [ ] A check for regexp and email formats.
+- [ ] Extra checks from 200 or 201 messages with missing keys to see 400 messages.
+- [ ] Option to turn off the description requirement.
+
+## In Consideration
+- [ ] Use LLM-as-a-judge to assess the error messages and check their spelling.
+
+# Contributing
+Contributions are welcome!
+
+The only requirement is 🐳 Docker.
+
+Test are containerized, run them using the VS Code task `test`. If you don't want to use VS Code, the command is `docker compose -f ./tests/docker-compose.yaml --project-directory ./tests up --build --abort-on-container-exit --exit-code-from test`. Run this before making a PR, please.
+
+There is also a development environment for VS Code, if you need it. On this environment, you can run the task `run-mock-server` to run one of the [mock servers](tests/test_servers) and see the output.
+
+You can add your own mock server, and then add integration tests. Just follow the same pattern as every test to make a call - `subprocess.run('pytest', '--openapi=http://your-server:8000`.
+
+Please reformat and lint before making a PR. The VS Task is `lint`, and if you don't want to use VS Code, the command is: `docker compose -f ./lint/docker-compose.yaml --project-directory ./lint up --build --abort-on-container-exit --exit-code-from linter`. Run this before making a PR, please.
+
+If you add a functionality, please add to the the documentation.
+
+Please submit a pull request or open an issue for any bugs or feature requests.
+
+The moment your PR is merged, you get a dev release. You can then set up the version number to use your changes.
+
+# License
+This project is licensed under the MIT License. See the [LICENSE](LICENSE) file for details.
+

pytest_openapi-0.1.0/pyproject.toml

@@ -0,0 +1,71 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "pytest-openapi"
+dynamic = ["version"]
+description = ""
+readme = "README.md"
+authors = [
+    { name = "Sinan Ozel", email = "coding@sinan.slmail.me" },
+]
+license = { file = "LICENSE" }
+dependencies = [
+    "pytest>=7.0.0",
+    "requests>=2.31.0",
+    "exrex>=0.11.0",
+]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent"
+]
+
+[project.optional-dependencies]
+test = [
+    "pytest>=7.0.0",          # For running tests
+    "pytest-depends>=1.0.1",  # For test dependency management
+    "pytest-mock>=3.14.0",    # For mocking in tests
+    "httpx>=0.28.1",          # To make more complex calls, such as streaming.
+]
+
+dev = [
+    "isort>=5.12.0",          # For import sorting
+    "ruff>=0.12.11",          # For linting
+    "black>=24.0.0",          # For code formatting (Black)
+    "docformatter>=1.7.5",    # For formatting docstrings
+]
+
+docs = [
+    "mkdocs-material>=9.0.0", # Material theme for MkDocs
+    "mkdocstrings[python]>=0.24.0",  # API documentation from docstrings
+]
+
+publish = [
+    "packaging>=25.0",
+]
+
+[tool.ruff]
+line-length = 80
+
+[tool.ruff.lint.per-file-ignores]
+"**/test_*.py" = ["RUF001"]
+
+[tool.black]
+line-length = 80
+target-version = ['py311']
+
+[tool.isort]
+profile = "black"
+line_length = 80
+
+[tool.setuptools.dynamic]
+version = { attr = "pytest_openapi.__version__" }
+
+[project.urls]
+Homepage = "https://github.com/sinan-ozel/pytest-openapi"
+Issues = "https://github.com/sinan-ozel/pytest-openapi/issues"
+
+[project.entry-points.pytest11]
+pytest_openapi = "pytest_openapi.plugin"

pytest_openapi-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

pytest_openapi-0.1.0/src/pytest_openapi/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"