schemathesis 3.26.2__tar.gz → 3.27.0__tar.gz

{schemathesis-3.26.2 → schemathesis-3.27.0}/.github/workflows/build.yml

@@ -83,6 +83,6 @@ jobs:
         run: python -m tox
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v4.1.1
+        uses: codecov/codecov-action@v4.2.0
         with:
           token: ${{ secrets.CODECOV_TOKEN }}

{schemathesis-3.26.2 → schemathesis-3.27.0}/.github/workflows/codeql-analysis.yml

@@ -38,7 +38,7 @@ jobs:
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3.24.9
+        uses: github/codeql-action/init@v3.24.10
         with:
           languages: ${{ matrix.language }}
           # If you wish to specify custom queries, you can do so here or in a config file.
@@ -49,7 +49,7 @@ jobs:
       # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
       # If this step fails, then you should remove it and run the build manually (see below)
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v3.24.9
+        uses: github/codeql-action/autobuild@v3.24.10
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 https://git.io/JvXDl
@@ -63,4 +63,4 @@ jobs:
       #   make release
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3.24.9
+        uses: github/codeql-action/analyze@v3.24.10

{schemathesis-3.26.2 → schemathesis-3.27.0}/.pre-commit-config.yaml

@@ -3,7 +3,7 @@ default_language_version:
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
       - id: check-yaml
       - id: end-of-file-fixer
@@ -38,12 +38,12 @@ repos:
         additional_dependencies: [types-click, types-requests, types-PyYAML]
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.3.4
+    rev: v0.3.7
     hooks:
       - id: ruff-format
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.3.4
+    rev: v0.3.7
     hooks:
       - id: ruff
 

{schemathesis-3.26.2 → schemathesis-3.27.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: schemathesis
-Version: 3.26.2
+Version: 3.27.0
 Summary: Property-based testing framework for Open API and GraphQL based apps
 Project-URL: Documentation, https://schemathesis.readthedocs.io/en/stable/
 Project-URL: Changelog, https://schemathesis.readthedocs.io/en/stable/changelog.html

{schemathesis-3.26.2 → schemathesis-3.27.0}/docs/api.rst

@@ -113,8 +113,11 @@ Loaders
 .. autofunction:: schemathesis.from_pytest_fixture
 .. autofunction:: schemathesis.from_uri
 .. autofunction:: schemathesis.from_wsgi
+.. autofunction:: schemathesis.graphql.from_path
 .. autofunction:: schemathesis.graphql.from_dict
+.. autofunction:: schemathesis.graphql.from_file
 .. autofunction:: schemathesis.graphql.from_url
+.. autofunction:: schemathesis.graphql.from_asgi
 .. autofunction:: schemathesis.graphql.from_wsgi
 
 Sanitizing Output

{schemathesis-3.26.2 → schemathesis-3.27.0}/docs/changelog.rst

@@ -1,9 +1,28 @@
 Changelog
 =========
 
-:version:`Unreleased <v3.26.2...HEAD>` - TBD
+:version:`Unreleased <v3.27.0...HEAD>` - TBD
 --------------------------------------------
 
+.. _v3.27.0:
+
+:version:`3.27.0 <v3.26.2...v3.27.0>` - 2024-04-14
+--------------------------------------------------
+
+**Added**
+
+- ``Case.as_transport_kwargs`` method to simplify the creation of transport-specific keyword arguments for sending requests.
+
+**Changed**
+
+- Make ``Case.call`` work with ``ASGI`` & ``WSGI`` applications.
+- Extend the JUnit XML report format to match CLI output including skipped tests, code samples, and more.
+
+**Deprecated**
+
+- ``Case.call_wsgi`` & ``Case.call_asgi`` in favor of ``Case.call``.
+- ``Case.as_requests_kwargs`` & ``Case.as_werkzeug_kwargs`` in favor of ``Case.as_transport_kwargs``.
+
 .. _v3.26.2:
 
 :version:`3.26.2 <v3.26.1...v3.26.2>` - 2024-04-06

{schemathesis-3.26.2 → schemathesis-3.27.0}/docs/cli.rst

@@ -398,39 +398,100 @@ This command will create an XML at a given path, as in the example below.
 .. code:: xml
 
     <?xml version="1.0" ?>
-    <testsuites disabled="0" errors="1" failures="1" tests="3" time="0.10743043999536894">
-        <testsuite disabled="0" errors="1" failures="1" name="schemathesis" skipped="0" tests="3" time="0.10743043999536894" hostname="bespin">
-            <testcase name="GET /api/failure" time="0.089057">
-                <failure type="failure" message="2. Received a response with 5xx status code: 500"/>
-            </testcase>
-            <testcase name="GET /api/malformed_json" time="0.011977">
-                <error type="error" message="json.decoder.JSONDecodeError: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)
-    ">Traceback (most recent call last):
-      File &quot;/home/user/work/schemathesis/src/schemathesis/runner/impl/core.py&quot;, line 87, in run_test
-        test(checks, targets, result, **kwargs)
-      File &quot;/home/user/work/schemathesis/src/schemathesis/runner/impl/core.py&quot;, line 150, in network_test
-        case: Case,
-      File &quot;/home/user/.pyenv/versions/3.8.0/envs/schemathesis/lib/python3.8/site-packages/hypothesis/core.py&quot;, line 1095, in wrapped_test
-        raise the_error_hypothesis_found
-      File &quot;/home/user/work/schemathesis/src/schemathesis/runner/impl/core.py&quot;, line 165, in network_test
-        run_checks(case, checks, result, response)
-      File &quot;/home/user/work/schemathesis/src/schemathesis/runner/impl/core.py&quot;, line 133, in run_checks
-        check(response, case)
-      File &quot;/home/user/work/schemathesis/src/schemathesis/checks.py&quot;, line 87, in response_schema_conformance
-        data = response.json()
-      File &quot;/home/user/.pyenv/versions/3.8.0/envs/schemathesis/lib/python3.8/site-packages/requests/models.py&quot;, line 889, in json
-        return complexjson.loads(
-      File &quot;/home/user/.pyenv/versions/3.8.0/lib/python3.8/json/__init__.py&quot;, line 357, in loads
-        return _default_decoder.decode(s)
-      File &quot;/home/user/.pyenv/versions/3.8.0/lib/python3.8/json/decoder.py&quot;, line 337, in decode
-        obj, end = self.raw_decode(s, idx=_w(s, 0).end())
-      File &quot;/home/user/.pyenv/versions/3.8.0/lib/python3.8/json/decoder.py&quot;, line 353, in raw_decode
-        obj, end = self.scan_once(s, idx)
-    json.decoder.JSONDecodeError: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)
-    </error>
-            </testcase>
-            <testcase name="GET /api/success" time="0.006397"/>
-        </testsuite>
+    <testsuites disabled="0" errors="0" failures="4" tests="4" time="1.7481054730014876">
+            <testsuite disabled="0" errors="0" failures="4" name="schemathesis" skipped="0" tests="4" time="1.7481054730014876" hostname="midgard">
+                    <testcase name="GET /response-conformance/missing-field" time="0.859204">
+                            <failure type="failure" message="1. Test Case ID: JA63GZ
+
+    - Response violates schema
+
+        'age' is a required property
+
+        Schema:
+
+            {
+                &quot;type&quot;: &quot;object&quot;,
+                &quot;properties&quot;: {
+                    &quot;id&quot;: {
+                        &quot;type&quot;: &quot;string&quot;
+                    },
+                    &quot;name&quot;: {
+                        &quot;type&quot;: &quot;string&quot;
+                    },
+                    &quot;age&quot;: {
+                        &quot;type&quot;: &quot;integer&quot;
+                    }
+                },
+                &quot;required&quot;: [
+                    &quot;id&quot;,
+                    &quot;name&quot;,
+                    &quot;age&quot;
+                ]
+            }
+
+        Value:
+
+            {
+                &quot;id&quot;: &quot;123&quot;,
+                &quot;name&quot;: &quot;Alice&quot;
+            }
+
+    [200] OK:
+
+        `{&quot;id&quot;:&quot;123&quot;,&quot;name&quot;:&quot;Alice&quot;}`
+
+    Reproduce with:
+
+        curl -X GET https://example.schemathesis.io/response-conformance/missing-field"/>
+                    </testcase>
+                    <testcase name="GET /response-conformance/malformed-json" time="0.068179">
+                            <failure type="failure" message="1. Test Case ID: Vn5hfI
+
+    - JSON deserialization error
+
+        Expecting property name enclosed in double quotes: line 1 column 2 (char 1)
+
+    [200] OK:
+
+        `{success: true}`
+
+    Reproduce with:
+
+        curl -X GET https://example.schemathesis.io/response-conformance/malformed-json"/>
+                    </testcase>
+                    <testcase name="GET /response-conformance/undocumented-status-code" time="0.756355">
+                            <failure type="failure" message="1. Test Case ID: jm2nOs
+
+    - Undocumented HTTP status code
+
+        Received: 404
+        Documented: 200, 400
+
+    [404] Not Found:
+
+        `{&quot;error&quot;:&quot;Not Found&quot;}`
+
+    Reproduce with:
+
+        curl -X GET 'https://example.schemathesis.io/response-conformance/undocumented-status-code?id=1'"/>
+                    </testcase>
+                    <testcase name="GET /response-conformance/incorrect-content-type" time="0.064367">
+                            <failure type="failure" message="1. Test Case ID: Sveexo
+
+    - Undocumented Content-Type
+
+        Received: text/plain
+        Documented: application/json
+
+    [200] OK:
+
+        `Success!`
+
+    Reproduce with:
+
+        curl -X GET https://example.schemathesis.io/response-conformance/incorrect-content-type"/>
+                    </testcase>
+            </testsuite>
     </testsuites>
 
 Base URL configuration

{schemathesis-3.26.2 → schemathesis-3.27.0}/docs/extending.rst

@@ -558,7 +558,7 @@ For the WSGI case, it will be ``schemathesis.utils.WSGIResponse``.
 ``process_call_kwargs``
 ~~~~~~~~~~~~~~~~~~~~~~~
 
-If you want to modify what keyword arguments will be given to ``case.call`` / ``case.call_wsgi`` / ``case.call_asgi`` in CLI, then you can use this hook:
+If you want to modify what keyword arguments will be given to ``case.call`` in CLI, then you can use this hook:
 
 .. code:: python
 

{schemathesis-3.26.2 → schemathesis-3.27.0}/docs/python.rst

@@ -226,7 +226,7 @@ If you maintain your API schema in Python code or your web framework (for exampl
 Web applications
 ~~~~~~~~~~~~~~~~
 
-Schemathesis natively supports testing of ASGI and WSGI compatible apps (e.g., Flask or FastAPI),
+Schemathesis natively supports testing of ASGI and WSGI compatible apps (e.g., FastAPI or Flask),
 which is significantly faster since it doesn't involve the network.
 
 .. code:: python
@@ -272,7 +272,7 @@ This approach requires an initialized application instance to generate the API s
             yield
             await db.disconnect()
 
-        return schemathesis.from_dict(app.openapi())
+        return schemathesis.from_dict(app.openapi(), app)
 
 
     schema = schemathesis.from_pytest_fixture("web_app")
@@ -300,7 +300,7 @@ When the received response is validated, Schemathesis runs the following checks:
 - ``response_schema_conformance``. The response content does not conform to the schema defined for this specific response;
 - ``response_headers_conformance``. The response headers does not contain all defined headers.
 
-Validation happens in the ``case.validate_response`` function, but you can add your code to verify the response conformance as you do in regular Python tests.
+Validation happens in the ``case.call_and_validate`` function, but you can add your code to verify the response conformance as you do in regular Python tests.
 By default, all available checks will be applied, but you can customize it by passing a tuple of checks explicitly:
 
 .. code-block:: python
@@ -312,8 +312,7 @@ By default, all available checks will be applied, but you can customize it by pa
 
     @schema.parametrize()
     def test_api(case):
-        response = case.call()
-        case.validate_response(response, checks=(not_a_server_error,))
+        case.call_and_validate(checks=(not_a_server_error,))
 
 The code above will run only the ``not_a_server_error`` check. Or a tuple of additional checks will be executed after ones from the ``checks`` argument:
 
@@ -328,8 +327,7 @@ The code above will run only the ``not_a_server_error`` check. Or a tuple of add
 
     @schema.parametrize()
     def test_api(case):
-        response = case.call()
-        case.validate_response(response, additional_checks=(my_check,))
+        case.call_and_validate(additional_checks=(my_check,))
 
 .. note::
 
@@ -346,8 +344,7 @@ You can also use the ``excluded_checks`` argument to exclude chhecks from runnin
 
     @schema.parametrize()
     def test_api(case):
-        response = case.call()
-        case.validate_response(response, excluded_checks=(not_a_server_error,))
+        case.call_and_validate(excluded_checks=(not_a_server_error,))
 
 The code above will run the default checks, and any additional checks, excluding the ``not_a_server_error`` check.
 
@@ -398,8 +395,7 @@ In the following example we test a hypothetical ``/api/auth/password/reset/`` op
             case.body["token"] = data.draw(
                 (st.emails() | st.just(user.email)).map(create_reset_password_token)
             )
-        response = case.call_asgi(app=app)
-        case.validate_response(response)
+        case.call_and_validate()
 
 Here we use the special `data strategy <https://hypothesis.readthedocs.io/en/latest/data.html#drawing-interactively-in-tests>`_ to change the ``case`` data in ~50% cases.
 The additional strategy in the conditional branch creates a valid password reset token from the given email.
@@ -449,11 +445,13 @@ This approach ensures that both types of tests can be executed, albeit in separa
     def test_explicit_examples(data, case, user):
         ...
 
-ASGI / WSGI support
+ASGI & WSGI support
 -------------------
 
-Schemathesis supports making calls to ASGI and WSGI-compliant applications instead of real network calls;
-in this case, the test execution will go much faster.
+Schemathesis supports making calls to `ASGI <https://asgi.readthedocs.io/en/latest/>`_ and `WSGI-compliant <https://docs.python.org/3/library/wsgiref.html>`_ applications instead of through real network calls,
+significantly speeding up test execution.
+
+Using Schemathesis with a Flask application (WSGI):
 
 .. code:: python
 
@@ -470,34 +468,50 @@ in this case, the test execution will go much faster.
 
     @app.route("/v1/users", methods=["GET"])
     def users():
-        return jsonify([{"name": "Robin"}])
+        return [{"name": "Robin"}]
 
 
+    # Load the schema from the WSGI app
     schema = schemathesis.from_wsgi("/schema.json", app)
 
 
     @schema.parametrize()
     def test_api(case):
-        response = case.call_wsgi()
-        case.validate_response(response)
+        # The test case will make a call to the application and validate the response
+        # against the defined schema automatically.
+        case.call_and_validate()
 
-If you don't supply the ``app`` argument to the loader, make sure you pass your test client when running tests:
+Running the example above with ``pytest`` will execute property-based tests against the Flask application.
 
-.. code-block:: python
+Using Schemathesis with a FastAPI application (ASGI):
 
-    @pytest.fixture()
-    def app_schema(client):
-        openapi = client.app.openapi()
-        return schemathesis.from_dict(openapi)
+.. code:: python
+
+    from fastapi import FastAPI
+    import schemathesis
 
+    # Enable the experimental Open API 3.1 support
+    schemathesis.experimental.OPEN_API_3_1.enable()
 
-    schema = schemathesis.from_pytest_fixture("app_schema")
+    app = FastAPI()
+
+
+    @app.get("/v1/users")
+    async def users():
+        return [{"name": "Robin"}]
+
+
+    # Load the schema from the ASGI app
+    schema = schemathesis.from_asgi("/openapi.json", app)
 
 
     @schema.parametrize()
-    def test_api(case, client):
-        # The `session` argument must be supplied.
-        case.call_and_validate(session=client)
+    def test_api(case):
+        # The test case will make a call to the application and validate the response
+        # against the defined schema automatically.
+        case.call_and_validate()
+
+Note that Schemathesis currently tests ASGI applications synchronously.
 
 Async support
 -------------

{schemathesis-3.26.2 → schemathesis-3.27.0}/example/requirements-pytest.txt

@@ -43,7 +43,7 @@ hypothesis-graphql==0.11.0
     # via schemathesis
 hypothesis-jsonschema==0.22.1
     # via schemathesis
-idna==3.6
+idna==3.7
     # via
     #   anyio
     #   httpx

{schemathesis-3.26.2 → schemathesis-3.27.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "schemathesis"
-version = "3.26.2"
+version = "3.27.0"
 description = "Property-based testing framework for Open API and GraphQL based apps"
 keywords = ["pytest", "hypothesis", "openapi", "graphql", "testing"]
 authors = [{ name = "Dmitry Dygalo", email = "dmitry@dygalo.dev" }]

{schemathesis-3.26.2 → schemathesis-3.27.0}/src/schemathesis/cli/__init__.py

@@ -1769,6 +1769,6 @@ def after_init_cli_run_handlers(
 def process_call_kwargs(context: HookContext, case: Case, kwargs: dict[str, Any]) -> None:
     """Called before every network call in CLI tests.
 
-    Aims to modify the argument passed to `case.call` / `case.call_wsgi` / `case.call_asgi`.
+    Aims to modify the argument passed to `case.call`.
     Note that you need to modify `kwargs` in-place.
     """

schemathesis-3.27.0/src/schemathesis/cli/junitxml.py

@@ -0,0 +1,112 @@
+from __future__ import annotations
+
+import base64
+import platform
+import textwrap
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+from junit_xml import TestCase, TestSuite, to_xml_report_file
+
+from ..models import Status
+from ..runner import events
+from ..runner.serialization import SerializedCheck, SerializedError
+from ..exceptions import prepare_response_payload, RuntimeErrorType
+from .handlers import EventHandler
+from .reporting import group_by_case, TEST_CASE_ID_TITLE, split_traceback, get_runtime_error_suggestion
+
+if TYPE_CHECKING:
+    from click.utils import LazyFile
+
+    from .context import ExecutionContext
+
+
+@dataclass
+class JunitXMLHandler(EventHandler):
+    file_handle: LazyFile
+    test_cases: list = field(default_factory=list)
+
+    def handle_event(self, context: ExecutionContext, event: events.ExecutionEvent) -> None:
+        if isinstance(event, events.AfterExecution):
+            test_case = TestCase(
+                f"{event.result.method} {event.result.path}",
+                elapsed_sec=event.elapsed_time,
+                allow_multiple_subelements=True,
+            )
+            if event.status == Status.failure:
+                for idx, (code_sample, group) in enumerate(
+                    group_by_case(event.result.checks, context.code_sample_style), 1
+                ):
+                    checks = sorted(group, key=lambda c: c.name != "not_a_server_error")
+                    test_case.add_failure_info(message=build_failure_message(idx, code_sample, checks))
+            elif event.status == Status.error:
+                test_case.add_error_info(message=build_error_message(context, event.result.errors[-1]))
+            elif event.status == Status.skip:
+                test_case.add_skipped_info(message=event.result.skip_reason)
+            self.test_cases.append(test_case)
+        if isinstance(event, events.Finished):
+            test_suites = [TestSuite("schemathesis", test_cases=self.test_cases, hostname=platform.node())]
+            to_xml_report_file(file_descriptor=self.file_handle, test_suites=test_suites, prettyprint=True)
+
+
+def build_failure_message(idx: int, code_sample: str, checks: list[SerializedCheck]) -> str:
+    from ..transports.responses import get_reason
+
+    message = ""
+    for check_idx, check in enumerate(checks):
+        if check_idx == 0:
+            message += f"{idx}. {TEST_CASE_ID_TITLE}: {check.example.id}\n"
+        message += f"\n- {check.title}\n"
+        formatted_message = check.formatted_message
+        if formatted_message:
+            message += f"\n{formatted_message}\n"
+        if check_idx + 1 == len(checks):
+            if check.response is not None:
+                status_code = check.response.status_code
+                reason = get_reason(status_code)
+                message += f"\n[{check.response.status_code}] {reason}:\n"
+                response_body = check.response.body
+                if response_body is not None:
+                    if not response_body:
+                        message += "\n    <EMPTY>\n"
+                    else:
+                        encoding = check.response.encoding or "utf8"
+                        try:
+                            payload = base64.b64decode(response_body).decode(encoding)
+                            payload = prepare_response_payload(payload)
+                            payload = textwrap.indent(f"\n`{payload}`\n", prefix="    ")
+                            message += payload
+                        except UnicodeDecodeError:
+                            message += "\n    <BINARY>\n"
+
+    message += f"\nReproduce with: \n\n    {code_sample}"
+    return message
+
+
+def build_error_message(context: ExecutionContext, error: SerializedError) -> str:
+    message = ""
+    if error.title:
+        if error.type == RuntimeErrorType.SCHEMA_GENERIC:
+            message = "Schema Error\n"
+        else:
+            message = f"{error.title}\n"
+        if error.message:
+            message += f"\n{error.message}\n"
+    elif error.message:
+        message = error.message
+    else:
+        message = error.exception
+    if error.extras:
+        extras = error.extras
+    elif context.show_trace and error.type.has_useful_traceback:
+        extras = split_traceback(error.exception_with_traceback)
+    else:
+        extras = []
+    if extras:
+        message += "\n"
+    for extra in extras:
+        message += f"    {extra}\n"
+    suggestion = get_runtime_error_suggestion(error.type, bold=str)
+    if suggestion is not None:
+        message += f"\nTip: {suggestion}"
+    return message