schemathesis 4.0.16__tar.gz → 4.0.17__tar.gz

{schemathesis-4.0.16 → schemathesis-4.0.17}/CHANGELOG.md

@@ -1,12 +1,19 @@
 # Changelog
 
-## [Unreleased](https://github.com/schemathesis/schemathesis/compare/v4.0.16...HEAD) - TBD
+## [Unreleased](https://github.com/schemathesis/schemathesis/compare/v4.0.17...HEAD) - TBD
+
+## [4.0.17](https://github.com/schemathesis/schemathesis/compare/v4.0.16...v4.0.17) - 2025-08-02
+
+### :bug: Fixed
+
+- Generating negative test cases incorrectly marked as invalid if `anyOf` or `oneOf` are present. [#2975](https://github.com/schemathesis/schemathesis/issues/2975)
+- Set correct data generation mode for stateful test cases if after adding stateful data, a negative test case becomes positive. [#2983](https://github.com/schemathesis/schemathesis/issues/2983) 
 
 ## [4.0.16](https://github.com/schemathesis/schemathesis/compare/v4.0.15...v4.0.16) - 2025-07-31
 
 ### :bug: Fixed
 
-- Internal error if the stateful phase is used with `--generation-unique-inputs`. [#2977](https://github.com/schemathesis/schemathesis/issues/2977) 
+- Internal error if the stateful phase is used with `--generation-unique-inputs`. [#2977](https://github.com/schemathesis/schemathesis/issues/2977)
 - Do not show auth headers for cases when the test case was sent specifically without auth.
 - Do not show unnecessary auth warning if the only test that happened was for missing auth.
 

{schemathesis-4.0.16 → schemathesis-4.0.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: schemathesis
-Version: 4.0.16
+Version: 4.0.17
 Summary: Property-based testing framework for Open API and GraphQL based apps
 Project-URL: Documentation, https://schemathesis.readthedocs.io/en/stable/
 Project-URL: Changelog, https://github.com/schemathesis/schemathesis/blob/master/CHANGELOG.md

{schemathesis-4.0.16 → schemathesis-4.0.17}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "schemathesis"
-version = "4.0.16"
+version = "4.0.17"
 description = "Property-based testing framework for Open API and GraphQL based apps"
 keywords = ["pytest", "hypothesis", "openapi", "graphql", "testing"]
 authors = [{ name = "Dmitry Dygalo", email = "dmitry@dygalo.dev" }]

{schemathesis-4.0.16 → schemathesis-4.0.17}/src/schemathesis/generation/coverage.py

@@ -597,12 +597,49 @@ def cover_schema_iter(
                         with _ignore_unfixable():
                             canonical = canonicalish(schema)
                             yield from cover_schema_iter(nctx, canonical, seen)
-                elif key == "anyOf" or key == "oneOf":
+                elif key == "anyOf":
                     nctx = ctx.with_negative()
-                    # NOTE: Other sub-schemas are not filtered out
+                    validators = [jsonschema.validators.validator_for(sub_schema)(sub_schema) for sub_schema in value]
                     for idx, sub_schema in enumerate(value):
                         with nctx.at(idx):
-                            yield from cover_schema_iter(nctx, sub_schema, seen)
+                            for value in cover_schema_iter(nctx, sub_schema, seen):
+                                # Negative value for this schema could be a positive value for another one
+                                if is_valid_for_others(value.value, idx, validators):
+                                    continue
+                                yield value
+                elif key == "oneOf":
+                    nctx = ctx.with_negative()
+                    validators = [jsonschema.validators.validator_for(sub_schema)(sub_schema) for sub_schema in value]
+                    for idx, sub_schema in enumerate(value):
+                        with nctx.at(idx):
+                            for value in cover_schema_iter(nctx, sub_schema, seen):
+                                if is_invalid_for_oneOf(value.value, idx, validators):
+                                    yield value
+
+
+def is_valid_for_others(value: Any, idx: int, validators: list[jsonschema.Validator]) -> bool:
+    for vidx, validator in enumerate(validators):
+        if idx == vidx:
+            # This one is being negated
+            continue
+        if validator.is_valid(value):
+            return True
+    return False
+
+
+def is_invalid_for_oneOf(value: Any, idx: int, validators: list[jsonschema.Validator]) -> bool:
+    valid_count = 0
+    for vidx, validator in enumerate(validators):
+        if idx == vidx:
+            # This one is being negated
+            continue
+        if validator.is_valid(value):
+            valid_count += 1
+            # Should circuit - no need to validate more, it is already invalid
+            if valid_count > 1:
+                return True
+    # No matching at all - we successfully generated invalid value
+    return valid_count == 0
 
 
 def _get_properties(schema: dict | bool) -> dict | bool:

{schemathesis-4.0.16 → schemathesis-4.0.17}/src/schemathesis/specs/openapi/stateful/__init__.py

@@ -4,6 +4,7 @@ from dataclasses import dataclass
 from functools import lru_cache
 from typing import TYPE_CHECKING, Any, Callable, Iterator
 
+import jsonschema
 from hypothesis import strategies as st
 from hypothesis.stateful import Bundle, Rule, precondition, rule
 
@@ -14,6 +15,7 @@ from schemathesis.engine.recorder import ScenarioRecorder
 from schemathesis.generation import GenerationMode
 from schemathesis.generation.case import Case
 from schemathesis.generation.hypothesis import strategies
+from schemathesis.generation.meta import ComponentInfo, ComponentKind
 from schemathesis.generation.stateful import STATEFUL_TESTS_LABEL
 from schemathesis.generation.stateful.state_machine import APIStateMachine, StepInput, StepOutput, _normalize_name
 from schemathesis.schemas import APIOperation
@@ -281,6 +283,15 @@ def into_step_input(
                     case.body = {**case.body, **new}
                 else:
                     case.body = new
+                if case.meta and case.meta.generation.mode == GenerationMode.NEGATIVE:
+                    # It is possible that the new body is now valid and the whole test case could be valid too
+                    for alternative in case.operation.body:
+                        if alternative.media_type == case.media_type:
+                            schema = alternative.as_json_schema(case.operation)
+                            if jsonschema.validators.validator_for(schema)(schema).is_valid(new):
+                                case.meta.components[ComponentKind.BODY] = ComponentInfo(mode=GenerationMode.POSITIVE)
+                                if all(info.mode == GenerationMode.POSITIVE for info in case.meta.components.values()):
+                                    case.meta.generation.mode = GenerationMode.POSITIVE
             return StepInput(case=case, transition=transition)
 
         return inner(output=_output)

{schemathesis-4.0.16 → schemathesis-4.0.17}/test/coverage/test_combinations.py

@@ -48,10 +48,10 @@ def assert_conform(values: list, schema: dict):
 
 
 def assert_not_conform(values: list, schema: dict):
+    if schema.get("format") == "unknown":
+        # Can't validate the format
+        return
     for entry in values:
-        if schema.get("format") == "unknown":
-            # Can't validate the format
-            continue
         try:
             jsonschema.validate(
                 entry,
@@ -59,7 +59,7 @@ def assert_not_conform(values: list, schema: dict):
                 cls=jsonschema.Draft7Validator,
                 format_checker=jsonschema.Draft202012Validator.FORMAT_CHECKER,
             )
-            raise ValueError(f"Value {entry} conforms to {schema}")
+            raise AssertionError(f"Value {entry} conforms to {schema}")
         except (jsonschema.ValidationError, ValueError):
             pass
 
@@ -1048,7 +1048,7 @@ def test_positive_pattern(pctx):
     covered = cover_schema(pctx, schema)
     assert covered == ["0000-0000", "00-0000", "00-00000", "0000-000000000000000", "000-000000000000000"]
     assert_unique(covered)
-    assert_not_conform(covered, schema)
+    assert_conform(covered, schema)
 
 
 def test_negative_pattern_with_incompatible_length(nctx):
@@ -1099,7 +1099,11 @@ def test_negative_pattern_with_incompatible_length(nctx):
                     {"type": "string", "maxLength": 5},
                 ],
             },
-            [4, AnyNumber(), AnyNumber(), "000000"],
+            (
+                [4, AnyNumber(), AnyNumber()],
+                [4, AnyNumber()],
+                [4],
+            ),
         ),
         (
             {
@@ -1116,7 +1120,7 @@ def test_negative_pattern_with_incompatible_length(nctx):
                     },
                 ]
             },
-            [ANY, "00000000000", AnyNumber(), AnyNumber()],
+            ["00000000000", AnyNumber(), AnyNumber()],
         ),
         (
             {
@@ -1137,7 +1141,63 @@ def test_negative_combinators(nctx, schema, expected):
             assert_not_conform(covered, schema)
             break
     else:
-        pytest.fail("Expected value didn't match")
+        pytest.fail(f"Expected value didn't match\nGot: {covered!r}\nExpected: {expected!r}")
+
+
+@pytest.mark.parametrize(
+    ["schema", "expected"],
+    [
+        (
+            {
+                "anyOf": [
+                    {"type": "number"},
+                    {"type": "null"},
+                ]
+            },
+            [
+                False,
+                "",
+                [
+                    None,
+                    None,
+                ],
+                {},
+            ],
+        ),
+        (
+            {
+                "oneOf": [
+                    {"type": "number"},
+                    {"type": "integer"},
+                    {"type": "null"},
+                ]
+            },
+            [
+                False,
+                "",
+                [
+                    None,
+                    None,
+                ],
+                {},
+                # Matching both, "number" and "integer", hence invalid
+                0,
+            ],
+        ),
+    ],
+)
+def test_negative_one_of(schema, expected):
+    # See GH-2975
+    nctx = CoverageContext(
+        location="body",
+        generation_modes=[GenerationMode.NEGATIVE],
+        is_required=True,
+        custom_formats=get_default_format_strategies(),
+        validator_cls=jsonschema.validators.Draft202012Validator,
+    )
+    covered = cover_schema(nctx, schema)
+    assert_not_conform(covered, schema)
+    assert covered == expected
 
 
 @pytest.mark.parametrize(

{schemathesis-4.0.16 → schemathesis-4.0.17}/test/specs/openapi/stateful/test_engine.py

@@ -4,6 +4,7 @@ from dataclasses import dataclass
 import hypothesis
 import hypothesis.errors
 import pytest
+from flask import Flask, jsonify, request
 
 import schemathesis
 from schemathesis.checks import not_a_server_error
@@ -11,7 +12,11 @@ from schemathesis.engine import Status, events
 from schemathesis.engine.context import EngineContext
 from schemathesis.engine.phases import Phase, PhaseName, stateful
 from schemathesis.generation import GenerationMode
-from schemathesis.specs.openapi.checks import ignored_auth, response_schema_conformance, use_after_free
+from schemathesis.specs.openapi.checks import (
+    ignored_auth,
+    response_schema_conformance,
+    use_after_free,
+)
 from test.utils import flaky
 
 
@@ -563,6 +568,110 @@ def test_negative_tests(engine_factory):
         pytest.fail(str(result.events))
 
 
+def test_negative_changing_to_positive(app_runner):
+    # See GH-2983
+    schema = {
+        "openapi": "3.1.0",
+        "paths": {
+            "/Customers/{id}": {
+                "put": {
+                    "operationId": "UpdateCustomer",
+                    "parameters": [
+                        {"name": "id", "in": "path", "schema": {"type": "integer"}},
+                    ],
+                    "requestBody": {
+                        "content": {"application/json": {"schema": {"$ref": "#/components/schemas/CustomerUpdate"}}}
+                    },
+                    "responses": {"default": {"description": "Ok"}},
+                }
+            },
+            "/Customers": {
+                "post": {
+                    "requestBody": {
+                        "content": {
+                            "application/json": {
+                                "schema": {"$ref": "#/components/schemas/CustomerCreate"},
+                            }
+                        }
+                    },
+                    "responses": {
+                        "201": {
+                            "links": {
+                                "Customer": {"$ref": "#/components/links/updateCustomer"},
+                            }
+                        }
+                    },
+                }
+            },
+        },
+        "components": {
+            "links": {
+                "updateCustomer": {
+                    "operationId": "UpdateCustomer",
+                    "parameters": {"id": "$response.body"},
+                    "requestBody": {"name": "firstname"},
+                }
+            },
+            "schemas": {
+                "CustomerUpdate": {"required": ["name"]},
+                "CustomerCreate": {"type": "object", "properties": {"name": {"type": "string"}}},
+            },
+        },
+    }
+
+    app = Flask(__name__)
+
+    CUSTOMERS = {}
+    NEXT_ID = 1
+
+    @app.route("/Customers/<int:id>", methods=["PUT"])
+    def update_customer(id):
+        if id not in CUSTOMERS:
+            return "", 404
+        data = request.get_json(force=True, silent=True)
+        if not data:
+            return jsonify({"error": ["Invalid JSON"]}), 400
+        CUSTOMERS[id].update(data)
+        return "", 204
+
+    @app.route("/Customers", methods=["POST"])
+    def create_customer():
+        nonlocal NEXT_ID
+        data = request.get_json(force=True, silent=True)
+        if not isinstance(data, dict):
+            return jsonify({"error": ["Invalid JSON"]}), 400
+
+        name = data.get("name")
+        if not (isinstance(name, str) and 1 <= len(name) <= 3):
+            return jsonify({"error": ["Invalid name. Must be 1 to 3 characters."]}), 400
+
+        customer_id = NEXT_ID
+        NEXT_ID += 1
+        data["id"] = customer_id
+        CUSTOMERS[customer_id] = data
+        return jsonify(customer_id), 201
+
+    app_port = app_runner.run_flask_app(app)
+
+    config = schemathesis.Config.from_dict(
+        {
+            "checks": {
+                "enabled": False,
+                "negative_data_rejection": {"enabled": True},
+            }
+        }
+    )
+    schema = schemathesis.openapi.from_dict(schema, config=config)
+    schema.config.update(base_url=f"http://127.0.0.1:{app_port}/")
+    schema.config.generation.update(database="none")
+    engine = stateful.execute(
+        engine=EngineContext(schema=schema, stop_event=threading.Event()),
+        phase=Phase(name=PhaseName.STATEFUL_TESTING, is_supported=True, is_enabled=True),
+    )
+    result = collect_result(engine)
+    assert result.events[-1].status == Status.SUCCESS
+
+
 def test_unique_inputs(engine_factory):
     engine = engine_factory(
         app_kwargs={"independent_500": True},