schemathesis 3.31.0__tar.gz → 3.31.1__tar.gz

{schemathesis-3.31.0 → schemathesis-3.31.1}/.github/workflows/codeql-analysis.yml

@@ -38,7 +38,7 @@ jobs:
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3.25.10
+        uses: github/codeql-action/init@v3.25.11
         with:
           languages: ${{ matrix.language }}
           # If you wish to specify custom queries, you can do so here or in a config file.
@@ -49,7 +49,7 @@ jobs:
       # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
       # If this step fails, then you should remove it and run the build manually (see below)
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v3.25.10
+        uses: github/codeql-action/autobuild@v3.25.11
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 https://git.io/JvXDl
@@ -63,4 +63,4 @@ jobs:
       #   make release
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3.25.10
+        uses: github/codeql-action/analyze@v3.25.11

{schemathesis-3.31.0 → schemathesis-3.31.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: schemathesis
-Version: 3.31.0
+Version: 3.31.1
 Summary: Property-based testing framework for Open API and GraphQL based apps
 Project-URL: Documentation, https://schemathesis.readthedocs.io/en/stable/
 Project-URL: Changelog, https://schemathesis.readthedocs.io/en/stable/changelog.html

{schemathesis-3.31.0 → schemathesis-3.31.1}/docs/changelog.rst

@@ -1,9 +1,22 @@
 Changelog
 =========
 
-:version:`Unreleased <v3.31.0...HEAD>` - TBD
+:version:`Unreleased <v3.31.1...HEAD>` - TBD
 --------------------------------------------
 
+.. _v3.31.1:
+
+:version:`3.31.1 <v3.31.0...v3.31.1>` - 2024-07-03
+--------------------------------------------------
+
+**Fixed**
+
+- Generating negative test cases for path and query parameters. :issue:`2312`
+
+**Changed**
+
+- Do not consider ignoring additional parameters as a failure in ``negative_data_rejection``.
+
 .. _v3.31.0:
 
 :version:`3.31.0 <v3.30.4...v3.31.0>` - 2024-06-30

{schemathesis-3.31.0 → schemathesis-3.31.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "schemathesis"
-version = "3.31.0"
+version = "3.31.1"
 description = "Property-based testing framework for Open API and GraphQL based apps"
 keywords = ["pytest", "hypothesis", "openapi", "graphql", "testing"]
 authors = [{ name = "Dmitry Dygalo", email = "dmitry@dygalo.dev" }]

{schemathesis-3.31.0 → schemathesis-3.31.1}/src/schemathesis/models.py

@@ -119,6 +119,19 @@ def prepare_request_data(kwargs: dict[str, Any]) -> PreparedRequestData:
     )
 
 
+@dataclass
+class GenerationMetadata:
+    """Stores various information about how data is generated."""
+
+    query: DataGenerationMethod | None
+    path_parameters: DataGenerationMethod | None
+    headers: DataGenerationMethod | None
+    cookies: DataGenerationMethod | None
+    body: DataGenerationMethod | None
+
+    __slots__ = ("query", "path_parameters", "headers", "cookies", "body")
+
+
 @dataclass(repr=False)
 class Case:
     """A single test case parameters."""
@@ -139,6 +152,8 @@ class Case:
     media_type: str | None = None
     source: CaseSource | None = None
 
+    meta: GenerationMetadata | None = None
+
     # The way the case was generated (None for manually crafted ones)
     data_generation_method: DataGenerationMethod | None = None
     _auth: requests.auth.AuthBase | None = None

{schemathesis-3.31.0 → schemathesis-3.31.1}/src/schemathesis/specs/openapi/_hypothesis.py

@@ -25,7 +25,7 @@ from ...generation import DataGenerationMethod, GenerationConfig
 from ...hooks import HookContext, HookDispatcher, apply_to_all_dispatchers
 from ...internal.copy import fast_deepcopy
 from ...internal.validation import is_illegal_surrogate
-from ...models import APIOperation, Case, cant_serialize
+from ...models import APIOperation, Case, GenerationMetadata, cant_serialize
 from ...serializers import Binary
 from ...transports.content_types import parse_content_type
 from ...transports.headers import has_invalid_characters, is_latin_1_encodable
@@ -36,7 +36,7 @@ from .formats import STRING_FORMATS
 from .media_types import MEDIA_TYPES
 from .negative import negative_schema
 from .negative.utils import can_negate
-from .parameters import OpenAPIBody, parameters_to_json_schema
+from .parameters import OpenAPIBody, OpenAPIParameter, parameters_to_json_schema
 from .utils import is_header_location
 
 HEADER_FORMAT = "_header_value"
@@ -207,6 +207,13 @@ def get_case_strategy(
         query=query_.value,
         body=body_.value,
         data_generation_method=generator,
+        meta=GenerationMetadata(
+            query=query_.generator,
+            path_parameters=path_parameters_.generator,
+            headers=headers_.generator,
+            cookies=cookies_.generator,
+            body=body_.generator,
+        ),
     )
     auth_context = auths.AuthContext(
         operation=operation,
@@ -347,6 +354,22 @@ def can_negate_headers(operation: APIOperation, location: str) -> bool:
     return any(header != {"type": "string"} for header in headers.values())
 
 
+def get_schema_for_location(
+    operation: APIOperation, location: str, parameters: Iterable[OpenAPIParameter]
+) -> dict[str, Any]:
+    schema = parameters_to_json_schema(operation, parameters)
+    if location == "path":
+        if not operation.schema.validate_schema:
+            # If schema validation is disabled, we try to generate data even if the parameter definition
+            # contains errors.
+            # In this case, we know that the `required` keyword should always be `True`.
+            schema["required"] = list(schema["properties"])
+        for prop in schema.get("properties", {}).values():
+            if prop.get("type") == "string":
+                prop.setdefault("minLength", 1)
+    return operation.schema.prepare_schema(schema)
+
+
 def get_parameters_strategy(
     operation: APIOperation,
     strategy_factory: StrategyFactory,
@@ -361,17 +384,7 @@ def get_parameters_strategy(
         nested_cache_key = (strategy_factory, location, tuple(sorted(exclude)))
         if operation in _PARAMETER_STRATEGIES_CACHE and nested_cache_key in _PARAMETER_STRATEGIES_CACHE[operation]:
             return _PARAMETER_STRATEGIES_CACHE[operation][nested_cache_key]
-        schema = parameters_to_json_schema(operation, parameters)
-        if location == "path":
-            if not operation.schema.validate_schema:
-                # If schema validation is disabled, we try to generate data even if the parameter definition
-                # contains errors.
-                # In this case, we know that the `required` keyword should always be `True`.
-                schema["required"] = list(schema["properties"])
-            for prop in schema.get("properties", {}).values():
-                if prop.get("type") == "string":
-                    prop.setdefault("minLength", 1)
-        schema = operation.schema.prepare_schema(schema)
+        schema = get_schema_for_location(operation, location, parameters)
         for name in exclude:
             # Values from `exclude` are not necessarily valid for the schema - they come from user-defined examples
             # that may be invalid

{schemathesis-3.31.0 → schemathesis-3.31.1}/src/schemathesis/specs/openapi/checks.py

@@ -145,7 +145,12 @@ def negative_data_rejection(response: GenericResponse, case: Case) -> bool | Non
 
     if not isinstance(case.operation.schema, BaseOpenAPISchema):
         return True
-    if case.data_generation_method and case.data_generation_method.is_negative and 200 <= response.status_code < 300:
+    if (
+        case.data_generation_method
+        and case.data_generation_method.is_negative
+        and 200 <= response.status_code < 300
+        and not has_only_additional_properties_in_non_body_parameters(case)
+    ):
         exc_class = get_negative_rejection_error(case.operation.verbose_name, response.status_code)
         raise exc_class(
             failures.AcceptedNegativeData.title,
@@ -154,6 +159,34 @@ def negative_data_rejection(response: GenericResponse, case: Case) -> bool | Non
     return None
 
 
+def has_only_additional_properties_in_non_body_parameters(case: Case) -> bool:
+    # Check if the case contains only additional properties in query, headers, or cookies.
+    # This function is used to determine if negation is solely in the form of extra properties,
+    # which are often ignored for backward-compatibility by the tested apps
+    from ._hypothesis import get_schema_for_location
+
+    meta = case.meta
+    if meta is None:
+        # Ignore manually created cases
+        return False
+    if (meta.body and meta.body.is_negative) or (meta.path_parameters and meta.path_parameters.is_negative):
+        # Body or path negations always imply other negations
+        return False
+    validator_cls = case.operation.schema.validator_cls  # type: ignore[attr-defined]
+    for container in ("query", "headers", "cookies"):
+        meta_for_location = getattr(meta, container)
+        value = getattr(case, container)
+        if value is not None and meta_for_location is not None and meta_for_location.is_negative:
+            parameters = getattr(case.operation, container)
+            value_without_additional_properties = {k: v for k, v in value.items() if k in parameters}
+            schema = get_schema_for_location(case.operation, container, parameters)
+            if not validator_cls(schema).is_valid(value_without_additional_properties):
+                # Other types of negation found
+                return False
+    # Only additional properties are added
+    return True
+
+
 def use_after_free(response: GenericResponse, original: Case) -> bool | None:
     from ...transports.responses import get_reason
     from .schemas import BaseOpenAPISchema

{schemathesis-3.31.0 → schemathesis-3.31.1}/src/schemathesis/specs/openapi/negative/mutations.py

@@ -81,6 +81,10 @@ class MutationContext:
     def is_path_location(self) -> bool:
         return self.location == "path"
 
+    @property
+    def is_query_location(self) -> bool:
+        return self.location == "query"
+
     def mutate(self, draw: Draw) -> Schema:
         # On the top level, Schemathesis creates "object" schemas for all parameter "in" values except "body", which is
         # taken as-is. Therefore, we can only apply mutations that won't change the Open API semantics of the schema.
@@ -203,8 +207,11 @@ def change_type(context: MutationContext, draw: Draw, schema: Schema) -> Mutatio
     if context.media_type == "application/x-www-form-urlencoded":
         # Form data should be an object, do not change it
         return MutationResult.FAILURE
-    # Headers are always strings, can't negate this
-    if context.is_header_location:
+    # For headers, query and path parameters, if the current type is string, then it already
+    # includes all possible values as those parameters will be stringified before sending,
+    # therefore it can't be negated.
+    types = get_type(schema)
+    if "string" in types and (context.is_header_location or context.is_path_location or context.is_query_location):
         return MutationResult.FAILURE
     candidates = _get_type_candidates(context, schema)
     if not candidates:

{schemathesis-3.31.0 → schemathesis-3.31.1}/src/schemathesis/specs/openapi/schemas.py

@@ -19,6 +19,7 @@ from typing import (
     Mapping,
     NoReturn,
     Sequence,
+    Type,
     TypeVar,
     cast,
 )
@@ -615,6 +616,12 @@ class BaseOpenAPISchema(BaseSchema):
     def get_tags(self, operation: APIOperation) -> list[str] | None:
         return operation.definition.raw.get("tags")
 
+    @property
+    def validator_cls(self) -> Type[jsonschema.Validator]:
+        if self.spec_version.startswith("3.1") and experimental.OPEN_API_3_1.is_enabled:
+            return jsonschema.Draft202012Validator
+        return jsonschema.Draft4Validator
+
     def validate_response(self, operation: APIOperation, response: GenericResponse) -> bool | None:
         responses = {str(key): value for key, value in operation.definition.raw.get("responses", {}).items()}
         status_code = str(response.status_code)
@@ -658,13 +665,9 @@ class BaseOpenAPISchema(BaseSchema):
         resolver = ConvertingResolver(
             self.location or "", self.raw_schema, nullable_name=self.nullable_name, is_response_schema=True
         )
-        if self.spec_version.startswith("3.1") and experimental.OPEN_API_3_1.is_enabled:
-            cls = jsonschema.Draft202012Validator
-        else:
-            cls = jsonschema.Draft4Validator
         with in_scopes(resolver, scopes):
             try:
-                jsonschema.validate(data, schema, cls=cls, resolver=resolver)
+                jsonschema.validate(data, schema, cls=self.validator_cls, resolver=resolver)
             except jsonschema.ValidationError as exc:
                 exc_class = get_schema_validation_error(operation.verbose_name, exc)
                 ctx = failures.ValidationErrorContext.from_exception(exc, output_config=operation.schema.output_config)

{schemathesis-3.31.0 → schemathesis-3.31.1}/test/specs/openapi/test_checks.py

@@ -1,6 +1,14 @@
 import pytest
 
-from schemathesis.specs.openapi.checks import ResourcePath, _is_prefix_operation
+import schemathesis
+from schemathesis.generation import DataGenerationMethod
+from schemathesis.models import Case, GenerationMetadata
+from schemathesis.specs.openapi.checks import (
+    ResourcePath,
+    _is_prefix_operation,
+    has_only_additional_properties_in_non_body_parameters,
+    negative_data_rejection,
+)
 
 
 @pytest.mark.parametrize(
@@ -64,3 +72,92 @@ from schemathesis.specs.openapi.checks import ResourcePath, _is_prefix_operation
 )
 def test_is_prefix_operation(lhs, lhs_vars, rhs, rhs_vars, expected):
     assert _is_prefix_operation(ResourcePath(lhs, lhs_vars), ResourcePath(rhs, rhs_vars)) == expected
+
+
+def build_metadata(path_parameters=None, query=None, headers=None, cookies=None, body=None):
+    return GenerationMetadata(path_parameters=path_parameters, query=query, headers=headers, cookies=cookies, body=body)
+
+
+@pytest.fixture
+def sample_schema(empty_open_api_3_schema):
+    empty_open_api_3_schema["paths"] = {
+        "/test": {
+            "post": {
+                "parameters": [
+                    {
+                        "in": "query",
+                        "name": "key",
+                        "schema": {"type": "integer", "minimum": 5},
+                    },
+                    {
+                        "in": "headers",
+                        "name": "X-Key",
+                        "schema": {"type": "integer", "minimum": 5},
+                    },
+                ]
+            }
+        }
+    }
+    return empty_open_api_3_schema
+
+
+@pytest.mark.parametrize(
+    "kwargs, expected",
+    (
+        ({}, False),
+        (
+            {"meta": build_metadata(body=DataGenerationMethod.negative)},
+            False,
+        ),
+        (
+            {
+                "query": {"key": 1},
+                "meta": build_metadata(query=DataGenerationMethod.negative),
+            },
+            False,
+        ),
+        (
+            {
+                "query": {"key": 1},
+                "headers": {"X-Key": 42},
+                "meta": build_metadata(query=DataGenerationMethod.negative),
+            },
+            False,
+        ),
+        (
+            {
+                "query": {"key": 5, "unknown": 3},
+                "meta": build_metadata(query=DataGenerationMethod.negative),
+            },
+            True,
+        ),
+        (
+            {
+                "query": {"key": 5, "unknown": 3},
+                "headers": {"X-Key": 42},
+                "meta": build_metadata(query=DataGenerationMethod.negative),
+            },
+            True,
+        ),
+    ),
+)
+def test_has_only_additional_properties_in_non_body_parameters(sample_schema, kwargs, expected):
+    schema = schemathesis.from_dict(sample_schema)
+    operation = schema["/test"]["POST"]
+    case = Case(operation=operation, generation_time=0.0, **kwargs)
+    assert has_only_additional_properties_in_non_body_parameters(case) is expected
+
+
+def test_negative_data_rejection_on_additional_properties(response_factory, sample_schema):
+    # See GH-2312
+    response = response_factory.requests()
+    schema = schemathesis.from_dict(sample_schema)
+    operation = schema["/test"]["POST"]
+    case = Case(
+        operation=operation,
+        generation_time=0.0,
+        meta=build_metadata(query=DataGenerationMethod.negative),
+        data_generation_method=DataGenerationMethod.negative,
+        query={"key": 5, "unknown": 3},
+    )
+    assert negative_data_rejection(response, case) is None

{schemathesis-3.31.0 → schemathesis-3.31.1}/test/specs/openapi/test_negative.py

@@ -92,43 +92,46 @@ def test_top_level_strategy(data, location, schema):
 
 
 @pytest.mark.parametrize(
-    "mutation, schema, validate",
+    "mutation, schema, location, validate",
     (
         # No constraints besides `type`
-        (negate_constraints, {"type": "integer"}, True),
+        (negate_constraints, {"type": "integer"}, "body", True),
         # Missing type (i.e. all types are possible)
-        (change_type, {}, True),
+        (change_type, {}, "body", True),
         # All types explicitly
-        (change_type, {"type": ["string", "integer", "number", "object", "array", "boolean", "null"]}, True),
+        (change_type, {"type": ["string", "integer", "number", "object", "array", "boolean", "null"]}, "body", True),
         # No properties to remove
-        (remove_required_property, {}, True),
+        (remove_required_property, {}, "body", True),
         # Non-"object" type
-        (remove_required_property, {"type": "array"}, True),
+        (remove_required_property, {"type": "array"}, "body", True),
         # No properties at all
-        (change_properties, {}, True),
+        (change_properties, {}, "body", True),
         # No properties that can be mutated
-        (change_properties, {"properties": {"foo": {}}}, True),
+        (change_properties, {"properties": {"foo": {}}}, "body", True),
         # No items
-        (change_items, {"type": "array"}, True),
+        (change_items, {"type": "array"}, "body", True),
         # `items` accept everything
-        (change_items, {"type": "array", "items": {}}, True),
-        (change_items, {"type": "array", "items": True}, False),
+        (change_items, {"type": "array", "items": {}}, "body", True),
+        (change_items, {"type": "array", "items": True}, "body", False),
         # `items` is equivalent to accept-everything schema
-        (change_items, {"type": "array", "items": {"uniqueItems": False}}, True),
+        (change_items, {"type": "array", "items": {"uniqueItems": False}}, "body", True),
         # The first element could be anything
-        (change_items, {"type": "array", "items": [{}]}, True),
+        (change_items, {"type": "array", "items": [{}]}, "body", True),
+        # Query and path parameters are always strings
+        (change_type, {"type": "string"}, "path", True),
+        (change_type, {"type": "string"}, "query", True),
     ),
 )
 @given(data=st.data())
 @settings(deadline=None, suppress_health_check=SUPPRESSED_HEALTH_CHECKS, max_examples=MAX_EXAMPLES)
-def test_failing_mutations(data, mutation, schema, validate):
+def test_failing_mutations(data, mutation, schema, location, validate):
     if validate:
         validate_schema(schema)
     original_schema = fast_deepcopy(schema)
     # When mutation can't be applied
     # Then it returns "failure"
     assert (
-        mutation(MutationContext(schema, {}, "body", "application/json"), data.draw, schema) == MutationResult.FAILURE
+        mutation(MutationContext(schema, {}, location, "application/json"), data.draw, schema) == MutationResult.FAILURE
     )
     # And doesn't mutate the input schema
     assert schema == original_schema
@@ -203,7 +206,7 @@ def test_successful_mutations(data, mutation, schema):
         {
             "type": "object",
             "properties": {
-                "foo": {"type": "string"},
+                "foo": {"type": "integer"},
             },
             "required": [
                 "foo",

{schemathesis-3.31.0 → schemathesis-3.31.1}/test/test_direct_access.py

@@ -40,7 +40,7 @@ def test_as_strategy(swagger_20):
     strategy = operation.as_strategy()
     assert isinstance(strategy, st.SearchStrategy)
     assert strategy.example() == Case(
-        operation, generation_time=ANY, data_generation_method=DataGenerationMethod.positive
+        operation, generation_time=ANY, data_generation_method=DataGenerationMethod.positive, meta=ANY
     )
 
 

{schemathesis-3.31.0 → schemathesis-3.31.1}/test/test_hypothesis.py

@@ -75,6 +75,7 @@ def test_get_examples(location, swagger_20):
         generation_time=ANY,
         data_generation_method=DataGenerationMethod.positive,
         media_type=media_type,
+        meta=ANY,
         **{container: expected},
     )