schemathesis 4.0.0a9__tar.gz → 4.0.0a10__tar.gz

{schemathesis-4.0.0a9 → schemathesis-4.0.0a10}/CHANGELOG.md

@@ -1,9 +1,16 @@
 # Changelog
 
-## [Unreleased](https://github.com/schemathesis/schemathesis/compare/v4.0.0-alpha.9...HEAD) - TBD
+## [Unreleased](https://github.com/schemathesis/schemathesis/compare/v4.0.0-alpha.10...HEAD) - TBD
 
-## [4.0.0-alpha.9](https://github.com/schemathesis/schemathesis/compare/v4.0.0-alpha.8...v4.0.0-alpha.9) - 2025-05-06
+## [4.0.0-alpha.10](https://github.com/schemathesis/schemathesis/compare/v4.0.0-alpha.9...v4.0.0-alpha.10) - 2025-05-07
+
+### :bug: Fixed
 
+- Generate negative test cases for large nested arrays during the coverage phase.
+- Generate strings larger than 10000 characters during the coverage phase.
+- Internal error on building a JUnit report if the same failures appear multiple times across multiple test phases. [#2814](https://github.com/schemathesis/schemathesis/issues/2814)
+
+## [4.0.0-alpha.9](https://github.com/schemathesis/schemathesis/compare/v4.0.0-alpha.8...v4.0.0-alpha.9) - 2025-05-06
 
 ### :wrench: Changed
 

{schemathesis-4.0.0a9 → schemathesis-4.0.0a10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: schemathesis
-Version: 4.0.0a9
+Version: 4.0.0a10
 Summary: Property-based testing framework for Open API and GraphQL based apps
 Project-URL: Documentation, https://schemathesis.readthedocs.io/en/stable/
 Project-URL: Changelog, https://schemathesis.readthedocs.io/en/stable/changelog.html

{schemathesis-4.0.0a9 → schemathesis-4.0.0a10}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "schemathesis"
-version = "4.0.0-alpha.9"
+version = "4.0.0-alpha.10"
 description = "Property-based testing framework for Open API and GraphQL based apps"
 keywords = ["pytest", "hypothesis", "openapi", "graphql", "testing"]
 authors = [{ name = "Dmitry Dygalo", email = "dmitry@dygalo.dev" }]

{schemathesis-4.0.0a9 → schemathesis-4.0.0a10}/src/schemathesis/cli/commands/run/handlers/junitxml.py

@@ -23,7 +23,7 @@ class JunitXMLHandler(EventHandler):
             label = event.recorder.label
             test_case = self.get_or_create_test_case(label)
             test_case.elapsed_sec += event.elapsed_time
-            if event.status == Status.FAILURE:
+            if event.status == Status.FAILURE and label in ctx.statistic.failures:
                 add_failure(test_case, ctx.statistic.failures[label].values(), ctx)
             elif event.status == Status.SKIP and event.skip_reason is not None:
                 test_case.add_skipped_info(output=event.skip_reason)

{schemathesis-4.0.0a9 → schemathesis-4.0.0a10}/src/schemathesis/core/__init__.py

@@ -4,6 +4,7 @@ import enum
 from dataclasses import dataclass
 
 SCHEMATHESIS_TEST_CASE_HEADER = "X-Schemathesis-TestCaseId"
+INTERNAL_BUFFER_SIZE = 32 * 1024
 
 
 class NotSet: ...

{schemathesis-4.0.0a9 → schemathesis-4.0.0a10}/src/schemathesis/generation/coverage.py

@@ -16,7 +16,7 @@ from hypothesis_jsonschema import from_schema
 from hypothesis_jsonschema._canonicalise import canonicalish
 from hypothesis_jsonschema._from_schema import STRING_FORMATS as BUILT_IN_STRING_FORMATS
 
-from schemathesis.core import NOT_SET
+from schemathesis.core import INTERNAL_BUFFER_SIZE, NOT_SET
 from schemathesis.core.compat import RefResolutionError
 from schemathesis.core.transforms import deepclone
 from schemathesis.core.validation import has_invalid_characters, is_latin_1_encodable
@@ -36,7 +36,6 @@ def json_recursive_strategy(strategy: st.SearchStrategy) -> st.SearchStrategy:
     return st.lists(strategy, max_size=3) | st.dictionaries(st.text(), strategy, max_size=3)
 
 
-BUFFER_SIZE = 8 * 1024
 NEGATIVE_MODE_MAX_LENGTH_WITH_PATTERN = 100
 NEGATIVE_MODE_MAX_ITEMS = 15
 FLOAT_STRATEGY: st.SearchStrategy = st.floats(allow_nan=False, allow_infinity=False).map(_replace_zero_with_nonzero)
@@ -161,7 +160,7 @@ class CoverageContext:
     def generate_from_schema(self, schema: dict | bool) -> Any:
         if isinstance(schema, bool):
             return 0
-        keys = sorted([k for k in schema if not k.startswith("x-") and k not in ["description", "example"]])
+        keys = sorted([k for k in schema if not k.startswith("x-") and k not in ["description", "example", "examples"]])
         if keys == ["type"] and isinstance(schema["type"], str) and schema["type"] in STRATEGIES_FOR_TYPE:
             return cached_draw(STRATEGIES_FOR_TYPE[schema["type"]])
         if keys == ["format", "type"]:
@@ -393,7 +392,7 @@ def cover_schema_iter(
                         if k not in seen:
                             yield value_
                             seen.add(k)
-                elif key == "minLength" and 0 < value < BUFFER_SIZE:
+                elif key == "minLength" and 0 < value < INTERNAL_BUFFER_SIZE:
                     if value == 1:
                         # In this case, the only possible negative string is an empty one
                         # The `pattern` value may require an non-empty one and the generation will fail
@@ -427,7 +426,7 @@ def cover_schema_iter(
                                     value, description="String smaller than minLength", location=ctx.current_path
                                 )
                                 seen.add(k)
-                elif key == "maxLength" and value < BUFFER_SIZE:
+                elif key == "maxLength" and value < INTERNAL_BUFFER_SIZE:
                     try:
                         min_length = max_length = value + 1
                         new_schema = {**schema, "minLength": min_length, "maxLength": max_length}
@@ -461,7 +460,7 @@ def cover_schema_iter(
                 elif key == "required":
                     template = template or ctx.generate_from_schema(_get_template_schema(schema, "object"))
                     yield from _negative_required(ctx, template, value)
-                elif key == "maxItems" and isinstance(value, int) and value < BUFFER_SIZE:
+                elif key == "maxItems" and isinstance(value, int) and value < INTERNAL_BUFFER_SIZE:
                     if value > NEGATIVE_MODE_MAX_ITEMS:
                         # It could be extremely slow to generate large arrays
                         # Generate values up to the limit and reuse them to construct the final array
@@ -471,7 +470,18 @@ def cover_schema_iter(
                             "maxItems": NEGATIVE_MODE_MAX_ITEMS,
                             "type": "array",
                         }
-                        array_value = ctx.generate_from_schema(new_schema)
+                        if "items" in schema and isinstance(schema["items"], dict):
+                            # The schema may have another large array nested, therefore generate covering cases
+                            # and use them to build an array for the current schema
+                            negative = [case.value for case in cover_schema_iter(ctx, schema["items"])]
+                            positive = [case.value for case in cover_schema_iter(ctx.with_positive(), schema["items"])]
+                            # Interleave positive & negative values
+                            array_value = [value for pair in zip(positive, negative) for value in pair][
+                                :NEGATIVE_MODE_MAX_ITEMS
+                            ]
+                        else:
+                            array_value = ctx.generate_from_schema(new_schema)
+
                         # Extend the array to be of length value + 1 by repeating its own elements
                         diff = value + 1 - len(array_value)
                         if diff > 0:
@@ -606,7 +616,7 @@ def _positive_string(ctx: CoverageContext, schema: dict) -> Generator[GeneratedV
 
     seen = set()
 
-    if min_length is not None and min_length < BUFFER_SIZE:
+    if min_length is not None and min_length < INTERNAL_BUFFER_SIZE:
         # Exactly the minimum length
         yield PositiveValue(
             ctx.generate_from_schema({**schema, "maxLength": min_length}), description="Minimum length string"
@@ -615,7 +625,7 @@ def _positive_string(ctx: CoverageContext, schema: dict) -> Generator[GeneratedV
 
         # One character more than minimum if possible
         larger = min_length + 1
-        if larger < BUFFER_SIZE and larger not in seen and (not max_length or larger <= max_length):
+        if larger < INTERNAL_BUFFER_SIZE and larger not in seen and (not max_length or larger <= max_length):
             yield PositiveValue(
                 ctx.generate_from_schema({**schema, "minLength": larger, "maxLength": larger}),
                 description="Near-boundary length string",
@@ -624,7 +634,7 @@ def _positive_string(ctx: CoverageContext, schema: dict) -> Generator[GeneratedV
 
     if max_length is not None:
         # Exactly the maximum length
-        if max_length < BUFFER_SIZE and max_length not in seen:
+        if max_length < INTERNAL_BUFFER_SIZE and max_length not in seen:
             yield PositiveValue(
                 ctx.generate_from_schema({**schema, "minLength": max_length}), description="Maximum length string"
             )
@@ -633,7 +643,7 @@ def _positive_string(ctx: CoverageContext, schema: dict) -> Generator[GeneratedV
         # One character less than maximum if possible
         smaller = max_length - 1
         if (
-            smaller < BUFFER_SIZE
+            smaller < INTERNAL_BUFFER_SIZE
             and smaller not in seen
             and (smaller > 0 and (min_length is None or smaller >= min_length))
         ):
@@ -765,7 +775,7 @@ def _positive_array(ctx: CoverageContext, schema: dict, template: list) -> Gener
                 yield PositiveValue(value, description="Near-boundary items array")
 
     if max_items is not None:
-        if max_items < BUFFER_SIZE and max_items not in seen:
+        if max_items < INTERNAL_BUFFER_SIZE and max_items not in seen:
             value = ctx.generate_from_schema({**schema, "minItems": max_items})
             key = _to_hashable_key(value)
             if key not in seen:
@@ -775,7 +785,7 @@ def _positive_array(ctx: CoverageContext, schema: dict, template: list) -> Gener
         # One item smaller than maximum if possible
         smaller = max_items - 1
         if (
-            smaller < BUFFER_SIZE
+            smaller < INTERNAL_BUFFER_SIZE
             and smaller > 0
             and smaller not in seen
             and (min_items is None or smaller >= min_items)

{schemathesis-4.0.0a9 → schemathesis-4.0.0a10}/src/schemathesis/generation/hypothesis/__init__.py

@@ -5,11 +5,14 @@ DEFAULT_DEADLINE = 15000
 
 
 def setup() -> None:
+    from hypothesis import core as root_core
+    from hypothesis.internal.conjecture import engine
     from hypothesis.internal.entropy import deterministic_PRNG
     from hypothesis.internal.reflection import is_first_param_referenced_in_function
-    from hypothesis.strategies._internal import core
+    from hypothesis.strategies._internal import collections, core
     from hypothesis_jsonschema import _from_schema, _resolve
 
+    from schemathesis.core import INTERNAL_BUFFER_SIZE
     from schemathesis.core.transforms import deepclone
 
     # Forcefully initializes Hypothesis' global PRNG to avoid races that initialize it
@@ -28,3 +31,6 @@ def setup() -> None:
     core.is_first_param_referenced_in_function = _is_first_param_referenced_in_function  # type: ignore
     _resolve.deepcopy = deepclone  # type: ignore
     _from_schema.deepcopy = deepclone  # type: ignore
+    root_core.BUFFER_SIZE = INTERNAL_BUFFER_SIZE  # type: ignore
+    engine.BUFFER_SIZE = INTERNAL_BUFFER_SIZE
+    collections.BUFFER_SIZE = INTERNAL_BUFFER_SIZE  # type: ignore

schemathesis-4.0.0a10/test/cli/__snapshots__/test_commands/test_rediscover_the_same_failure_in_different_phases_and_store_junit[Open API 3.0].raw

@@ -0,0 +1,89 @@
+Exit code: 1
+---
+Stdout:
+Schemathesis dev
+━━━━━━━━━━━━━━━━
+
+
+ ✅  Loaded specification from http://127.0.0.1/schema.yaml (in 0.00s)
+
+     Base URL:         http://127.0.0.1/api
+     Specification:    Open API 3.0.2
+     Operations:       3 selected / 3 total
+
+
+ ✅  API capabilities:
+
+     Supports NULL byte in headers:    ✘
+
+ ❌  Examples (in 0.00s)
+
+     ✅ 1 passed  ❌ 1 failed  ⏭  1 skipped
+
+ ❌  Coverage (in 0.00s)
+
+     ✅ 2 passed  ❌ 1 failed
+
+ ❌  Fuzzing (in 0.00s)
+
+     ✅ 2 passed  ❌ 1 failed
+
+ ❌  Stateful (in 0.00s)
+
+     Scenarios:    N
+     API Links:    N covered / 4 selected / 4 total
+
+     ✅ N passed  ❌ 1 failed
+
+=================================== FAILURES ===================================
+_____________________________ GET /users/{user_id} _____________________________
+1. Test Case ID: <PLACEHOLDER>
+
+- Custom check failed: `always_fails`
+
+    Failed!
+
+[404] Not Found:
+
+    `{"message": "Not found"}`
+
+Reproduce with:
+
+    curl -X GET 'http://127.0.0.1/api/users/0?user_id=test-id&code=0&common=0'
+
+=================================== WARNINGS ===================================
+
+Schemathesis configuration: 1 operation returned only 4xx responses during unit tests
+
+  - PATCH /users/{user_id}
+
+Tip: Check base URL or adjust data generation settings
+
+=================================== SUMMARY ====================================
+
+API Operations:
+  Selected: 3/3
+  Tested: 3
+
+Test Phases:
+  ✅ API probing
+  ❌ Examples
+  ❌ Coverage
+  ❌ Fuzzing
+  ❌ Stateful
+
+Failures:
+  ❌ Custom check failed: `always_fails`: 1
+
+Warnings:
+  ⚠️ Schemathesis configuration: 1 operation returned only 4xx responses during unit tests
+
+Test cases:
+  N generated, N found N unique failures
+
+Reports:
+  - JUNIT: /tmp/test_rediscover_the_same_failu0/reports/junit.xml
+
+Seed: 42
+
+============================== 1 failure in 1.00s ==============================

{schemathesis-4.0.0a9 → schemathesis-4.0.0a10}/test/cli/test_commands.py

@@ -1640,3 +1640,32 @@ class EventCounter(cli.EventHandler):
         )
         == snapshot_cli
     )
+
+
+@pytest.mark.openapi_version("3.0")
+@pytest.mark.operations("create_user", "get_user", "update_user")
+@pytest.mark.skipif(platform.system() == "Windows", reason="Requires a more complex test setup")
+def test_rediscover_the_same_failure_in_different_phases_and_store_junit(ctx, cli, schema_url, tmp_path, snapshot_cli):
+    # See GH-2814
+    report_dir = tmp_path / "reports"
+    with ctx.check(
+        r"""
+@schemathesis.check
+def always_fails(ctx, response, case):
+    if case.operation.label == "GET /users/{user_id}":
+        raise AssertionError("Failed!")
+"""
+    ) as module:
+        assert (
+            cli.main(
+                "run",
+                schema_url,
+                "-c",
+                "always_fails",
+                "--max-examples=1",
+                f"--report-dir={report_dir}",
+                "--report=junit",
+                hooks=module,
+            )
+            == snapshot_cli
+        )

{schemathesis-4.0.0a9 → schemathesis-4.0.0a10}/test/coverage/test_combinations.py

@@ -1215,6 +1215,22 @@ def test_generate_large_string(request, ctx, expected):
     assert cover_schema(ctx, schema) == expected
 
 
+def test_generate_very_large_string(nctx):
+    schema = {
+        "properties": {
+            "name": {"maxLength": 10000, "minLength": 1, "pattern": "^[\\w\\W]*$", "type": "string"},
+        },
+        "required": ["name"],
+        "type": "object",
+    }
+
+    assert 10001 in {
+        len(item["name"])
+        for item in cover_schema(nctx, schema)
+        if isinstance(item, dict) and isinstance(item.get("name"), str)
+    }
+
+
 def test_large_string_with_complex_pattern(nctx):
     schema = {
         "maxLength": 4000,
@@ -1462,3 +1478,37 @@ def test_large_arrays(nctx):
         for item in cover_schema(nctx, schema)
         if isinstance(item, dict) and isinstance(item["questions"], list)
     }
+
+
+def test_large_arrays_nested(nctx):
+    schema = {
+        "properties": {
+            "questions": {
+                "items": {
+                    "properties": {
+                        "answers": {
+                            "items": {
+                                "type": "null",
+                            },
+                            "maxItems": 100,
+                            "type": "array",
+                        },
+                        "id": {"minLength": 6, "pattern": "^[0-9]+$", "type": "string"},
+                    },
+                    "required": ["id"],
+                    "type": "object",
+                },
+                "maxItems": 500,
+                "minItems": 1,
+                "type": "array",
+            },
+        },
+        "required": ["questions"],
+        "type": "object",
+    }
+
+    assert 501 in {
+        len(item["questions"])
+        for item in cover_schema(nctx, schema)
+        if isinstance(item, dict) and isinstance(item.get("questions"), list)
+    }

{schemathesis-4.0.0a9 → schemathesis-4.0.0a10}/test/coverage/test_phase.py

@@ -538,64 +538,75 @@ def test_with_examples_openapi_3_request_body(ctx):
         schema,
         [GenerationMode.POSITIVE],
         [
-            {"body": {"address": {}, "age": 25, "name": "John Doe", "tags": ["designer", "ui/ux"]}},
             {
                 "body": {
-                    "address": {"street": "456 Elm St"},
-                    "age": 25,
                     "name": "John Doe",
-                    "tags": ["designer", "ui/ux"],
+                    "age": 30,
+                    "tags": ["developer", "python"],
+                    "address": {"street": "123 Main St", "city": "Somewhere"},
                 }
             },
-            {"body": {"address": {"city": "Anytown"}, "age": 25, "name": "John Doe", "tags": ["designer", "ui/ux"]}},
             {
                 "body": {
-                    "address": {"street": "456 Elm St", "city": "Somewhere"},
-                    "age": 25,
                     "name": "John Doe",
-                    "tags": ["designer", "ui/ux"],
+                    "age": 30,
+                    "tags": ["developer", "python"],
+                    "address": {"street": "456 Elm St", "city": "Anytown"},
                 }
             },
+            {"body": {"name": "John Doe", "age": 30, "tags": ["developer", "python"], "address": {}}},
             {
                 "body": {
-                    "address": {
-                        "city": "Anytown",
-                        "street": "123 Main St",
-                    },
-                    "age": 25,
                     "name": "John Doe",
-                    "tags": [
-                        "",
-                    ],
-                },
+                    "age": 30,
+                    "tags": ["developer", "python"],
+                    "address": {"street": "123 Main St"},
+                }
             },
+            {"body": {"name": "John Doe", "age": 30, "tags": ["developer", "python"], "address": {"city": "Anytown"}}},
             {
                 "body": {
-                    "address": {"city": "Anytown", "street": "123 Main St"},
-                    "age": 25,
                     "name": "John Doe",
+                    "age": 30,
                     "tags": ["developer", "python"],
+                    "address": {"street": "456 Elm St", "city": "Somewhere"},
                 }
             },
             {
                 "body": {
-                    "address": {"city": "Anytown", "street": "123 Main St"},
+                    "name": "John Doe",
                     "age": 30,
+                    "tags": [""],
+                    "address": {"street": "123 Main St", "city": "Anytown"},
+                }
+            },
+            {
+                "body": {
                     "name": "John Doe",
+                    "age": 30,
                     "tags": ["designer", "ui/ux"],
+                    "address": {"street": "123 Main St", "city": "Anytown"},
                 }
             },
             {
                 "body": {
-                    "address": {"city": "Anytown", "street": "123 Main St"},
+                    "name": "John Doe",
                     "age": 25,
+                    "tags": ["developer", "python"],
+                    "address": {"street": "123 Main St", "city": "Anytown"},
+                }
+            },
+            {
+                "body": {
                     "name": "Jane Smith",
-                    "tags": ["designer", "ui/ux"],
+                    "age": 30,
+                    "tags": ["developer", "python"],
+                    "address": {"street": "123 Main St", "city": "Anytown"},
                 }
             },
-            {"body": {"age": 25, "name": "John Doe"}},
-            {"body": {"age": 25, "name": "John Doe", "tags": ["designer", "ui/ux"]}},
-            {"body": {"address": {"city": "Anytown", "street": "123 Main St"}, "age": 25, "name": "John Doe"}},
+            {"body": {"name": "John Doe", "age": 30}},
+            {"body": {"name": "John Doe", "age": 30, "tags": ["developer", "python"]}},
+            {"body": {"name": "John Doe", "age": 30, "address": {"street": "123 Main St", "city": "Anytown"}}},
             {
                 "body": {
                     "name": "Jane Smith",

{schemathesis-4.0.0a9 → schemathesis-4.0.0a10}/test/test_hypothesis.py

@@ -494,7 +494,7 @@ def test_health_check_failed_large_base_example(ctx, cli, snapshot_cli, openapi3
                         "required": True,
                         "content": {
                             "application/json": {
-                                "schema": {"type": "array", "items": {"type": "integer"}, "minItems": 10000}
+                                "schema": {"type": "array", "items": {"type": "integer"}, "minItems": 100000}
                             }
                         },
                     },