schemathesis 4.0.23__tar.gz → 4.0.25__tar.gz

{schemathesis-4.0.23 → schemathesis-4.0.25}/.github/workflows/build.yml

@@ -66,6 +66,7 @@ jobs:
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v5.4.3
         with:
+          files: coverage.xml
           token: ${{ secrets.CODECOV_TOKEN }}
 
   # On windows redirecting output to a file could lead to an error due to the default IO encoding being CP-1252

schemathesis-4.0.25/.github/workflows/memory-benchmark.yml

@@ -0,0 +1,48 @@
+name: Memory Benchmark
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+permissions:
+  contents: read
+  pull-requests: write
+  issues: write
+
+jobs:
+  memory-benchmark:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4.2.1
+        with:
+          fetch-depth: 1
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+
+      - uses: astral-sh/setup-uv@v6
+
+      - run: uv pip install '.' 'memray==1.18.0' --system
+
+      - run: python benches/memory/measure.py
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: memory-benchmark-results
+          path: memory-benchmark-results/
+          retention-days: 30
+
+      - uses: peter-evans/find-comment@v2
+        id: fc
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: 'github-actions[bot]'
+          body-includes: Memory Benchmark Results
+
+      - uses: peter-evans/create-or-update-comment@v3
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ github.event.pull_request.number }}
+          body-path: memory-benchmark-results/report.md
+          edit-mode: replace

{schemathesis-4.0.23 → schemathesis-4.0.25}/.pre-commit-config.yaml

@@ -3,7 +3,7 @@ default_language_version:
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
+    rev: v6.0.0
     hooks:
       - id: end-of-file-fixer
         exclude: ^.*\.raw$
@@ -33,11 +33,11 @@ repos:
         additional_dependencies: [types-requests, types-PyYAML]
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.12.7
+    rev: v0.12.8
     hooks:
       - id: ruff-format
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.12.7
+    rev: v0.12.8
     hooks:
       - id: ruff-check

{schemathesis-4.0.23 → schemathesis-4.0.25}/CHANGELOG.md

@@ -1,6 +1,36 @@
 # Changelog
 
-## [Unreleased](https://github.com/schemathesis/schemathesis/compare/v4.0.23...HEAD) - TBD
+## [Unreleased](https://github.com/schemathesis/schemathesis/compare/v4.0.25...HEAD) - TBD
+
+## [4.0.25](https://github.com/schemathesis/schemathesis/compare/v4.0.24...v4.0.25) - 2025-08-10
+
+### :bug: Fixed
+
+- `workers = "auto"` being incorrectly rejected in configuration files. [#3015](https://github.com/schemathesis/schemathesis/issues/3015)
+
+### :wrench: Changed
+
+- Improve error message for invalid `workers` values.
+
+### :scroll: Documentation
+
+- Add a search bar. [#3013](https://github.com/schemathesis/schemathesis/issues/3013)
+
+### :racing_car: Performance
+
+- Dramatically improved performance and memory usage. For the coverage phase expect order of magnitude improvements that scale with schema complexity.
+
+Examples:
+  - PetStore API: **12s → 1.5s** (8x faster)
+  - Complex endpoint schema with nested objects and 40+ fields: **330s → 4.5s** (73x faster)
+
+These improvements come from removing unnecessary memory allocations in unnecessary pretty-printing operations in Hypothesis.
+
+## [4.0.24](https://github.com/schemathesis/schemathesis/compare/v4.0.23...v4.0.24) - 2025-08-09
+
+### :bug: Fixed
+
+- Internal error on incorrect value for `--tls-verify` when the schema URL is accessed via HTTPs. [#3007](https://github.com/schemathesis/schemathesis/issues/3007)
 
 ## [4.0.23](https://github.com/schemathesis/schemathesis/compare/v4.0.22...v4.0.23) - 2025-08-07
 

{schemathesis-4.0.23 → schemathesis-4.0.25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: schemathesis
-Version: 4.0.23
+Version: 4.0.25
 Summary: Property-based testing framework for Open API and GraphQL based apps
 Project-URL: Documentation, https://schemathesis.readthedocs.io/en/stable/
 Project-URL: Changelog, https://github.com/schemathesis/schemathesis/blob/master/CHANGELOG.md

schemathesis-4.0.25/benches/memory/hooks.py

@@ -0,0 +1,15 @@
+from unittest.mock import patch
+
+import requests
+
+from schemathesis.core.transport import Response
+
+RESPONSE = Response(
+    status_code=200,
+    headers={"Content-Type": ["application/json"]},
+    content=b"{}",
+    request=requests.Request(method="GET", url="http://127.0.0.1/test").prepare(),
+    elapsed=0.1,
+    verify=False,
+)
+patch("schemathesis.Case.call", return_value=RESPONSE).start()

schemathesis-4.0.25/benches/memory/measure.py

@@ -0,0 +1,110 @@
+import json
+import os
+import subprocess
+import tempfile
+from pathlib import Path
+from typing import Any
+
+
+def run_memray(schema_path: str, args: list[str]) -> dict[str, Any]:
+    with tempfile.NamedTemporaryFile(suffix=".bin") as tmp:
+        memray_output = tmp.name
+
+        stats_output = memray_output + ".json"
+        cmd = (
+            ["memray", "run", "--force", "-o", memray_output, "-m", "schemathesis.cli", "run"]
+            + args
+            + ["-u http://127.0.0.1:8000", schema_path]
+        )
+
+        subprocess.run(
+            cmd,
+            stdout=subprocess.PIPE,
+            text=True,
+            env={**os.environ, "SCHEMATHESIS_HOOKS": "benches.memory.hooks"},
+        )
+
+        subprocess.run(
+            ["memray", "stats", "--force", "-o", stats_output, "--json", memray_output],
+            stdout=subprocess.PIPE,
+            text=True,
+        )
+
+        with open(stats_output, "r") as f:
+            stats = json.load(f)
+
+        return {
+            "peak_memory": stats["metadata"]["peak_memory"],
+            "total_bytes_allocated": stats["total_bytes_allocated"],
+            "total_num_allocations": stats["total_num_allocations"],
+        }
+
+
+def format_bytes(value: int | float) -> str:
+    for unit in ["B", "KB", "MB", "GB"]:
+        if value < 1024.0:
+            return f"{value:.2f} {unit}"
+        value /= 1024.0
+    return f"{value:.2f} TB"
+
+
+HERE = Path(__file__).parent
+
+
+def main():
+    scenarios_path = HERE / "scenarios.json"
+    with open(scenarios_path, "r") as f:
+        config = json.load(f)
+
+    schemas_dir = HERE / "schemas"
+    results = []
+
+    print("Running memory benchmarks...")  # noqa: T201
+    for scenario in config["scenarios"]:
+        print(f"  Running {scenario['name']}...")  # noqa: T201
+        schema_path = schemas_dir / scenario["schema"]
+
+        result = run_memray(str(schema_path), scenario["args"])
+        result["name"] = scenario["name"]
+        result["schema"] = scenario["schema"]
+        result["args"] = " ".join(scenario["args"])
+        results.append(result)
+
+    report = ["## 📊 Memory Benchmark Results\n"]
+    report.append("| Scenario | Schema | Peak Memory | Total Allocated | Total Allocations |")
+    report.append("|----------|--------|-------------|-----------------|-------------------|")
+
+    for r in results:
+        peak = format_bytes(r["peak_memory"])
+        total = format_bytes(r["total_bytes_allocated"])
+        allocations = f"{r['total_num_allocations']:,}"
+
+        report.append(f"| {r['name']} | {r['schema']} | {peak} | {total} | {allocations} |")
+
+    report.append("\n<details>")
+    report.append("<summary>📝 Benchmark Details</summary>\n")
+
+    for r in results:
+        report.append(f"### {r['name']}")
+        report.append(f"- **Command args**: `{r['args']}`")
+        report.append(f"- **Peak Memory**: {r['peak_memory']:,} bytes")
+        report.append(f"- **Total Allocated**: {r['total_bytes_allocated']:,} bytes")
+        report.append(f"- **Total Allocations**: {r['total_num_allocations']:,}")
+        report.append("")
+
+    report.append("</details>")
+
+    output_dir = Path(os.environ.get("GITHUB_WORKSPACE", ".")) / "memory-benchmark-results"
+    output_dir.mkdir(exist_ok=True)
+
+    with open(output_dir / "results.json", "w") as f:
+        json.dump(results, f, indent=2)
+
+    with open(output_dir / "report.md", "w") as f:
+        f.write("\n".join(report))
+
+    print(f"Results saved to {output_dir}")  # noqa: T201
+
+
+if __name__ == "__main__":
+    main()

schemathesis-4.0.25/benches/memory/scenarios.json

@@ -0,0 +1,20 @@
+{
+  "scenarios": [
+    {
+      "name": "large_payload_fuzzing",
+      "schema": "large_payload.yaml",
+      "args": [
+        "--phases=fuzzing",
+        "--continue-on-failure"
+      ]
+    },
+    {
+      "name": "large_payload_coverage",
+      "schema": "large_payload.yaml",
+      "args": [
+        "--phases=coverage",
+        "--continue-on-failure"
+      ]
+    }
+  ]
+}

schemathesis-4.0.25/benches/memory/schemas/large_payload.yaml

@@ -0,0 +1,217 @@
+openapi: 3.0.3
+info:
+  title: Large Payload API
+  version: 1.0.0
+  description: API with complex nested payloads for memory testing
+
+paths:
+  /api/v1/organizations/{orgId}/projects:
+    parameters:
+      - name: orgId
+        in: path
+        required: true
+        schema:
+          type: string
+          format: uuid
+    post:
+      summary: Create a new project with complex data
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/ProjectCreationRequest'
+      responses:
+        '201':
+          description: Project created successfully
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ProjectResponse'
+        '400':
+          description: Invalid request
+        '422':
+          description: Validation error
+
+  /api/v1/organizations/{orgId}/projects/{projectId}:
+    parameters:
+      - name: orgId
+        in: path
+        required: true
+        schema:
+          type: string
+          format: uuid
+      - name: projectId
+        in: path
+        required: true
+        schema:
+          type: string
+          format: uuid
+    put:
+      summary: Update project with full data
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/ProjectUpdateRequest'
+      responses:
+        '200':
+          description: Project updated successfully
+
+components:
+  schemas:
+    ProjectCreationRequest:
+      type: object
+      required:
+        - name
+        - description
+        - projectType
+        - startDate
+        - endDate
+        - teamMembers
+        - tags
+      properties:
+        name:
+          type: string
+          minLength: 3
+          maxLength: 100
+          pattern: '^[a-zA-Z0-9\s\-_]+$'
+          description: Project name
+
+        description:
+          type: string
+          minLength: 10
+          maxLength: 5000
+          description: Detailed project description
+
+        projectType:
+          type: string
+          enum: ['research', 'development', 'infrastructure', 'marketing', 'operations']
+
+        startDate:
+          type: string
+          format: date
+
+        endDate:
+          type: string
+          format: date
+
+        estimatedHours:
+          type: integer
+          minimum: 1
+          maximum: 100000
+
+        priority:
+          type: integer
+          minimum: 1
+          maximum: 10
+          default: 5
+
+        isConfidential:
+          type: boolean
+          default: false
+
+        teamMembers:
+          type: array
+          minItems: 1
+          maxItems: 100
+          items:
+            type: object
+            required:
+              - userId
+              - role
+              - allocation
+            properties:
+              userId:
+                type: string
+                format: uuid
+              role:
+                type: string
+                enum: ['lead', 'developer', 'designer', 'analyst', 'tester', 'manager', 'consultant']
+              allocation:
+                type: number
+                minimum: 0
+                maximum: 100
+                description: Percentage allocation
+              permissions:
+                type: array
+                items:
+                  type: string
+                  enum: ['read', 'write', 'delete', 'admin', 'approve', 'review']
+              joinDate:
+                type: string
+                format: date
+              skills:
+                type: array
+                maxItems: 30
+                items:
+                  type: string
+                  maxLength: 50
+
+        tags:
+          type: array
+          minItems: 1
+          maxItems: 30
+          items:
+            type: string
+            minLength: 2
+            maxLength: 50
+            pattern: '^[a-z0-9\-]+$'
+
+        dependencies:
+          type: array
+          maxItems: 20
+          items:
+            type: object
+            properties:
+              projectId:
+                type: string
+                format: uuid
+              dependencyType:
+                type: string
+                enum: ['blocks', 'requires', 'relates_to']
+              description:
+                type: string
+                maxLength: 500
+
+    ProjectUpdateRequest:
+      allOf:
+        - $ref: '#/components/schemas/ProjectCreationRequest'
+        - type: object
+          properties:
+            status:
+              type: string
+              enum: ['draft', 'active', 'paused', 'completed', 'cancelled', 'archived']
+            completionPercentage:
+              type: number
+              minimum: 0
+              maximum: 100
+            lastModifiedBy:
+              type: string
+              format: email
+            lastModifiedAt:
+              type: string
+              format: date-time
+
+    ProjectResponse:
+      allOf:
+        - $ref: '#/components/schemas/ProjectCreationRequest'
+        - type: object
+          required:
+            - id
+            - createdAt
+            - updatedAt
+          properties:
+            id:
+              type: string
+              format: uuid
+            createdAt:
+              type: string
+              format: date-time
+            updatedAt:
+              type: string
+              format: date-time
+            version:
+              type: integer
+              minimum: 1

{schemathesis-4.0.23 → schemathesis-4.0.25}/docs/reference/configuration.md

@@ -369,6 +369,10 @@ These settings can only be applied at the project level.
     workers = 4       # Use exactly 4 workers
     ```
 
+    ```toml
+    workers = "auto"  # Allocate workers based on the number of available CPU cores
+    ```
+
 #### `wait-for-schema`
 
 !!! note ""

{schemathesis-4.0.23 → schemathesis-4.0.25}/docs/resources.md

@@ -98,6 +98,12 @@ This page collects articles, papers, videos, tutorials, and integrations about S
 
 ## Videos
 
+- [Fuzzing REST APIs for Security Testing](https://youtu.be/ZdshB1qcgvw) by **Alina Kostetska** at **RoboCon 2024**
+
+    - **Description**: Security-focused presentation demonstrating how to use Schemathesis / Robot Framework integration for fuzzing REST APIs to discover vulnerabilities.
+
+    - **Date**: 04 Nov 2024
+
 - [API Fuzzing: What it is and why you should use it](https://youtu.be/wX3GMJY9B6A) by **José Haro Peralta**
 
     - **Description**: Technical overview and demonstration of Schemathesis capabilities.

{schemathesis-4.0.23 → schemathesis-4.0.25}/mkdocs.yml

@@ -113,5 +113,6 @@ plugins:
         paths: [src]
         options:
           show_source: false
+- search
 
 copyright: Copyright © 2020-2025 Dmitry Dygalo & Schemathesis.io

{schemathesis-4.0.23 → schemathesis-4.0.25}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "schemathesis"
-version = "4.0.23"
+version = "4.0.25"
 description = "Property-based testing framework for Open API and GraphQL based apps"
 keywords = ["pytest", "hypothesis", "openapi", "graphql", "testing"]
 authors = [{ name = "Dmitry Dygalo", email = "dmitry@dygalo.dev" }]

{schemathesis-4.0.23 → schemathesis-4.0.25}/src/schemathesis/config/_error.py

@@ -140,6 +140,13 @@ def _format_anyof_error(error: ValidationError) -> str:
             f"Error in {section} section:\n  At least one filter is required when defining [[operations]].\n\n"
             "Please specify at least one include or exclude filter property (e.g., include-path, exclude-tag, etc.)."
         )
+    elif list(error.schema_path) == ["properties", "workers", "anyOf"]:
+        return (
+            f"Invalid value for 'workers': {repr(error.instance)}\n\n"
+            f"Expected either:\n"
+            f"  - A positive integer (e.g., workers = 4)\n"
+            f'  - The string "auto" for automatic detection (workers = "auto")'
+        )
     return error.message
 
 

{schemathesis-4.0.23 → schemathesis-4.0.25}/src/schemathesis/config/schema.json

@@ -94,7 +94,14 @@
       "type": "string"
     },
     "workers": {
-      "type": "integer"
+      "anyOf": [
+        {
+          "type": "integer"
+        },
+        {
+          "const": "auto"
+        }
+      ]
     },
     "wait-for-schema": {
       "type": "number",
@@ -491,7 +498,14 @@
           "type": "string"
         },
         "workers": {
-          "type": "integer"
+          "anyOf": [
+            {
+              "type": "integer"
+            },
+            {
+              "const": "auto"
+            }
+          ]
         },
         "wait-for-schema": {
           "type": "number",

{schemathesis-4.0.23 → schemathesis-4.0.25}/src/schemathesis/core/errors.py

@@ -330,6 +330,7 @@ class LoaderErrorKind(str, enum.Enum):
     CONNECTION_SSL = "connection_ssl"
     CONNECTION_OTHER = "connection_other"
     NETWORK_OTHER = "network_other"
+    INVALID_CERTIFICATE = "invalid_certificate"
 
     # HTTP error codes
     HTTP_SERVER_ERROR = "http_server_error"

{schemathesis-4.0.23 → schemathesis-4.0.25}/src/schemathesis/core/loaders.py

@@ -67,6 +67,9 @@ def make_request(func: Callable[..., requests.Response], url: str, **kwargs: Any
         return raise_for_status(response)
     except requests.RequestException as exc:
         handle_request_error(exc)
+    except OSError as exc:
+        # Possible with certificate errors
+        raise LoaderError(message=str(exc), kind=LoaderErrorKind.INVALID_CERTIFICATE, url=url, extras=[]) from None
 
 
 WAIT_FOR_SCHEMA_INTERVAL = 0.05

{schemathesis-4.0.23 → schemathesis-4.0.25}/src/schemathesis/generation/hypothesis/__init__.py

@@ -7,6 +7,7 @@ def setup() -> None:
     from hypothesis.internal.entropy import deterministic_PRNG
     from hypothesis.internal.reflection import is_first_param_referenced_in_function
     from hypothesis.strategies._internal import collections, core
+    from hypothesis.vendor import pretty
     from hypothesis_jsonschema import _from_schema, _resolve
 
     from schemathesis.core import INTERNAL_BUFFER_SIZE
@@ -26,6 +27,14 @@ def setup() -> None:
         return is_first_param_referenced_in_function(f)
 
     core.is_first_param_referenced_in_function = _is_first_param_referenced_in_function  # type: ignore
+
+    class RepresentationPrinter(pretty.RepresentationPrinter):
+        def pretty(self, obj: object) -> None:
+            # This one takes way too much - in the coverage phase it may give >2 orders of magnitude improvement
+            # depending on the schema size (~300 seconds -> 4.5 seconds in one of the benchmarks)
+            return None
+
+    root_core.RepresentationPrinter = RepresentationPrinter  # type: ignore
     _resolve.deepcopy = deepclone  # type: ignore
     _from_schema.deepcopy = deepclone  # type: ignore
     root_core.BUFFER_SIZE = INTERNAL_BUFFER_SIZE  # type: ignore

schemathesis-4.0.25/test/cli/__snapshots__/test_commands/test_invalid_tls_verify[Open API 3.0].raw

@@ -0,0 +1,12 @@
+Exit code: 1
+---
+Stdout:
+Schemathesis dev
+━━━━━━━━━━━━━━━━
+
+
+ ❌  Failed to load specification from https://127.0.0.1/schema.yaml after 0.00s
+
+ Schema Loading Error
+
+ Could not find a suitable TLS CA certificate bundle, invalid path: falst

{schemathesis-4.0.23 → schemathesis-4.0.25}/test/cli/test_commands.py

@@ -1303,6 +1303,12 @@ def test_rate_limit(cli, schema_url):
     assert cli.run(schema_url, "--rate-limit=1/s").exit_code == ExitCode.OK
 
 
+@pytest.mark.openapi_version("3.0")
+@pytest.mark.operations("failure")
+def test_invalid_tls_verify(cli, schema_url, snapshot_cli):
+    assert cli.run(schema_url.replace("http", "https"), "--tls-verify=falst") == snapshot_cli
+
+
 @pytest.mark.parametrize("version", ["3.0.2", "3.1.0"])
 def test_invalid_schema_with_disabled_validation(
     ctx, cli, openapi_3_schema_with_invalid_security, version, snapshot_cli, openapi3_base_url

schemathesis-4.0.25/test/config/__snapshots__/test_config/test_configs[common.incorrect-workers].raw

@@ -0,0 +1,5 @@
+Invalid value for 'workers': 'foo'
+
+Expected either:
+  - A positive integer (e.g., workers = 4)
+  - The string "auto" for automatic detection (workers = "auto")

schemathesis-4.0.25/test/config/__snapshots__/test_doc_examples/test_configs[workers_auto_allocate_workers].raw

@@ -0,0 +1 @@
+SchemathesisConfig(projects=ProjectsConfig(default=ProjectConfig(workers=4)))

schemathesis-4.0.25/test/config/configs/common/incorrect-workers.toml

@@ -0,0 +1 @@
+workers = "foo"