schemathesis 4.0.0a1__tar.gz → 4.0.0a3__tar.gz

{schemathesis-4.0.0a1 → schemathesis-4.0.0a3}/.github/workflows/build.yml

@@ -44,7 +44,7 @@ jobs:
       - uses: actions/checkout@v4.2.1
         with:
           fetch-depth: 1
-      - uses: ammaraskar/sphinx-action@7.0.0
+      - uses: ammaraskar/sphinx-action@8.1.3
         with:
           docs-folder: "docs/"
 

{schemathesis-4.0.0a1 → schemathesis-4.0.0a3}/.github/workflows/codeql-analysis.yml

@@ -38,7 +38,7 @@ jobs:
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3.28.0
+        uses: github/codeql-action/init@v3.28.1
         with:
           languages: ${{ matrix.language }}
           # If you wish to specify custom queries, you can do so here or in a config file.
@@ -49,7 +49,7 @@ jobs:
       # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
       # If this step fails, then you should remove it and run the build manually (see below)
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v3.28.0
+        uses: github/codeql-action/autobuild@v3.28.1
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 https://git.io/JvXDl
@@ -63,4 +63,4 @@ jobs:
       #   make release
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3.28.0
+        uses: github/codeql-action/analyze@v3.28.1

{schemathesis-4.0.0a1 → schemathesis-4.0.0a3}/.github/workflows/release.yml

@@ -21,6 +21,17 @@ jobs:
     steps:
       - uses: actions/checkout@v4.2.1
 
+      - name: Detect pre-release
+        id: detect-prerelease
+        run: |
+          TAG_NAME="${{ github.event.release.tag_name }}"
+          if [ "${{ github.event.release.prerelease }}" == "true" ] || [[ "$TAG_NAME" =~ -(alpha|beta|rc)[0-9]*$ ]]; then
+            echo "is_prerelease=true" >> $GITHUB_ENV
+          else
+            echo "is_prerelease=false" >> $GITHUB_ENV
+          fi
+        shell: bash
+
       - name: Docker meta
         id: meta
         uses: docker/metadata-action@v5
@@ -33,7 +44,7 @@ jobs:
           tags: |
             type=semver,pattern={{raw}}${{ matrix.tag-suffix }}
             type=semver,pattern={{version}}${{ matrix.tag-suffix }}
-            type=raw,value=stable${{ matrix.tag-suffix }}
+            ${{ env.is_prerelease == 'false' && 'type=raw,value=stable${{ matrix.tag-suffix }}' || '' }}
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3

{schemathesis-4.0.0a1 → schemathesis-4.0.0a3}/.pre-commit-config.yaml

@@ -33,12 +33,12 @@ repos:
         additional_dependencies: [types-requests, types-PyYAML]
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.9.1
+    rev: v0.9.3
     hooks:
       - id: ruff-format
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.9.1
+    rev: v0.9.3
     hooks:
       - id: ruff
 

{schemathesis-4.0.0a1 → schemathesis-4.0.0a3}/.readthedocs.yml

@@ -9,3 +9,6 @@ python:
     - method: pip
       path: .
     - requirements: docs/requirements.txt
+
+sphinx:
+  configuration: docs/conf.py

{schemathesis-4.0.0a1 → schemathesis-4.0.0a3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: schemathesis
-Version: 4.0.0a1
+Version: 4.0.0a3
 Summary: Property-based testing framework for Open API and GraphQL based apps
 Project-URL: Documentation, https://schemathesis.readthedocs.io/en/stable/
 Project-URL: Changelog, https://schemathesis.readthedocs.io/en/stable/changelog.html

{schemathesis-4.0.0a1 → schemathesis-4.0.0a3}/benches/schema.py

@@ -225,7 +225,7 @@ def test_rewritten_components(raw_schema):
 def test_links_count(raw_schema):
     schema = schemathesis.openapi.from_dict(raw_schema)
 
-    _ = schema.links_count
+    _ = schema.statistic.links.total
 
 
 @pytest.mark.benchmark

{schemathesis-4.0.0a1 → schemathesis-4.0.0a3}/docs/changelog.rst

@@ -1,8 +1,57 @@
 Changelog
 =========
 
-:version:`Unreleased <v4.0.0a1...HEAD>` - TBD
----------------------------------------------
+:version:`Unreleased <v4.0.0-alpha.3...HEAD>` - TBD
+---------------------------------------------------
+
+.. _v4.0.0-alpha.3:
+
+:version:`4.0.0-alpha.3 <v4.0.0-alpha.2...v4.0.0-alpha.3>` - 2025-01-27
+-----------------------------------------------------------------------
+
+**Added**
+
+- Add ``LoadingStarted`` & ``LoadingFinished`` to the public API.
+- Display the random generator seed in CLI output.
+
+**Changed**:
+
+- Improve control over API calls in stateful testing to make test scenarios more diverse.
+- Improve error message in CLI when the schema has no Open API links.
+- Improve error message in CLI when the schema contains incorrect Open API links.
+- Display the number of covered, selected, and total Open API links in stateful testing output.
+- Vary random seed in subsequent state machine re-runs to avoid repeating all previous sequences.
+
+**Fixed**
+
+- Handling of complex regex patterns with multiple quantifiers to respect length constraints during test generation.
+- Internal error during the coverage phase if negated parameter has no ``type``.
+- Internal error in CLI if run with ``PYTHONIOENCODING`` environment variable that is not ``utf8``.
+- Overriding of multiple incoming links defined for the same status code in OpenAPI specification.
+- Support for API links where operations can form a cycle (e.g., DELETE -> POST -> DELETE).
+- Incorrect Open API link selection when target operations are filtered out.
+- False positive in the ``ensure_resource_availability`` check.
+- Calculating the number of Open API links selected for testing.
+
+.. _v4.0.0-alpha.2:
+
+:version:`4.0.0-alpha.2 <v4.0.0a1...v4.0.0-alpha.2>` - 2025-01-20
+-----------------------------------------------------------------
+
+**Added**
+
+- Improved visibility into Open API link extraction success/failure status :issue:`823`
+
+**Changed**
+
+- Unified test progress display with multi-spinner interface and single progress bar :issue:`2372`
+- Optimized stateful testing by generating only required test data for Open API links.
+- Cleaner display of schema loading errors.
+
+**Fixed**
+
+- More accurate exception deduplication based on source location instead of messages.
+- Stricter validation of ``--include-*`` and ``--exclude-*`` CLI options.
 
 .. _v4.0.0a1:
 
@@ -61,7 +110,8 @@ A detailed migration guide and complete changelog will follow.
 - Support for ``pytest<7.0``.
 - CLI Options: ``--endpoint``, ``--method``, ``--tag``, ``--operation-id``, ``--skip-deprecated-operations``,
   ``--show-trace``, ``--debug-output-file``, ``--hypothesis-deadline``, ``--hypothesis-report-multiple-bugs``,
-  ``--hypothesis-verbosity``, ``--store-network-log``, ``--pre-run``, ``--dry-run``.
+  ``--hypothesis-verbosity``, ``--store-network-log``, ``--pre-run``, ``--dry-run``, ``--contrib-openapi-formats-uuid``,
+  ``--validate-schema``.
 - Most loader configuration moved to ``schema.configure`` method.
 - ``add_case`` hook.
 - ``schemathesis.contrib.unique_data``.

{schemathesis-4.0.0a1 → schemathesis-4.0.0a3}/docs/continuous_integration.rst

@@ -20,13 +20,6 @@ For initial setup in a GitHub Actions workflow:
       - uses: schemathesis/action@v1
         with:
           schema: 'http://127.0.0.1:5000/api/openapi.json'
-          # Optional, for PR comments
-          token: ${{ secrets.SCHEMATHESIS_TOKEN }}
-
-To get test results as comments in your GitHub pull requests, both the ``token`` and the `GitHub app`_ are required.
-Obtain the token by signing up on `Schemathesis.io <https://app.schemathesis.io/auth/sign-up/?utm_source=oss_docs&utm_content=ci>`_.
-
-.. image:: https://raw.githubusercontent.com/schemathesis/schemathesis/master/img/service_github_report.png
 
 To add headers like ``Authorization``:
 

{schemathesis-4.0.0a1 → schemathesis-4.0.0a3}/docs/getting-started.rst

@@ -77,16 +77,6 @@ Run Schemathesis tests as a part of your CI/CD pipeline. Add this YAML configura
 
 For more details, check out our `GitHub Action <https://github.com/schemathesis/action>`_ repository.
 
-GitHub App
-^^^^^^^^^^
-
-Receive automatic comments in your pull requests and updates on GitHub checks status. Requires usage of our `SaaS platform <https://app.schemathesis.io/auth/sign-up/?utm_source=oss_docs&utm_content=index_note>`_.
-
-1. Install the `GitHub app <https://github.com/apps/schemathesis>`_.
-2. Enable in your repository settings.
-
-.. image:: https://raw.githubusercontent.com/schemathesis/schemathesis/master/img/service_github_report.png
-
 Service
 -------
 

{schemathesis-4.0.0a1 → schemathesis-4.0.0a3}/example/tests/extensions.py

@@ -1,5 +1,3 @@
-from datetime import timedelta
-
 from hypothesis import strategies as st
 
 import schemathesis
@@ -25,7 +23,7 @@ def filter_body(ctx, body):
 @schemathesis.check
 def not_so_slow(ctx, response, case):
     """Custom response check."""
-    assert response.elapsed < timedelta(milliseconds=100), "Response is slow!"
+    assert response.elapsed < 100, "Response is slow!"
 
 
 @schemathesis.target

{schemathesis-4.0.0a1 → schemathesis-4.0.0a3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "schemathesis"
-version = "4.0.0a1"
+version = "4.0.0-alpha.3"
 description = "Property-based testing framework for Open API and GraphQL based apps"
 keywords = ["pytest", "hypothesis", "openapi", "graphql", "testing"]
 authors = [{ name = "Dmitry Dygalo", email = "dmitry@dygalo.dev" }]

{schemathesis-4.0.0a1 → schemathesis-4.0.0a3}/src/schemathesis/checks.py

@@ -4,6 +4,7 @@ import json
 from typing import TYPE_CHECKING, Any, Callable, Iterable, Iterator, Optional
 
 from schemathesis.core.failures import (
+    CustomFailure,
     Failure,
     FailureGroup,
     MalformedJson,
@@ -72,7 +73,7 @@ class CheckContext:
 
     def record_case(self, *, parent_id: str, case: Case) -> None:
         if self.recorder is not None:
-            self.recorder.record_case(parent_id=parent_id, case=case)
+            self.recorder.record_case(parent_id=parent_id, transition=None, case=case)
 
     def record_response(self, *, case_id: str, response: Response) -> None:
         if self.recorder is not None:
@@ -135,10 +136,11 @@ def run_checks(
         except Failure as failure:
             on_failure(name, collected, failure.with_traceback(None))
         except AssertionError as exc:
-            custom_failure = Failure.from_assertion(
-                name=name,
+            custom_failure = CustomFailure(
                 operation=case.operation.label,
-                exc=exc,
+                title=f"Custom check failed: `{name}`",
+                message=str(exc),
+                exception=exc,
             )
             on_failure(name, collected, custom_failure)
         except FailureGroup as group:

{schemathesis-4.0.0a1 → schemathesis-4.0.0a3}/src/schemathesis/cli/__init__.py

@@ -1,11 +1,22 @@
 from __future__ import annotations
 
 from schemathesis.cli.commands import Group, run, schemathesis
+from schemathesis.cli.commands.run.context import ExecutionContext
+from schemathesis.cli.commands.run.events import LoadingFinished, LoadingStarted
 from schemathesis.cli.commands.run.executor import handler
 from schemathesis.cli.commands.run.handlers import EventHandler
 from schemathesis.cli.ext.groups import GROUPS
 
-__all__ = ["schemathesis", "run", "EventHandler", "add_group", "handler"]
+__all__ = [
+    "schemathesis",
+    "run",
+    "EventHandler",
+    "ExecutionContext",
+    "LoadingStarted",
+    "LoadingFinished",
+    "add_group",
+    "handler",
+]
 
 
 def add_group(name: str, *, index: int | None = None) -> Group:

{schemathesis-4.0.0a1 → schemathesis-4.0.0a3}/src/schemathesis/cli/commands/run/__init__.py

@@ -91,7 +91,7 @@ DEFAULT_PHASES = ("unit", "stateful")
     help="Comma-separated list of checks to run against API responses",
     type=RegistryChoice(CHECKS, with_all=True),
     default=("not_a_server_error",),
-    callback=validation.convert_checks,
+    callback=validation.reduce_list,
     show_default=True,
     metavar="",
 )
@@ -102,7 +102,7 @@ DEFAULT_PHASES = ("unit", "stateful")
     help="Comma-separated list of checks to skip during testing",
     type=RegistryChoice(CHECKS, with_all=True),
     default=(),
-    callback=validation.convert_checks,
+    callback=validation.reduce_list,
     show_default=True,
     metavar="",
 )
@@ -356,7 +356,7 @@ DEFAULT_PHASES = ("unit", "stateful")
     help="Guide input generation to values more likely to expose bugs via targeted property-based testing",
     type=RegistryChoice(TARGETS),
     default=None,
-    callback=validation.convert_checks,
+    callback=validation.reduce_list,
     show_default=True,
     metavar="TARGET",
 )
@@ -588,7 +588,7 @@ def run(
         max_response_time=max_response_time,
     ).into()
 
-    if exit_first:
+    if exit_first and max_failures is None:
         max_failures = 1
 
     cassette_config = None

{schemathesis-4.0.0a1 → schemathesis-4.0.0a3}/src/schemathesis/cli/commands/run/events.py

@@ -3,7 +3,7 @@ import uuid
 
 from schemathesis.core import Specification
 from schemathesis.engine import events
-from schemathesis.schemas import ApiOperationsCount
+from schemathesis.schemas import ApiStatistic
 
 
 class LoadingStarted(events.EngineEvent):
@@ -16,15 +16,28 @@ class LoadingStarted(events.EngineEvent):
 
 
 class LoadingFinished(events.EngineEvent):
-    __slots__ = ("id", "timestamp", "location", "duration", "base_url", "specification", "operations_count")
+    __slots__ = (
+        "id",
+        "timestamp",
+        "location",
+        "duration",
+        "base_url",
+        "base_path",
+        "specification",
+        "statistic",
+        "schema",
+    )
 
     def __init__(
         self,
+        *,
         location: str,
         start_time: float,
         base_url: str,
+        base_path: str,
         specification: Specification,
-        operations_count: ApiOperationsCount,
+        statistic: ApiStatistic,
+        schema: dict,
     ) -> None:
         self.id = uuid.uuid4()
         self.timestamp = time.time()
@@ -32,4 +45,6 @@ class LoadingFinished(events.EngineEvent):
         self.duration = self.timestamp - start_time
         self.base_url = base_url
         self.specification = specification
-        self.operations_count = operations_count
+        self.statistic = statistic
+        self.schema = schema
+        self.base_path = base_path

{schemathesis-4.0.0a1 → schemathesis-4.0.0a3}/src/schemathesis/cli/commands/run/executor.py

@@ -19,7 +19,7 @@ from schemathesis.core.errors import LoaderError
 from schemathesis.core.output import OutputConfig
 from schemathesis.engine import from_schema
 from schemathesis.engine.config import EngineConfig
-from schemathesis.engine.events import EventGenerator, FatalError
+from schemathesis.engine.events import EventGenerator, FatalError, Interrupted
 from schemathesis.filters import FilterSet
 
 CUSTOM_HANDLERS: list[type[EventHandler]] = []
@@ -70,6 +70,9 @@ def into_event_stream(config: RunConfig) -> EventGenerator:
     try:
         schema = load_schema(loader_config)
         schema.filter_set = config.filter_set
+    except KeyboardInterrupt:
+        yield Interrupted(phase=None)
+        return
     except LoaderError as exc:
         yield FatalError(exception=exc)
         return
@@ -79,7 +82,9 @@ def into_event_stream(config: RunConfig) -> EventGenerator:
         start_time=loading_started.timestamp,
         base_url=schema.get_base_url(),
         specification=schema.specification,
-        operations_count=schema.count_operations(),
+        statistic=schema.statistic,
+        schema=schema.raw_schema,
+        base_path=schema.base_path,
     )
 
     try:
@@ -101,6 +106,7 @@ def _execute(event_stream: EventGenerator, config: RunConfig) -> None:
     handlers.append(
         OutputHandler(
             workers_num=config.engine.execution.workers_num,
+            seed=config.engine.execution.seed,
             rate_limit=config.rate_limit,
             wait_for_schema=config.wait_for_schema,
             cassette_config=config.cassette,
@@ -112,7 +118,7 @@ def _execute(event_stream: EventGenerator, config: RunConfig) -> None:
 
     def shutdown() -> None:
         for _handler in handlers:
-            _handler.shutdown()
+            _handler.shutdown(ctx)
 
     for handler in handlers:
         handler.start(ctx)

{schemathesis-4.0.0a1 → schemathesis-4.0.0a3}/src/schemathesis/cli/commands/run/filters.py

@@ -1,11 +1,12 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
-from typing import Callable, Literal, Sequence
+from typing import Any, Callable, Literal, Sequence
 
 import click
 
 from schemathesis.cli.ext.groups import grouped_option
+from schemathesis.core.errors import IncorrectUsage
 from schemathesis.filters import FilterSet, expression_to_filter_function, is_deprecated
 
 
@@ -148,29 +149,24 @@ class FilterArguments:
         if exclude_by_function:
             filter_set.exclude(exclude_by_function)
         for name_ in self.exclude_name:
-            filter_set.exclude(name=name_)
+            apply_exclude_filter(filter_set, "name", name=name_)
         for method in self.exclude_method:
-            filter_set.exclude(method=method)
+            apply_exclude_filter(filter_set, "method", method=method)
         for path in self.exclude_path:
-            filter_set.exclude(path=path)
+            apply_exclude_filter(filter_set, "path", path=path)
         for tag in self.exclude_tag:
-            filter_set.exclude(tag=tag)
+            apply_exclude_filter(filter_set, "tag", tag=tag)
         for operation_id in self.exclude_operation_id:
-            filter_set.exclude(operation_id=operation_id)
-        if (
-            self.exclude_name_regex
-            or self.exclude_method_regex
-            or self.exclude_path_regex
-            or self.exclude_tag_regex
-            or self.exclude_operation_id_regex
+            apply_exclude_filter(filter_set, "operation-id", operation_id=operation_id)
+        for key, value, name in (
+            ("name_regex", self.exclude_name_regex, "name-regex"),
+            ("method_regex", self.exclude_method_regex, "method-regex"),
+            ("path_regex", self.exclude_path_regex, "path-regex"),
+            ("tag_regex", self.exclude_tag_regex, "tag-regex"),
+            ("operation_id_regex", self.exclude_operation_id_regex, "operation-id-regex"),
         ):
-            filter_set.exclude(
-                name_regex=self.exclude_name_regex,
-                method_regex=self.exclude_method_regex,
-                path_regex=self.exclude_path_regex,
-                tag_regex=self.exclude_tag_regex,
-                operation_id_regex=self.exclude_operation_id_regex,
-            )
+            if value:
+                apply_exclude_filter(filter_set, name, **{key: value})
 
         # Exclude deprecated operations
         if self.exclude_deprecated:
@@ -179,6 +175,18 @@ class FilterArguments:
         return filter_set
 
 
+def apply_exclude_filter(filter_set: FilterSet, option_name: str, **kwargs: Any) -> None:
+    """Apply an exclude filter with proper error handling."""
+    try:
+        filter_set.exclude(**kwargs)
+    except IncorrectUsage as e:
+        if str(e) == "Filter already exists":
+            raise click.UsageError(
+                f"Filter for {option_name} already exists. You can't simultaneously include and exclude the same thing."
+            ) from None
+        raise click.UsageError(str(e)) from None
+
+
 def validate_unique_filter(values: Sequence[str], arg_name: str) -> None:
     if len(values) != len(set(values)):
         duplicates = ",".join(sorted({value for value in values if values.count(value) > 1}))

{schemathesis-4.0.0a1 → schemathesis-4.0.0a3}/src/schemathesis/cli/commands/run/handlers/base.py

@@ -15,4 +15,4 @@ class EventHandler:
 
     def start(self, ctx: ExecutionContext) -> None: ...
 
-    def shutdown(self) -> None: ...
+    def shutdown(self, ctx: ExecutionContext) -> None: ...

{schemathesis-4.0.0a1 → schemathesis-4.0.0a3}/src/schemathesis/cli/commands/run/handlers/cassettes.py

@@ -77,10 +77,8 @@ class CassetteWriter(EventHandler):
     def handle_event(self, ctx: ExecutionContext, event: events.EngineEvent) -> None:
         if isinstance(event, events.ScenarioFinished):
             self.queue.put(Process(recorder=event.recorder))
-        elif isinstance(event, events.EngineFinished):
-            self.shutdown()
 
-    def shutdown(self) -> None:
+    def shutdown(self, ctx: ExecutionContext) -> None:
         self.queue.put(Finalize())
         self._stop_worker()