schemathesis 3.25.5__py3-none-any.whl → 4.0.0a1__py3-none-any.whl

schemathesis/__init__.py

@@ -1,82 +1,44 @@
 from __future__ import annotations
-from typing import Any
-
-from . import auths, checks, experimental, contrib, fixups, graphql, hooks, runner, serializers, targets  # noqa: E402
-from ._lazy_import import lazy_import
-from .generation import DataGenerationMethod, GenerationConfig  # noqa: E402
-from .constants import SCHEMATHESIS_VERSION  # noqa: E402
-from .models import Case  # noqa: E402
-from .specs import openapi  # noqa: E402
 
+from schemathesis import auths, contrib, engine, errors, experimental, graphql, hooks, openapi, pytest, python
+from schemathesis.checks import CheckContext, CheckFunction, check
+from schemathesis.core.output import OutputConfig, sanitization
+from schemathesis.core.transport import Response
+from schemathesis.core.version import SCHEMATHESIS_VERSION
+from schemathesis.generation import GenerationConfig, GenerationMode, HeaderConfig
+from schemathesis.generation.case import Case
+from schemathesis.generation.targets import TargetContext, TargetFunction, target
 
 __version__ = SCHEMATHESIS_VERSION
 
-# Default loaders
-from_aiohttp = openapi.from_aiohttp
-from_asgi = openapi.from_asgi
-from_dict = openapi.from_dict
-from_file = openapi.from_file
-from_path = openapi.from_path
-from_pytest_fixture = openapi.from_pytest_fixture
-from_uri = openapi.from_uri
-from_wsgi = openapi.from_wsgi
-
 # Public API
 auth = auths.GLOBAL_AUTH_STORAGE
-check = checks.register
 hook = hooks.register
-serializer = serializers.register
-target = targets.register
-
-# Backward compatibility
-register_check = checks.register
-register_target = targets.register
-register_string_format = openapi.format
 
 __all__ = [
-    "auths",
-    "checks",
-    "experimental",
-    "contrib",
-    "fixups",
-    "graphql",
-    "hooks",
-    "runner",
-    "serializers",
-    "targets",
-    "DataGenerationMethod",
-    "SCHEMATHESIS_VERSION",
     "Case",
-    "openapi",
+    "CheckContext",
+    "CheckFunction",
+    "GenerationMode",
+    "GenerationConfig",
+    "HeaderConfig",
+    "OutputConfig",
+    "Response",
+    "TargetContext",
+    "TargetFunction",
     "__version__",
-    "from_aiohttp",
-    "from_asgi",
-    "from_dict",
-    "from_file",
-    "from_path",
-    "from_pytest_fixture",
-    "from_uri",
-    "from_wsgi",
     "auth",
     "check",
+    "contrib",
+    "engine",
+    "errors",
+    "experimental",
+    "graphql",
     "hook",
-    "serializer",
+    "hooks",
+    "openapi",
+    "pytest",
+    "python",
+    "sanitization",
     "target",
-    "register_check",
-    "register_target",
-    "register_string_format",
 ]
-
-
-def _load_generic_response() -> Any:
-    from .transports.responses import GenericResponse
-
-    return GenericResponse
-
-
-_imports = {"GenericResponse": _load_generic_response}
-
-
-def __getattr__(name: str) -> Any:
-    # Some modules are relatively heavy, hence load them lazily to improve startup time for CLI
-    return lazy_import(__name__, name, _imports, globals())

schemathesis/auths.py

@@ -1,31 +1,34 @@
 """Support for custom API authentication mechanisms."""
+
 from __future__ import annotations
-import inspect
+
 import threading
 import time
-import warnings
 from dataclasses import dataclass, field
 from typing import (
     TYPE_CHECKING,
     Any,
     Callable,
     Generic,
+    Protocol,
     TypeVar,
+    Union,
     overload,
     runtime_checkable,
-    Protocol,
 )
 
-from .exceptions import UsageError
-from .filters import FilterSet, FilterValue, MatcherFunc, attach_filter_chain
-from .types import GenericTest
+from schemathesis.core.errors import IncorrectUsage
+from schemathesis.core.marks import Mark
+from schemathesis.filters import FilterSet, FilterValue, MatcherFunc, attach_filter_chain
+from schemathesis.generation.case import Case
 
 if TYPE_CHECKING:
-    from .models import APIOperation, Case
     import requests.auth
 
+    from schemathesis.schemas import APIOperation
+
 DEFAULT_REFRESH_INTERVAL = 300
-AUTH_STORAGE_ATTRIBUTE_NAME = "_schemathesis_auth"
+AuthStorageMark = Mark["AuthStorage"](attr_name="auth_storage")
 Auth = TypeVar("Auth")
 
 
@@ -41,6 +44,9 @@ class AuthContext:
     app: Any | None
 
 
+CacheKeyFunction = Callable[["Case", "AuthContext"], Union[str, int]]
+
+
 @runtime_checkable
 class AuthProvider(Generic[Auth], Protocol):
     """Get authentication data for an API and set it on the generated test cases."""
@@ -96,16 +102,24 @@ class CachingAuthProvider(Generic[Auth]):
 
     def get(self, case: Case, context: AuthContext) -> Auth | None:
         """Get cached auth value."""
-        if self.cache_entry is None or self.timer() >= self.cache_entry.expires:
+        cache_entry = self._get_cache_entry(case, context)
+        if cache_entry is None or self.timer() >= cache_entry.expires:
             with self._refresh_lock:
-                if not (self.cache_entry is None or self.timer() >= self.cache_entry.expires):
+                cache_entry = self._get_cache_entry(case, context)
+                if not (cache_entry is None or self.timer() >= cache_entry.expires):
                     # Another thread updated the cache
-                    return self.cache_entry.data
+                    return cache_entry.data
                 # We know that optional auth is possible only inside a higher-level wrapper
-                data: Auth = _provider_get(self.provider, case, context)  # type: ignore[assignment]
-                self.cache_entry = CacheEntry(data=data, expires=self.timer() + self.refresh_interval)
+                data: Auth = self.provider.get(case, context)  # type: ignore[assignment]
+                self._set_cache_entry(data, case, context)
                 return data
-        return self.cache_entry.data
+        return cache_entry.data
+
+    def _get_cache_entry(self, case: Case, context: AuthContext) -> CacheEntry[Auth] | None:
+        return self.cache_entry
+
+    def _set_cache_entry(self, data: Auth, case: Case, context: AuthContext) -> None:
+        self.cache_entry = CacheEntry(data=data, expires=self.timer() + self.refresh_interval)
 
     def set(self, case: Case, data: Auth, context: AuthContext) -> None:
         """Set auth data on the `Case` instance.
@@ -115,11 +129,29 @@ class CachingAuthProvider(Generic[Auth]):
         self.provider.set(case, data, context)
 
 
+def _noop_key_function(case: Case, context: AuthContext) -> str:
+    # Never used
+    raise NotImplementedError
+
+
+@dataclass
+class KeyedCachingAuthProvider(CachingAuthProvider[Auth]):
+    cache_by_key: CacheKeyFunction = _noop_key_function
+    cache_entries: dict[str | int, CacheEntry[Auth] | None] = field(default_factory=dict)
+
+    def _get_cache_entry(self, case: Case, context: AuthContext) -> CacheEntry[Auth] | None:
+        key = self.cache_by_key(case, context)
+        return self.cache_entries.get(key)
+
+    def _set_cache_entry(self, data: Auth, case: Case, context: AuthContext) -> None:
+        key = self.cache_by_key(case, context)
+        self.cache_entries[key] = CacheEntry(data=data, expires=self.timer() + self.refresh_interval)
+
+
 class FilterableRegisterAuth(Protocol):
     """Protocol that adds filters to the return value of `register`."""
 
-    def __call__(self, provider_class: type[AuthProvider]) -> type[AuthProvider]:
-        pass
+    def __call__(self, provider_class: type[AuthProvider]) -> type[AuthProvider]: ...
 
     def apply_to(
         self,
@@ -131,8 +163,7 @@ class FilterableRegisterAuth(Protocol):
         method_regex: str | None = None,
         path: FilterValue | None = None,
         path_regex: str | None = None,
-    ) -> FilterableRegisterAuth:
-        pass
+    ) -> FilterableRegisterAuth: ...
 
     def skip_for(
         self,
@@ -144,15 +175,13 @@ class FilterableRegisterAuth(Protocol):
         method_regex: str | None = None,
         path: FilterValue | None = None,
         path_regex: str | None = None,
-    ) -> FilterableRegisterAuth:
-        pass
+    ) -> FilterableRegisterAuth: ...
 
 
 class FilterableApplyAuth(Protocol):
     """Protocol that adds filters to the return value of `apply`."""
 
-    def __call__(self, test: GenericTest) -> GenericTest:
-        pass
+    def __call__(self, test: Callable) -> Callable: ...
 
     def apply_to(
         self,
@@ -164,8 +193,7 @@ class FilterableApplyAuth(Protocol):
         method_regex: str | None = None,
         path: FilterValue | None = None,
         path_regex: str | None = None,
-    ) -> FilterableApplyAuth:
-        pass
+    ) -> FilterableApplyAuth: ...
 
     def skip_for(
         self,
@@ -177,8 +205,7 @@ class FilterableApplyAuth(Protocol):
         method_regex: str | None = None,
         path: FilterValue | None = None,
         path_regex: str | None = None,
-    ) -> FilterableApplyAuth:
-        pass
+    ) -> FilterableApplyAuth: ...
 
 
 class FilterableRequestsAuth(Protocol):
@@ -194,8 +221,7 @@ class FilterableRequestsAuth(Protocol):
         method_regex: str | None = None,
         path: FilterValue | None = None,
         path_regex: str | None = None,
-    ) -> FilterableRequestsAuth:
-        pass
+    ) -> FilterableRequestsAuth: ...
 
     def skip_for(
         self,
@@ -207,8 +233,7 @@ class FilterableRequestsAuth(Protocol):
         method_regex: str | None = None,
         path: FilterValue | None = None,
         path_regex: str | None = None,
-    ) -> FilterableRequestsAuth:
-        pass
+    ) -> FilterableRequestsAuth: ...
 
 
 @dataclass
@@ -220,7 +245,7 @@ class SelectiveAuthProvider(Generic[Auth]):
 
     def get(self, case: Case, context: AuthContext) -> Auth | None:
         if self.filter_set.match(context):
-            return _provider_get(self.provider, case, context)
+            return self.provider.get(case, context)
         return None
 
     def set(self, case: Case, data: Auth, context: AuthContext) -> None:
@@ -243,8 +268,8 @@ class AuthStorage(Generic[Auth]):
         self,
         *,
         refresh_interval: int | None = DEFAULT_REFRESH_INTERVAL,
-    ) -> FilterableRegisterAuth:
-        pass
+        cache_by_key: CacheKeyFunction | None = None,
+    ) -> FilterableRegisterAuth: ...
 
     @overload
     def __call__(
@@ -252,26 +277,26 @@ class AuthStorage(Generic[Auth]):
         provider_class: type[AuthProvider],
         *,
         refresh_interval: int | None = DEFAULT_REFRESH_INTERVAL,
-    ) -> FilterableApplyAuth:
-        pass
+        cache_by_key: CacheKeyFunction | None = None,
+    ) -> FilterableApplyAuth: ...
 
     def __call__(
         self,
         provider_class: type[AuthProvider] | None = None,
         *,
         refresh_interval: int | None = DEFAULT_REFRESH_INTERVAL,
+        cache_by_key: CacheKeyFunction | None = None,
     ) -> FilterableRegisterAuth | FilterableApplyAuth:
         if provider_class is not None:
-            return self.apply(provider_class, refresh_interval=refresh_interval)
-        return self.register(refresh_interval=refresh_interval)
+            return self.apply(provider_class, refresh_interval=refresh_interval, cache_by_key=cache_by_key)
+        return self.register(refresh_interval=refresh_interval, cache_by_key=cache_by_key)
 
     def set_from_requests(self, auth: requests.auth.AuthBase) -> FilterableRequestsAuth:
         """Use `requests` auth instance as an auth provider."""
         filter_set = FilterSet()
         self.providers.append(SelectiveAuthProvider(provider=RequestsAuth(auth), filter_set=filter_set))
 
-        class _FilterableRequestsAuth:
-            pass
+        class _FilterableRequestsAuth: ...
 
         attach_filter_chain(_FilterableRequestsAuth, "apply_to", filter_set.include)
         attach_filter_chain(_FilterableRequestsAuth, "skip_for", filter_set.exclude)
@@ -283,6 +308,7 @@ class AuthStorage(Generic[Auth]):
         *,
         provider_class: type[AuthProvider],
         refresh_interval: int | None = DEFAULT_REFRESH_INTERVAL,
+        cache_by_key: CacheKeyFunction | None = None,
         filter_set: FilterSet,
     ) -> None:
         if not issubclass(provider_class, AuthProvider):
@@ -292,16 +318,27 @@ class AuthStorage(Generic[Auth]):
             )
         provider: AuthProvider
         # Apply caching if desired
+        instance = provider_class()
         if refresh_interval is not None:
-            provider = CachingAuthProvider(provider_class(), refresh_interval=refresh_interval)
+            if cache_by_key is None:
+                provider = CachingAuthProvider(instance, refresh_interval=refresh_interval)
+            else:
+                provider = KeyedCachingAuthProvider(
+                    instance, refresh_interval=refresh_interval, cache_by_key=cache_by_key
+                )
         else:
-            provider = provider_class()
+            provider = instance
         # Store filters if any
         if not filter_set.is_empty():
             provider = SelectiveAuthProvider(provider, filter_set)
         self.providers.append(provider)
 
-    def register(self, *, refresh_interval: int | None = DEFAULT_REFRESH_INTERVAL) -> FilterableRegisterAuth:
+    def register(
+        self,
+        *,
+        refresh_interval: int | None = DEFAULT_REFRESH_INTERVAL,
+        cache_by_key: CacheKeyFunction | None = None,
+    ) -> FilterableRegisterAuth:
         """Register a new auth provider.
 
         .. code-block:: python
@@ -323,7 +360,12 @@ class AuthStorage(Generic[Auth]):
         filter_set = FilterSet()
 
         def wrapper(provider_class: type[AuthProvider]) -> type[AuthProvider]:
-            self._set_provider(provider_class=provider_class, refresh_interval=refresh_interval, filter_set=filter_set)
+            self._set_provider(
+                provider_class=provider_class,
+                refresh_interval=refresh_interval,
+                filter_set=filter_set,
+                cache_by_key=cache_by_key,
+            )
             return provider_class
 
         attach_filter_chain(wrapper, "apply_to", filter_set.include)
@@ -339,7 +381,11 @@ class AuthStorage(Generic[Auth]):
         self.providers = []
 
     def apply(
-        self, provider_class: type[AuthProvider], *, refresh_interval: int | None = DEFAULT_REFRESH_INTERVAL
+        self,
+        provider_class: type[AuthProvider],
+        *,
+        refresh_interval: int | None = DEFAULT_REFRESH_INTERVAL,
+        cache_by_key: CacheKeyFunction | None = None,
     ) -> FilterableApplyAuth:
         """Register auth provider only on one test function.
 
@@ -360,10 +406,16 @@ class AuthStorage(Generic[Auth]):
         """
         filter_set = FilterSet()
 
-        def wrapper(test: GenericTest) -> GenericTest:
-            auth_storage = self.add_auth_storage(test)
+        def wrapper(test: Callable) -> Callable:
+            if AuthStorageMark.is_set(test):
+                raise IncorrectUsage(f"`{test.__name__}` has already been decorated with `apply`.")
+            auth_storage = self.__class__()
+            AuthStorageMark.set(test, auth_storage)
             auth_storage._set_provider(
-                provider_class=provider_class, refresh_interval=refresh_interval, filter_set=filter_set
+                provider_class=provider_class,
+                refresh_interval=refresh_interval,
+                filter_set=filter_set,
+                cache_by_key=cache_by_key,
             )
             return test
 
@@ -372,45 +424,18 @@ class AuthStorage(Generic[Auth]):
 
         return wrapper  # type: ignore[return-value]
 
-    @classmethod
-    def add_auth_storage(cls, test: GenericTest) -> AuthStorage:
-        """Attach a new auth storage instance to the test if it is not already present."""
-        if not hasattr(test, AUTH_STORAGE_ATTRIBUTE_NAME):
-            setattr(test, AUTH_STORAGE_ATTRIBUTE_NAME, cls())
-        else:
-            raise UsageError(f"`{test.__name__}` has already been decorated with `apply`.")
-        return getattr(test, AUTH_STORAGE_ATTRIBUTE_NAME)
-
     def set(self, case: Case, context: AuthContext) -> None:
         """Set authentication data on a generated test case."""
         if not self.is_defined:
-            raise UsageError("No auth provider is defined.")
+            raise IncorrectUsage("No auth provider is defined.")
         for provider in self.providers:
-            data: Auth | None = _provider_get(provider, case, context)
+            data: Auth | None = provider.get(case, context)
             if data is not None:
                 provider.set(case, data, context)
+                case._has_explicit_auth = True
                 break
 
 
-def _provider_get(auth_provider: AuthProvider, case: Case, context: AuthContext) -> Auth | None:
-    # A shim to provide a compatibility layer between previously used convention for `AuthProvider.get`
-    # where it used to accept a single `context` argument
-    method = auth_provider.get
-    parameters = inspect.signature(method).parameters
-    if len(parameters) == 1:
-        # Old calling convention
-        warnings.warn(
-            "The method 'get' of your AuthProvider is using the old calling convention, "
-            "which is deprecated and will be removed in Schemathesis 4.0. "
-            "Please update it to accept both 'case' and 'context' as arguments.",
-            DeprecationWarning,
-            stacklevel=1,
-        )
-        return method(context)  # type: ignore
-    # New calling convention
-    return method(case, context)
-
-
 def set_on_case(case: Case, context: AuthContext, auth_storage: AuthStorage | None) -> None:
     """Set authentication data on this case.
 
@@ -424,11 +449,6 @@ def set_on_case(case: Case, context: AuthContext, auth_storage: AuthStorage | No
         GLOBAL_AUTH_STORAGE.set(case, context)
 
 
-def get_auth_storage_from_test(test: GenericTest) -> AuthStorage | None:
-    """Extract the currently attached auth storage from a test function."""
-    return getattr(test, AUTH_STORAGE_ATTRIBUTE_NAME, None)
-
-
 # Global auth API
 GLOBAL_AUTH_STORAGE: AuthStorage = AuthStorage()
 register = GLOBAL_AUTH_STORAGE.register

schemathesis/checks.py

@@ -1,68 +1,148 @@
 from __future__ import annotations
-from typing import TYPE_CHECKING
+
 import json
+from typing import TYPE_CHECKING, Any, Callable, Iterable, Iterator, Optional
 
-from . import failures
-from .exceptions import get_server_error, get_response_parsing_error
-from .specs.openapi.checks import (
-    content_type_conformance,
-    response_headers_conformance,
-    response_schema_conformance,
-    status_code_conformance,
+from schemathesis.core.failures import (
+    Failure,
+    FailureGroup,
+    MalformedJson,
+    MaxResponseTimeConfig,
+    ResponseTimeExceeded,
+    ServerError,
 )
+from schemathesis.core.registries import Registry
+from schemathesis.core.transport import Response
+from schemathesis.generation.overrides import Override
 
 if TYPE_CHECKING:
-    from .transports.responses import GenericResponse
-    from .models import Case, CheckFunction
+    from requests.models import CaseInsensitiveDict
+
+    from schemathesis.engine.recorder import ScenarioRecorder
+    from schemathesis.generation.case import Case
+
+CheckFunction = Callable[["CheckContext", "Response", "Case"], Optional[bool]]
+ChecksConfig = dict[CheckFunction, Any]
+
+
+class CheckContext:
+    """Context for Schemathesis checks.
+
+    Provides access to broader test execution data beyond individual test cases.
+    """
+
+    override: Override | None
+    auth: tuple[str, str] | None
+    headers: CaseInsensitiveDict | None
+    config: ChecksConfig
+    transport_kwargs: dict[str, Any] | None
+    recorder: ScenarioRecorder | None
+
+    __slots__ = ("override", "auth", "headers", "config", "transport_kwargs", "recorder")
+
+    def __init__(
+        self,
+        override: Override | None,
+        auth: tuple[str, str] | None,
+        headers: CaseInsensitiveDict | None,
+        config: ChecksConfig,
+        transport_kwargs: dict[str, Any] | None,
+        recorder: ScenarioRecorder | None = None,
+    ) -> None:
+        self.override = override
+        self.auth = auth
+        self.headers = headers
+        self.config = config
+        self.transport_kwargs = transport_kwargs
+        self.recorder = recorder
+
+    def find_parent(self, *, case_id: str) -> Case | None:
+        if self.recorder is not None:
+            return self.recorder.find_parent(case_id=case_id)
+        return None
+
+    def find_related(self, *, case_id: str) -> Iterator[Case]:
+        if self.recorder is not None:
+            yield from self.recorder.find_related(case_id=case_id)
 
+    def find_response(self, *, case_id: str) -> Response | None:
+        if self.recorder is not None:
+            return self.recorder.find_response(case_id=case_id)
+        return None
 
-def not_a_server_error(response: GenericResponse, case: Case) -> bool | None:
+    def record_case(self, *, parent_id: str, case: Case) -> None:
+        if self.recorder is not None:
+            self.recorder.record_case(parent_id=parent_id, case=case)
+
+    def record_response(self, *, case_id: str, response: Response) -> None:
+        if self.recorder is not None:
+            self.recorder.record_response(case_id=case_id, response=response)
+
+
+CHECKS = Registry[CheckFunction]()
+check = CHECKS.register
+
+
+@check
+def not_a_server_error(ctx: CheckContext, response: Response, case: Case) -> bool | None:
     """A check to verify that the response is not a server-side error."""
-    from .specs.graphql.schemas import GraphQLCase
+    from .specs.graphql.schemas import GraphQLSchema
     from .specs.graphql.validation import validate_graphql_response
-    from .transports.responses import get_json
 
     status_code = response.status_code
     if status_code >= 500:
-        exc_class = get_server_error(status_code)
-        raise exc_class(failures.ServerError.title, context=failures.ServerError(status_code=status_code))
-    if isinstance(case, GraphQLCase):
+        raise ServerError(operation=case.operation.label, status_code=status_code)
+    if isinstance(case.operation.schema, GraphQLSchema):
         try:
-            data = get_json(response)
-            validate_graphql_response(data)
+            data = response.json()
+            validate_graphql_response(case, data)
         except json.JSONDecodeError as exc:
-            exc_class = get_response_parsing_error(exc)
-            context = failures.JSONDecodeErrorContext.from_exception(exc)
-            raise exc_class(context.title, context=context) from exc
+            raise MalformedJson.from_exception(operation=case.operation.label, exc=exc) from None
     return None
 
 
-DEFAULT_CHECKS: tuple[CheckFunction, ...] = (not_a_server_error,)
-OPTIONAL_CHECKS = (
-    status_code_conformance,
-    content_type_conformance,
-    response_headers_conformance,
-    response_schema_conformance,
-)
-ALL_CHECKS: tuple[CheckFunction, ...] = DEFAULT_CHECKS + OPTIONAL_CHECKS
-
-
-def register(check: CheckFunction) -> CheckFunction:
-    """Register a new check for schemathesis CLI.
-
-    :param check: A function to validate API responses.
+def max_response_time(ctx: CheckContext, response: Response, case: Case) -> bool | None:
+    config = ctx.config.get(max_response_time, MaxResponseTimeConfig())
+    elapsed = response.elapsed
+    if elapsed > config.limit:
+        raise ResponseTimeExceeded(
+            operation=case.operation.label,
+            message=f"Actual: {elapsed:.2f}ms\nLimit: {config.limit * 1000:.2f}ms",
+            elapsed=elapsed,
+            deadline=config.limit,
+        )
+    return None
 
-    .. code-block:: python
 
-        @schemathesis.check
-        def new_check(response, case):
-            # some awesome assertions!
-            ...
-    """
-    from . import cli
+def run_checks(
+    *,
+    case: Case,
+    response: Response,
+    ctx: CheckContext,
+    checks: Iterable[CheckFunction],
+    on_failure: Callable[[str, set[Failure], Failure], None],
+    on_success: Callable[[str, Case], None] | None = None,
+) -> set[Failure]:
+    """Run a set of checks against a response."""
+    collected: set[Failure] = set()
 
-    global ALL_CHECKS
+    for check in checks:
+        name = check.__name__
+        try:
+            skip_check = check(ctx, response, case)
+            if not skip_check and on_success:
+                on_success(name, case)
+        except Failure as failure:
+            on_failure(name, collected, failure.with_traceback(None))
+        except AssertionError as exc:
+            custom_failure = Failure.from_assertion(
+                name=name,
+                operation=case.operation.label,
+                exc=exc,
+            )
+            on_failure(name, collected, custom_failure)
+        except FailureGroup as group:
+            for sub_failure in group.exceptions:
+                on_failure(name, collected, sub_failure)
 
-    ALL_CHECKS += (check,)
-    cli.CHECKS_TYPE.choices += (check.__name__,)  # type: ignore
-    return check
+    return collected