schemathesis 4.0.0a3__py3-none-any.whl → 4.0.0a5__py3-none-any.whl

schemathesis/engine/phases/stateful/_executor.py

@@ -134,7 +134,7 @@ def execute_state_machine_loop(
             return result
 
         def validate_response(
-            self, response: Response, case: Case, additional_checks: tuple[CheckFunction, ...] = ()
+            self, response: Response, case: Case, additional_checks: tuple[CheckFunction, ...] = (), **kwargs: Any
         ) -> None:
             self.recorder.record_response(case_id=case.id, response=response)
             ctx.collect_metric(case, response)

schemathesis/engine/phases/unit/__init__.py

@@ -11,11 +11,12 @@ import warnings
 from queue import Queue
 from typing import TYPE_CHECKING, Any
 
+from schemathesis.core.errors import InvalidSchema
 from schemathesis.core.result import Ok
 from schemathesis.engine import Status, events
 from schemathesis.engine.phases import PhaseName, PhaseSkipReason
 from schemathesis.engine.recorder import ScenarioRecorder
-from schemathesis.generation.hypothesis.builder import HypothesisTestConfig
+from schemathesis.generation.hypothesis.builder import HypothesisTestConfig, HypothesisTestMode
 from schemathesis.generation.hypothesis.reporting import ignore_hypothesis_output
 
 from ._pool import TaskProducer, WorkerPool
@@ -34,6 +35,12 @@ def execute(engine: EngineContext, phase: Phase) -> events.EventGenerator:
     Implemented as a producer-consumer pattern via a task queue.
     The main thread provides an iterator over API operations and worker threads create test functions and run them.
     """
+    if phase.name == PhaseName.EXAMPLES:
+        mode = HypothesisTestMode.EXAMPLES
+    elif phase.name == PhaseName.COVERAGE:
+        mode = HypothesisTestMode.COVERAGE
+    else:
+        mode = HypothesisTestMode.FUZZING
     producer = TaskProducer(engine)
     workers_num = engine.config.execution.workers_num
 
@@ -45,7 +52,13 @@ def execute(engine: EngineContext, phase: Phase) -> events.EventGenerator:
     is_executed = False
 
     with WorkerPool(
-        workers_num=workers_num, producer=producer, worker_factory=worker_task, ctx=engine, suite_id=suite_started.id
+        workers_num=workers_num,
+        producer=producer,
+        worker_factory=worker_task,
+        ctx=engine,
+        mode=mode,
+        phase=phase.name,
+        suite_id=suite_started.id,
     ) as pool:
         try:
             while True:
@@ -74,7 +87,7 @@ def execute(engine: EngineContext, phase: Phase) -> events.EventGenerator:
         except KeyboardInterrupt:
             engine.stop()
             status = Status.INTERRUPTED
-            yield events.Interrupted(phase=PhaseName.UNIT_TESTING)
+            yield events.Interrupted(phase=phase.name)
 
     if not is_executed:
         phase.skip_reason = PhaseSkipReason.NOTHING_TO_TEST
@@ -86,13 +99,52 @@ def execute(engine: EngineContext, phase: Phase) -> events.EventGenerator:
     yield events.PhaseFinished(phase=phase, status=status, payload=None)
 
 
-def worker_task(*, events_queue: Queue, producer: TaskProducer, ctx: EngineContext, suite_id: uuid.UUID) -> None:
-    from hypothesis.errors import HypothesisWarning
+def worker_task(
+    *,
+    events_queue: Queue,
+    producer: TaskProducer,
+    ctx: EngineContext,
+    mode: HypothesisTestMode,
+    phase: PhaseName,
+    suite_id: uuid.UUID,
+) -> None:
+    from hypothesis.errors import HypothesisWarning, InvalidArgument
 
     from schemathesis.generation.hypothesis.builder import create_test
 
     from ._executor import run_test, test_func
 
+    def on_error(error: Exception, *, method: str | None = None, path: str | None = None) -> None:
+        if method and path:
+            label = f"{method.upper()} {path}"
+            scenario_started = events.ScenarioStarted(label=label, phase=phase, suite_id=suite_id)
+            events_queue.put(scenario_started)
+
+            events_queue.put(events.NonFatalError(error=error, phase=phase, label=label, related_to_operation=True))
+
+            events_queue.put(
+                events.ScenarioFinished(
+                    id=scenario_started.id,
+                    suite_id=suite_id,
+                    phase=phase,
+                    label=label,
+                    status=Status.ERROR,
+                    recorder=ScenarioRecorder(label="Error"),
+                    elapsed_time=0.0,
+                    skip_reason=None,
+                    is_final=True,
+                )
+            )
+        else:
+            events_queue.put(
+                events.NonFatalError(
+                    error=error,
+                    phase=phase,
+                    label=path or "-",
+                    related_to_operation=False,
+                )
+            )
+
     warnings.filterwarnings("ignore", message="The recursion limit will not be reset", category=HypothesisWarning)
     with ignore_hypothesis_output():
         try:
@@ -104,62 +156,34 @@ def worker_task(*, events_queue: Queue, producer: TaskProducer, ctx: EngineConte
                 if isinstance(result, Ok):
                     operation = result.ok()
                     as_strategy_kwargs = get_strategy_kwargs(ctx, operation)
-                    test_function = create_test(
-                        operation=operation,
-                        test_func=test_func,
-                        config=HypothesisTestConfig(
-                            settings=ctx.config.execution.hypothesis_settings,
-                            seed=ctx.config.execution.seed,
-                            generation=ctx.config.execution.generation,
-                            as_strategy_kwargs=as_strategy_kwargs,
-                        ),
-                    )
+                    try:
+                        test_function = create_test(
+                            operation=operation,
+                            test_func=test_func,
+                            config=HypothesisTestConfig(
+                                modes=[mode],
+                                settings=ctx.config.execution.hypothesis_settings,
+                                seed=ctx.config.execution.seed,
+                                generation=ctx.config.execution.generation,
+                                as_strategy_kwargs=as_strategy_kwargs,
+                            ),
+                        )
+                    except (InvalidSchema, InvalidArgument) as exc:
+                        on_error(exc, method=operation.method, path=operation.path)
+                        continue
 
                     # The test is blocking, meaning that even if CTRL-C comes to the main thread, this tasks will continue
                     # executing. However, as we set a stop event, it will be checked before the next network request.
                     # However, this is still suboptimal, as there could be slow requests and they will block for longer
-                    for event in run_test(operation=operation, test_function=test_function, ctx=ctx, suite_id=suite_id):
+                    for event in run_test(
+                        operation=operation, test_function=test_function, ctx=ctx, phase=phase, suite_id=suite_id
+                    ):
                         events_queue.put(event)
                 else:
                     error = result.err()
-                    if error.method:
-                        label = f"{error.method.upper()} {error.full_path}"
-                        scenario_started = events.ScenarioStarted(
-                            label=label, phase=PhaseName.UNIT_TESTING, suite_id=suite_id
-                        )
-                        events_queue.put(scenario_started)
-
-                        events_queue.put(
-                            events.NonFatalError(
-                                error=error, phase=PhaseName.UNIT_TESTING, label=label, related_to_operation=True
-                            )
-                        )
-
-                        events_queue.put(
-                            events.ScenarioFinished(
-                                id=scenario_started.id,
-                                suite_id=suite_id,
-                                phase=PhaseName.UNIT_TESTING,
-                                label=label,
-                                status=Status.ERROR,
-                                recorder=ScenarioRecorder(label="Error"),
-                                elapsed_time=0.0,
-                                skip_reason=None,
-                                is_final=True,
-                            )
-                        )
-                    else:
-                        assert error.full_path is not None
-                        events_queue.put(
-                            events.NonFatalError(
-                                error=error,
-                                phase=PhaseName.UNIT_TESTING,
-                                label=error.full_path,
-                                related_to_operation=False,
-                            )
-                        )
+                    on_error(error, method=error.method, path=error.path)
         except KeyboardInterrupt:
-            events_queue.put(events.Interrupted(phase=PhaseName.UNIT_TESTING))
+            events_queue.put(events.Interrupted(phase=phase))
 
 
 def get_strategy_kwargs(ctx: EngineContext, operation: APIOperation) -> dict[str, Any]:

schemathesis/engine/phases/unit/_executor.py

@@ -52,13 +52,17 @@ if TYPE_CHECKING:
 
 
 def run_test(
-    *, operation: APIOperation, test_function: Callable, ctx: EngineContext, suite_id: uuid.UUID
+    *,
+    operation: APIOperation,
+    test_function: Callable,
+    ctx: EngineContext,
+    phase: PhaseName,
+    suite_id: uuid.UUID,
 ) -> events.EventGenerator:
     """A single test run with all error handling needed."""
     import hypothesis.errors
 
-    # To simplify connecting `before` and `after` events in external systems
-    scenario_started = events.ScenarioStarted(label=operation.label, phase=PhaseName.UNIT_TESTING, suite_id=suite_id)
+    scenario_started = events.ScenarioStarted(label=operation.label, phase=phase, suite_id=suite_id)
     yield scenario_started
     errors: list[Exception] = []
     skip_reason = None
@@ -66,8 +70,19 @@ def run_test(
     recorder = ScenarioRecorder(label=operation.label)
 
     def non_fatal_error(error: Exception) -> events.NonFatalError:
-        return events.NonFatalError(
-            error=error, phase=PhaseName.UNIT_TESTING, label=operation.label, related_to_operation=True
+        return events.NonFatalError(error=error, phase=phase, label=operation.label, related_to_operation=True)
+
+    def scenario_finished(status: Status) -> events.ScenarioFinished:
+        return events.ScenarioFinished(
+            id=scenario_started.id,
+            suite_id=suite_id,
+            phase=phase,
+            label=operation.label,
+            recorder=recorder,
+            status=status,
+            elapsed_time=time.monotonic() - test_start_time,
+            skip_reason=skip_reason,
+            is_final=False,
         )
 
     try:
@@ -111,7 +126,8 @@ def run_test(
         status = Status.ERROR
         yield non_fatal_error(hypothesis.errors.Unsatisfiable("Failed to generate test cases for this API operation"))
     except KeyboardInterrupt:
-        yield events.Interrupted(phase=PhaseName.UNIT_TESTING)
+        yield scenario_finished(Status.INTERRUPTED)
+        yield events.Interrupted(phase=phase)
         return
     except AssertionError as exc:  # May come from `hypothesis-jsonschema` or `hypothesis`
         status = Status.ERROR
@@ -131,7 +147,6 @@ def run_test(
                     exc,
                     path=operation.path,
                     method=operation.method,
-                    full_path=operation.schema.get_full_path(operation.path),
                 )
             )
     except HypothesisRefResolutionError:
@@ -165,7 +180,7 @@ def run_test(
             yield non_fatal_error(exc)
     if (
         status == Status.SUCCESS
-        and ctx.config.execution.no_failfast
+        and ctx.config.execution.continue_on_failure
         and any(check.status == Status.FAILURE for checks in recorder.checks.values() for check in checks)
     ):
         status = Status.FAILURE
@@ -194,20 +209,10 @@ def run_test(
     if invalid_headers:
         status = Status.ERROR
         yield non_fatal_error(InvalidHeadersExample.from_headers(invalid_headers))
-    test_elapsed_time = time.monotonic() - test_start_time
     for error in deduplicate_errors(errors):
         yield non_fatal_error(error)
-    yield events.ScenarioFinished(
-        id=scenario_started.id,
-        suite_id=suite_id,
-        phase=PhaseName.UNIT_TESTING,
-        label=operation.label,
-        recorder=recorder,
-        status=status,
-        elapsed_time=test_elapsed_time,
-        skip_reason=skip_reason,
-        is_final=False,
-    )
+
+    yield scenario_finished(status)
 
 
 def setup_hypothesis_database_key(test: Callable, operation: APIOperation) -> None:
@@ -280,7 +285,7 @@ def test_func(*, ctx: EngineContext, case: Case, recorder: ScenarioRecorder) ->
         ctx=ctx.get_check_context(recorder),
         checks=ctx.config.execution.checks,
         response=response,
-        no_failfast=ctx.config.execution.no_failfast,
+        continue_on_failure=ctx.config.execution.continue_on_failure,
         recorder=recorder,
     )
 
@@ -291,7 +296,7 @@ def validate_response(
     ctx: CheckContext,
     checks: Iterable[CheckFunction],
     response: Response,
-    no_failfast: bool,
+    continue_on_failure: bool,
     recorder: ScenarioRecorder,
 ) -> None:
     failures = set()
@@ -318,5 +323,5 @@ def validate_response(
         on_success=on_success,
     )
 
-    if failures and not no_failfast:
+    if failures and not continue_on_failure:
         raise FailureGroup(list(failures)) from None

schemathesis/engine/phases/unit/_pool.py

@@ -7,9 +7,11 @@ from types import TracebackType
 from typing import TYPE_CHECKING, Callable
 
 from schemathesis.core.result import Result
+from schemathesis.engine.phases import PhaseName
 
 if TYPE_CHECKING:
     from schemathesis.engine.context import EngineContext
+    from schemathesis.generation.hypothesis.builder import HypothesisTestMode
 
 
 class TaskProducer:
@@ -34,12 +36,16 @@ class WorkerPool:
         producer: TaskProducer,
         worker_factory: Callable,
         ctx: EngineContext,
+        mode: HypothesisTestMode,
+        phase: PhaseName,
         suite_id: uuid.UUID,
     ) -> None:
         self.workers_num = workers_num
         self.producer = producer
         self.worker_factory = worker_factory
         self.ctx = ctx
+        self.mode = mode
+        self.phase = phase
         self.suite_id = suite_id
         self.workers: list[threading.Thread] = []
         self.events_queue: Queue = Queue()
@@ -51,6 +57,8 @@ class WorkerPool:
                 target=self.worker_factory,
                 kwargs={
                     "ctx": self.ctx,
+                    "mode": self.mode,
+                    "phase": self.phase,
                     "events_queue": self.events_queue,
                     "producer": self.producer,
                     "suite_id": self.suite_id,

schemathesis/errors.py

@@ -4,17 +4,19 @@ from schemathesis.core.errors import (
     IncorrectUsage,
     InternalError,
     InvalidHeadersExample,
-    InvalidLinkDefinition,
     InvalidRateLimit,
     InvalidRegexPattern,
     InvalidRegexType,
     InvalidSchema,
+    InvalidStateMachine,
+    InvalidTransition,
     LoaderError,
     NoLinksFound,
     OperationNotFound,
     SchemathesisError,
     SerializationError,
     SerializationNotPossible,
+    TransitionValidationError,
     UnboundPrefix,
 )
 
@@ -22,16 +24,18 @@ __all__ = [
     "IncorrectUsage",
     "InternalError",
     "InvalidHeadersExample",
-    "InvalidLinkDefinition",
     "InvalidRateLimit",
     "InvalidRegexPattern",
     "InvalidRegexType",
     "InvalidSchema",
+    "InvalidStateMachine",
+    "InvalidTransition",
     "LoaderError",
     "OperationNotFound",
     "NoLinksFound",
     "SchemathesisError",
     "SerializationError",
     "SerializationNotPossible",
+    "TransitionValidationError",
     "UnboundPrefix",
 ]

schemathesis/experimental/__init__.py

@@ -64,12 +64,6 @@ class ExperimentSet:
 
 ENV_PREFIX = "SCHEMATHESIS_EXPERIMENTAL"
 GLOBAL_EXPERIMENTS = ExperimentSet()
-COVERAGE_PHASE = GLOBAL_EXPERIMENTS.create_experiment(
-    name="Coverage phase",
-    env_var="COVERAGE_PHASE",
-    description="Generate covering test cases",
-    discussion_url="https://github.com/schemathesis/schemathesis/discussions/2418",
-)
 POSITIVE_DATA_ACCEPTANCE = GLOBAL_EXPERIMENTS.create_experiment(
     name="Positive Data Acceptance",
     env_var="POSITIVE_DATA_ACCEPTANCE",

schemathesis/filters.py

@@ -268,6 +268,8 @@ class FilterSet:
                 # To match anything the regex should match the expected value, hence passing them together is useless
                 raise IncorrectUsage(ERROR_EXPECTED_AND_REGEX)
             if expected is not None:
+                if attribute == "method":
+                    expected = _normalize_method(expected)
                 matchers.append(Matcher.for_value(attribute, expected))
             if regex is not None:
                 matchers.append(Matcher.for_regex(attribute, regex))
@@ -283,6 +285,12 @@ class FilterSet:
             self._excludes.add(filter_)
 
 
+def _normalize_method(value: FilterValue) -> FilterValue:
+    if isinstance(value, list):
+        return [item.upper() for item in value]
+    return value.upper()
+
+
 def attach_filter_chain(
     target: Callable,
     attribute: str,

schemathesis/generation/coverage.py

@@ -437,7 +437,12 @@ def cover_schema_iter(
                 elif key == "required":
                     template = template or ctx.generate_from_schema(_get_template_schema(schema, "object"))
                     yield from _negative_required(ctx, template, value)
-                elif key == "additionalProperties" and not value and "pattern" not in schema:
+                elif (
+                    key == "additionalProperties"
+                    and not value
+                    and "pattern" not in schema
+                    and schema.get("type") in ["object", None]
+                ):
                     template = template or ctx.generate_from_schema(_get_template_schema(schema, "object"))
                     yield NegativeValue(
                         {**template, UNKNOWN_PROPERTY_KEY: UNKNOWN_PROPERTY_VALUE},