schemathesis 4.0.0a4__py3-none-any.whl → 4.0.0a6__py3-none-any.whl

schemathesis/engine/phases/unit/__init__.py

@@ -11,11 +11,12 @@ import warnings
 from queue import Queue
 from typing import TYPE_CHECKING, Any
 
+from schemathesis.core.errors import InvalidSchema
 from schemathesis.core.result import Ok
 from schemathesis.engine import Status, events
 from schemathesis.engine.phases import PhaseName, PhaseSkipReason
 from schemathesis.engine.recorder import ScenarioRecorder
-from schemathesis.generation.hypothesis.builder import HypothesisTestConfig
+from schemathesis.generation.hypothesis.builder import HypothesisTestConfig, HypothesisTestMode
 from schemathesis.generation.hypothesis.reporting import ignore_hypothesis_output
 
 from ._pool import TaskProducer, WorkerPool
@@ -34,6 +35,12 @@ def execute(engine: EngineContext, phase: Phase) -> events.EventGenerator:
     Implemented as a producer-consumer pattern via a task queue.
     The main thread provides an iterator over API operations and worker threads create test functions and run them.
     """
+    if phase.name == PhaseName.EXAMPLES:
+        mode = HypothesisTestMode.EXAMPLES
+    elif phase.name == PhaseName.COVERAGE:
+        mode = HypothesisTestMode.COVERAGE
+    else:
+        mode = HypothesisTestMode.FUZZING
     producer = TaskProducer(engine)
     workers_num = engine.config.execution.workers_num
 
@@ -45,7 +52,13 @@ def execute(engine: EngineContext, phase: Phase) -> events.EventGenerator:
     is_executed = False
 
     with WorkerPool(
-        workers_num=workers_num, producer=producer, worker_factory=worker_task, ctx=engine, suite_id=suite_started.id
+        workers_num=workers_num,
+        producer=producer,
+        worker_factory=worker_task,
+        ctx=engine,
+        mode=mode,
+        phase=phase.name,
+        suite_id=suite_started.id,
     ) as pool:
         try:
             while True:
@@ -74,7 +87,7 @@ def execute(engine: EngineContext, phase: Phase) -> events.EventGenerator:
         except KeyboardInterrupt:
             engine.stop()
             status = Status.INTERRUPTED
-            yield events.Interrupted(phase=PhaseName.UNIT_TESTING)
+            yield events.Interrupted(phase=phase.name)
 
     if not is_executed:
         phase.skip_reason = PhaseSkipReason.NOTHING_TO_TEST
@@ -86,13 +99,52 @@ def execute(engine: EngineContext, phase: Phase) -> events.EventGenerator:
     yield events.PhaseFinished(phase=phase, status=status, payload=None)
 
 
-def worker_task(*, events_queue: Queue, producer: TaskProducer, ctx: EngineContext, suite_id: uuid.UUID) -> None:
-    from hypothesis.errors import HypothesisWarning
+def worker_task(
+    *,
+    events_queue: Queue,
+    producer: TaskProducer,
+    ctx: EngineContext,
+    mode: HypothesisTestMode,
+    phase: PhaseName,
+    suite_id: uuid.UUID,
+) -> None:
+    from hypothesis.errors import HypothesisWarning, InvalidArgument
 
     from schemathesis.generation.hypothesis.builder import create_test
 
     from ._executor import run_test, test_func
 
+    def on_error(error: Exception, *, method: str | None = None, path: str | None = None) -> None:
+        if method and path:
+            label = f"{method.upper()} {path}"
+            scenario_started = events.ScenarioStarted(label=label, phase=phase, suite_id=suite_id)
+            events_queue.put(scenario_started)
+
+            events_queue.put(events.NonFatalError(error=error, phase=phase, label=label, related_to_operation=True))
+
+            events_queue.put(
+                events.ScenarioFinished(
+                    id=scenario_started.id,
+                    suite_id=suite_id,
+                    phase=phase,
+                    label=label,
+                    status=Status.ERROR,
+                    recorder=ScenarioRecorder(label="Error"),
+                    elapsed_time=0.0,
+                    skip_reason=None,
+                    is_final=True,
+                )
+            )
+        else:
+            events_queue.put(
+                events.NonFatalError(
+                    error=error,
+                    phase=phase,
+                    label=path or "-",
+                    related_to_operation=False,
+                )
+            )
+
     warnings.filterwarnings("ignore", message="The recursion limit will not be reset", category=HypothesisWarning)
     with ignore_hypothesis_output():
         try:
@@ -104,61 +156,34 @@ def worker_task(*, events_queue: Queue, producer: TaskProducer, ctx: EngineConte
                 if isinstance(result, Ok):
                     operation = result.ok()
                     as_strategy_kwargs = get_strategy_kwargs(ctx, operation)
-                    test_function = create_test(
-                        operation=operation,
-                        test_func=test_func,
-                        config=HypothesisTestConfig(
-                            settings=ctx.config.execution.hypothesis_settings,
-                            seed=ctx.config.execution.seed,
-                            generation=ctx.config.execution.generation,
-                            as_strategy_kwargs=as_strategy_kwargs,
-                        ),
-                    )
+                    try:
+                        test_function = create_test(
+                            operation=operation,
+                            test_func=test_func,
+                            config=HypothesisTestConfig(
+                                modes=[mode],
+                                settings=ctx.config.execution.hypothesis_settings,
+                                seed=ctx.config.execution.seed,
+                                generation=ctx.config.execution.generation,
+                                as_strategy_kwargs=as_strategy_kwargs,
+                            ),
+                        )
+                    except (InvalidSchema, InvalidArgument) as exc:
+                        on_error(exc, method=operation.method, path=operation.path)
+                        continue
 
                     # The test is blocking, meaning that even if CTRL-C comes to the main thread, this tasks will continue
                     # executing. However, as we set a stop event, it will be checked before the next network request.
                     # However, this is still suboptimal, as there could be slow requests and they will block for longer
-                    for event in run_test(operation=operation, test_function=test_function, ctx=ctx, suite_id=suite_id):
+                    for event in run_test(
+                        operation=operation, test_function=test_function, ctx=ctx, phase=phase, suite_id=suite_id
+                    ):
                         events_queue.put(event)
                 else:
                     error = result.err()
-                    if error.method:
-                        label = f"{error.method.upper()} {error.path}"
-                        scenario_started = events.ScenarioStarted(
-                            label=label, phase=PhaseName.UNIT_TESTING, suite_id=suite_id
-                        )
-                        events_queue.put(scenario_started)
-
-                        events_queue.put(
-                            events.NonFatalError(
-                                error=error, phase=PhaseName.UNIT_TESTING, label=label, related_to_operation=True
-                            )
-                        )
-
-                        events_queue.put(
-                            events.ScenarioFinished(
-                                id=scenario_started.id,
-                                suite_id=suite_id,
-                                phase=PhaseName.UNIT_TESTING,
-                                label=label,
-                                status=Status.ERROR,
-                                recorder=ScenarioRecorder(label="Error"),
-                                elapsed_time=0.0,
-                                skip_reason=None,
-                                is_final=True,
-                            )
-                        )
-                    else:
-                        events_queue.put(
-                            events.NonFatalError(
-                                error=error,
-                                phase=PhaseName.UNIT_TESTING,
-                                label=error.path,
-                                related_to_operation=False,
-                            )
-                        )
+                    on_error(error, method=error.method, path=error.path)
         except KeyboardInterrupt:
-            events_queue.put(events.Interrupted(phase=PhaseName.UNIT_TESTING))
+            events_queue.put(events.Interrupted(phase=phase))
 
 
 def get_strategy_kwargs(ctx: EngineContext, operation: APIOperation) -> dict[str, Any]:

schemathesis/engine/phases/unit/_executor.py

@@ -52,13 +52,17 @@ if TYPE_CHECKING:
 
 
 def run_test(
-    *, operation: APIOperation, test_function: Callable, ctx: EngineContext, suite_id: uuid.UUID
+    *,
+    operation: APIOperation,
+    test_function: Callable,
+    ctx: EngineContext,
+    phase: PhaseName,
+    suite_id: uuid.UUID,
 ) -> events.EventGenerator:
     """A single test run with all error handling needed."""
     import hypothesis.errors
 
-    # To simplify connecting `before` and `after` events in external systems
-    scenario_started = events.ScenarioStarted(label=operation.label, phase=PhaseName.UNIT_TESTING, suite_id=suite_id)
+    scenario_started = events.ScenarioStarted(label=operation.label, phase=phase, suite_id=suite_id)
     yield scenario_started
     errors: list[Exception] = []
     skip_reason = None
@@ -66,15 +70,13 @@ def run_test(
     recorder = ScenarioRecorder(label=operation.label)
 
     def non_fatal_error(error: Exception) -> events.NonFatalError:
-        return events.NonFatalError(
-            error=error, phase=PhaseName.UNIT_TESTING, label=operation.label, related_to_operation=True
-        )
+        return events.NonFatalError(error=error, phase=phase, label=operation.label, related_to_operation=True)
 
     def scenario_finished(status: Status) -> events.ScenarioFinished:
         return events.ScenarioFinished(
             id=scenario_started.id,
             suite_id=suite_id,
-            phase=PhaseName.UNIT_TESTING,
+            phase=phase,
             label=operation.label,
             recorder=recorder,
             status=status,
@@ -125,7 +127,7 @@ def run_test(
         yield non_fatal_error(hypothesis.errors.Unsatisfiable("Failed to generate test cases for this API operation"))
     except KeyboardInterrupt:
         yield scenario_finished(Status.INTERRUPTED)
-        yield events.Interrupted(phase=PhaseName.UNIT_TESTING)
+        yield events.Interrupted(phase=phase)
         return
     except AssertionError as exc:  # May come from `hypothesis-jsonschema` or `hypothesis`
         status = Status.ERROR
@@ -178,7 +180,7 @@ def run_test(
             yield non_fatal_error(exc)
     if (
         status == Status.SUCCESS
-        and ctx.config.execution.no_failfast
+        and ctx.config.execution.continue_on_failure
         and any(check.status == Status.FAILURE for checks in recorder.checks.values() for check in checks)
     ):
         status = Status.FAILURE
@@ -283,7 +285,7 @@ def test_func(*, ctx: EngineContext, case: Case, recorder: ScenarioRecorder) ->
         ctx=ctx.get_check_context(recorder),
         checks=ctx.config.execution.checks,
         response=response,
-        no_failfast=ctx.config.execution.no_failfast,
+        continue_on_failure=ctx.config.execution.continue_on_failure,
         recorder=recorder,
     )
 
@@ -294,7 +296,7 @@ def validate_response(
     ctx: CheckContext,
     checks: Iterable[CheckFunction],
     response: Response,
-    no_failfast: bool,
+    continue_on_failure: bool,
     recorder: ScenarioRecorder,
 ) -> None:
     failures = set()
@@ -321,5 +323,5 @@ def validate_response(
         on_success=on_success,
     )
 
-    if failures and not no_failfast:
+    if failures and not continue_on_failure:
         raise FailureGroup(list(failures)) from None

schemathesis/engine/phases/unit/_pool.py

@@ -7,9 +7,11 @@ from types import TracebackType
 from typing import TYPE_CHECKING, Callable
 
 from schemathesis.core.result import Result
+from schemathesis.engine.phases import PhaseName
 
 if TYPE_CHECKING:
     from schemathesis.engine.context import EngineContext
+    from schemathesis.generation.hypothesis.builder import HypothesisTestMode
 
 
 class TaskProducer:
@@ -34,12 +36,16 @@ class WorkerPool:
         producer: TaskProducer,
         worker_factory: Callable,
         ctx: EngineContext,
+        mode: HypothesisTestMode,
+        phase: PhaseName,
         suite_id: uuid.UUID,
     ) -> None:
         self.workers_num = workers_num
         self.producer = producer
         self.worker_factory = worker_factory
         self.ctx = ctx
+        self.mode = mode
+        self.phase = phase
         self.suite_id = suite_id
         self.workers: list[threading.Thread] = []
         self.events_queue: Queue = Queue()
@@ -51,6 +57,8 @@ class WorkerPool:
                 target=self.worker_factory,
                 kwargs={
                     "ctx": self.ctx,
+                    "mode": self.mode,
+                    "phase": self.phase,
                     "events_queue": self.events_queue,
                     "producer": self.producer,
                     "suite_id": self.suite_id,

schemathesis/experimental/__init__.py

@@ -64,12 +64,6 @@ class ExperimentSet:
 
 ENV_PREFIX = "SCHEMATHESIS_EXPERIMENTAL"
 GLOBAL_EXPERIMENTS = ExperimentSet()
-COVERAGE_PHASE = GLOBAL_EXPERIMENTS.create_experiment(
-    name="Coverage phase",
-    env_var="COVERAGE_PHASE",
-    description="Generate covering test cases",
-    discussion_url="https://github.com/schemathesis/schemathesis/discussions/2418",
-)
 POSITIVE_DATA_ACCEPTANCE = GLOBAL_EXPERIMENTS.create_experiment(
     name="Positive Data Acceptance",
     env_var="POSITIVE_DATA_ACCEPTANCE",