schemathesis 4.0.0a4__py3-none-any.whl → 4.0.0a6__py3-none-any.whl

schemathesis/cli/commands/run/handlers/output.py

@@ -22,11 +22,13 @@ from schemathesis.core.output import prepare_response_payload
 from schemathesis.core.result import Err, Ok
 from schemathesis.core.version import SCHEMATHESIS_VERSION
 from schemathesis.engine import Status, events
+from schemathesis.engine.config import EngineConfig
 from schemathesis.engine.errors import EngineErrorInfo
 from schemathesis.engine.phases import PhaseName, PhaseSkipReason
 from schemathesis.engine.phases.probes import ProbeOutcome
-from schemathesis.engine.recorder import Interaction
+from schemathesis.engine.recorder import Interaction, ScenarioRecorder
 from schemathesis.experimental import GLOBAL_EXPERIMENTS
+from schemathesis.generation.modes import GenerationMode
 from schemathesis.schemas import ApiStatistic
 
 if TYPE_CHECKING:
@@ -323,7 +325,8 @@ class ProbingProgressManager:
 
 @dataclass
 class WarningData:
-    missing_auth: dict[int, list[str]] = field(default_factory=dict)
+    missing_auth: dict[int, set[str]] = field(default_factory=dict)
+    only_4xx_responses: set[str] = field(default_factory=set)  # operations that only returned 4xx
 
 
 @dataclass
@@ -459,9 +462,10 @@ class UnitTestProgressManager:
         if self.stats[Status.FAILURE]:
             parts.append(f"❌ {self.stats[Status.FAILURE]:{width}d} failed")
         if self.stats[Status.ERROR]:
-            parts.append(f"🚫 {self.stats[Status.ERROR]:{width}d} errors")
+            suffix = "s" if self.stats[Status.ERROR] > 1 else ""
+            parts.append(f"🚫 {self.stats[Status.ERROR]:{width}d} error{suffix}")
         if self.stats[Status.SKIP] or self.stats[Status.INTERRUPTED]:
-            parts.append(f"⏭️ {self.stats[Status.SKIP] + self.stats[Status.INTERRUPTED]:{width}d} skipped")
+            parts.append(f"⏭  {self.stats[Status.SKIP] + self.stats[Status.INTERRUPTED]:{width}d} skipped")
         return "  ".join(parts)
 
     def _update_stats_display(self) -> None:
@@ -513,7 +517,11 @@ class UnitTestProgressManager:
         if operation := self.current_operations.pop(label, None):
             if not self.current_operations:
                 assert self.title_task_id is not None
-                self.title_progress.update(self.title_task_id)
+                if self.current == self.total - 1:
+                    description = f"  {self.title}"
+                else:
+                    description = self.title
+                self.title_progress.update(self.title_task_id, description=description)
             self.operations_progress.update(operation.task_id, visible=False)
 
     def update_stats(self, status: Status) -> None:
@@ -542,7 +550,7 @@ class UnitTestProgressManager:
         elif self.stats[Status.SUCCESS] > 0:
             icon = "✅"
         elif self.stats[Status.SKIP] > 0:
-            icon = "⏭️"
+            icon = "⏭ "
         else:
             icon = default_icon
         return icon
@@ -661,7 +669,7 @@ class StatefulProgressManager:
         from rich.text import Text
 
         # Initialize progress displays
-        self.title_task_id = self.title_progress.add_task("Stateful tests")
+        self.title_task_id = self.title_progress.add_task("Stateful")
         self.progress_task_id = self.progress_bar.add_task(
             "", scenarios=0, links=f"0 covered / {self.links_selected} selected / {self.links_total} total links"
         )
@@ -710,9 +718,10 @@ class StatefulProgressManager:
         if self.stats[Status.FAILURE]:
             parts.append(f"❌ {self.stats[Status.FAILURE]} failed")
         if self.stats[Status.ERROR]:
-            parts.append(f"🚫 {self.stats[Status.ERROR]} errors")
+            suffix = "s" if self.stats[Status.ERROR] > 1 else ""
+            parts.append(f"🚫 {self.stats[Status.ERROR]} error{suffix}")
         if self.stats[Status.SKIP]:
-            parts.append(f"⏭️ {self.stats[Status.SKIP]} skipped")
+            parts.append(f"⏭  {self.stats[Status.SKIP]} skipped")
         return "  ".join(parts)
 
     def _update_stats_display(self) -> None:
@@ -729,7 +738,7 @@ class StatefulProgressManager:
         elif self.stats[Status.SUCCESS] > 0:
             icon = "✅"
         elif self.stats[Status.SKIP] > 0:
-            icon = "⏭️"
+            icon = "⏭ "
         else:
             icon = default_icon
         return icon
@@ -755,39 +764,18 @@ class StatefulProgressManager:
 
 
 def format_duration(duration_ms: int) -> str:
-    """Format duration in milliseconds to human readable string."""
-    parts = []
-
-    # Convert to components
-    ms = duration_ms % 1000
-    seconds = (duration_ms // 1000) % 60
-    minutes = (duration_ms // (1000 * 60)) % 60
-    hours = duration_ms // (1000 * 60 * 60)
-
-    # Add non-empty components
-    if hours > 0:
-        parts.append(f"{hours} h")
-    if minutes > 0:
-        parts.append(f"{minutes} m")
-    if seconds > 0:
-        parts.append(f"{seconds} s")
-    if ms > 0:
-        parts.append(f"{ms} ms")
-
-    # Handle zero duration
-    if not parts:
-        return "0 ms"
-
-    return " ".join(parts)
+    """Format duration in milliseconds to seconds with 2 decimal places."""
+    return f"{duration_ms / 1000:.2f}s"
 
 
 @dataclass
 class OutputHandler(EventHandler):
     workers_num: int
-    # Seed can't be absent in the deterministic mode
+    # Seed can be absent in the deterministic mode
     seed: int | None
     rate_limit: str | None
     wait_for_schema: float | None
+    engine_config: EngineConfig
 
     loading_manager: LoadingProgressManager | None = None
     probing_manager: ProbingProgressManager | None = None
@@ -798,9 +786,9 @@ class OutputHandler(EventHandler):
     skip_reasons: list[str] = field(default_factory=list)
     report_config: ReportConfig | None = None
     warnings: WarningData = field(default_factory=WarningData)
-    errors: list[events.NonFatalError] = field(default_factory=list)
+    errors: set[events.NonFatalError] = field(default_factory=set)
     phases: dict[PhaseName, tuple[Status, PhaseSkipReason | None]] = field(
-        default_factory=lambda: {phase: (Status.SKIP, None) for phase in PhaseName}
+        default_factory=lambda: dict.fromkeys(PhaseName, (Status.SKIP, None))
     )
     console: Console = field(default_factory=_default_console)
 
@@ -820,7 +808,7 @@ class OutputHandler(EventHandler):
         elif isinstance(event, events.FatalError):
             self._on_fatal_error(ctx, event)
         elif isinstance(event, events.NonFatalError):
-            self.errors.append(event)
+            self.errors.add(event)
         elif isinstance(event, LoadingStarted):
             self._on_loading_started(event)
         elif isinstance(event, LoadingFinished):
@@ -885,8 +873,8 @@ class OutputHandler(EventHandler):
         phase = event.phase
         if phase.name == PhaseName.PROBING and phase.is_enabled:
             self._start_probing()
-        elif phase.name == PhaseName.UNIT_TESTING and phase.is_enabled:
-            self._start_unit_tests()
+        elif phase.name in [PhaseName.EXAMPLES, PhaseName.COVERAGE, PhaseName.FUZZING] and phase.is_enabled:
+            self._start_unit_tests(phase.name)
         elif phase.name == PhaseName.STATEFUL_TESTING and phase.is_enabled and phase.skip_reason is None:
             self._start_stateful_tests()
 
@@ -894,11 +882,12 @@ class OutputHandler(EventHandler):
         self.probing_manager = ProbingProgressManager(console=self.console)
         self.probing_manager.start()
 
-    def _start_unit_tests(self) -> None:
+    def _start_unit_tests(self, phase: PhaseName) -> None:
         assert self.statistic is not None
+        assert self.unit_tests_manager is None
         self.unit_tests_manager = UnitTestProgressManager(
             console=self.console,
-            title="Unit tests",
+            title=phase.value,
             total=self.statistic.operations.selected,
         )
         self.unit_tests_manager.start()
@@ -907,7 +896,7 @@ class OutputHandler(EventHandler):
         assert self.statistic is not None
         self.stateful_tests_manager = StatefulProgressManager(
             console=self.console,
-            title="Stateful tests",
+            title="Stateful",
             links_selected=self.statistic.links.selected,
             links_total=self.statistic.links.total,
         )
@@ -963,7 +952,7 @@ class OutputHandler(EventHandler):
             elif event.status == Status.SKIP:
                 message = Padding(
                     Text.assemble(
-                        ("⏭️  ", ""),
+                        ("⏭   ", ""),
                         ("API probing skipped", Style(color="yellow")),
                     ),
                     BLOCK_PADDING,
@@ -981,8 +970,7 @@ class OutputHandler(EventHandler):
                 )
             self.console.print(message)
             self.console.print()
-        elif phase.name == PhaseName.STATEFUL_TESTING and phase.is_enabled:
-            assert self.stateful_tests_manager is not None
+        elif phase.name == PhaseName.STATEFUL_TESTING and phase.is_enabled and self.stateful_tests_manager is not None:
             self.stateful_tests_manager.stop()
             if event.status == Status.ERROR:
                 title, summary = self.stateful_tests_manager.get_completion_message("🚫")
@@ -1011,27 +999,29 @@ class OutputHandler(EventHandler):
             self.console.print(Padding(Text(summary, style="bright_white"), (0, 0, 0, 5)))
             self.console.print()
             self.stateful_tests_manager = None
-        elif phase.name == PhaseName.UNIT_TESTING and phase.is_enabled:
-            assert self.unit_tests_manager is not None
+        elif (
+            phase.name in [PhaseName.EXAMPLES, PhaseName.COVERAGE, PhaseName.FUZZING]
+            and phase.is_enabled
+            and self.unit_tests_manager is not None
+        ):
             self.unit_tests_manager.stop()
             if event.status == Status.ERROR:
                 message = self.unit_tests_manager.get_completion_message("🚫")
             else:
                 message = self.unit_tests_manager.get_completion_message()
             self.console.print(Padding(Text(message, style="white"), BLOCK_PADDING))
-            if event.status != Status.INTERRUPTED:
-                self.console.print()
+            self.console.print()
             self.unit_tests_manager = None
 
     def _on_scenario_started(self, event: events.ScenarioStarted) -> None:
-        if event.phase == PhaseName.UNIT_TESTING:
+        if event.phase in [PhaseName.EXAMPLES, PhaseName.COVERAGE, PhaseName.FUZZING]:
             # We should display execution result + percentage in the end. For example:
             assert event.label is not None
             assert self.unit_tests_manager is not None
             self.unit_tests_manager.start_operation(event.label)
 
     def _on_scenario_finished(self, event: events.ScenarioFinished) -> None:
-        if event.phase == PhaseName.UNIT_TESTING:
+        if event.phase in [PhaseName.EXAMPLES, PhaseName.COVERAGE, PhaseName.FUZZING]:
             assert self.unit_tests_manager is not None
             if event.label:
                 self.unit_tests_manager.finish_operation(event.label)
@@ -1050,9 +1040,39 @@ class OutputHandler(EventHandler):
             self.stateful_tests_manager.update(links_seen, event.status)
 
     def _check_warnings(self, event: events.ScenarioFinished) -> None:
+        statistic = aggregate_status_codes(event.recorder.interactions.values())
+
+        if statistic.total == 0:
+            return
+
         for status_code in (401, 403):
-            if has_too_many_responses_with_status(event.recorder.interactions.values(), status_code):
-                self.warnings.missing_auth.setdefault(status_code, []).append(event.recorder.label)
+            if statistic.ratio_for(status_code) >= TOO_MANY_RESPONSES_THRESHOLD:
+                self.warnings.missing_auth.setdefault(status_code, set()).add(event.recorder.label)
+
+        # Warn if all positive test cases got 4xx in return and no failure was found
+        def all_positive_are_rejected(recorder: ScenarioRecorder) -> bool:
+            seen_positive = False
+            for case in recorder.cases.values():
+                if not (case.value.meta is not None and case.value.meta.generation.mode == GenerationMode.POSITIVE):
+                    continue
+                seen_positive = True
+                interaction = recorder.interactions.get(case.value.id)
+                if not (interaction is not None and interaction.response is not None):
+                    continue
+                # At least one positive response for positive test case
+                if 200 <= interaction.response.status_code < 300:
+                    return False
+            # If there are positive test cases, and we ended up here, then there are no 2xx responses for them
+            # Otherwise, there are no positive test cases at all and this check should pass
+            return seen_positive
+
+        if (
+            event.status == Status.SUCCESS
+            and GenerationMode.POSITIVE in self.engine_config.execution.generation.modes
+            and all_positive_are_rejected(event.recorder)
+            and statistic.should_warn_about_only_4xx()
+        ):
+            self.warnings.only_4xx_responses.add(event.recorder.label)
 
     def _on_interrupted(self, event: events.Interrupted) -> None:
         from rich.padding import Padding
@@ -1122,36 +1142,58 @@ class OutputHandler(EventHandler):
 
     def display_warnings(self) -> None:
         display_section_name("WARNINGS")
-        total = sum(len(endpoints) for endpoints in self.warnings.missing_auth.values())
-        suffix = "" if total == 1 else "s"
-        click.echo(
-            _style(
-                f"\nMissing or invalid API credentials: {total} API operation{suffix} returned authentication errors\n",
-                fg="yellow",
+        click.echo()
+        if self.warnings.missing_auth:
+            total = sum(len(endpoints) for endpoints in self.warnings.missing_auth.values())
+            suffix = "" if total == 1 else "s"
+            click.echo(
+                _style(
+                    f"Missing or invalid API credentials: {total} API operation{suffix} returned authentication errors\n",
+                    fg="yellow",
+                )
             )
-        )
 
-        for status_code, operations in self.warnings.missing_auth.items():
-            status_text = "Unauthorized" if status_code == 401 else "Forbidden"
-            count = len(operations)
+            for status_code, operations in self.warnings.missing_auth.items():
+                status_text = "Unauthorized" if status_code == 401 else "Forbidden"
+                count = len(operations)
+                suffix = "" if count == 1 else "s"
+                click.echo(
+                    _style(
+                        f"{status_code} {status_text} ({count} operation{suffix}):",
+                        fg="yellow",
+                    )
+                )
+                # Show first few API operations
+                for endpoint in sorted(operations)[:3]:
+                    click.echo(_style(f"  - {endpoint}", fg="yellow"))
+                if len(operations) > 3:
+                    click.echo(_style(f"  + {len(operations) - 3} more", fg="yellow"))
+                click.echo()
+            click.echo(_style("Tip: ", bold=True, fg="yellow"), nl=False)
+            click.echo(_style(f"Use {bold('--auth')} ", fg="yellow"), nl=False)
+            click.echo(_style(f"or {bold('-H')} ", fg="yellow"), nl=False)
+            click.echo(_style("to provide authentication credentials", fg="yellow"))
+            click.echo()
+
+        if self.warnings.only_4xx_responses:
+            count = len(self.warnings.only_4xx_responses)
             suffix = "" if count == 1 else "s"
             click.echo(
                 _style(
-                    f"{status_code} {status_text} ({count} operation{suffix}):",
+                    f"Schemathesis configuration: {count} operation{suffix} returned only 4xx responses during unit tests\n",
                     fg="yellow",
                 )
             )
-            # Show first few API operations
-            for endpoint in operations[:3]:
+
+            for endpoint in sorted(self.warnings.only_4xx_responses)[:3]:
                 click.echo(_style(f"  - {endpoint}", fg="yellow"))
-            if len(operations) > 3:
-                click.echo(_style(f"  + {len(operations) - 3} more", fg="yellow"))
+            if len(self.warnings.only_4xx_responses) > 3:
+                click.echo(_style(f"  + {len(self.warnings.only_4xx_responses) - 3} more", fg="yellow"))
+            click.echo()
+
+            click.echo(_style("Tip: ", bold=True, fg="yellow"), nl=False)
+            click.echo(_style("Check base URL or adjust data generation settings", fg="yellow"))
             click.echo()
-        click.echo(_style("Tip: ", bold=True, fg="yellow"), nl=False)
-        click.echo(_style(f"Use {bold('--auth')} ", fg="yellow"), nl=False)
-        click.echo(_style(f"or {bold('-H')} ", fg="yellow"), nl=False)
-        click.echo(_style("to provide authentication credentials", fg="yellow"))
-        click.echo()
 
     def display_experiments(self) -> None:
         display_section_name("EXPERIMENTS")
@@ -1235,7 +1277,7 @@ class OutputHandler(EventHandler):
                 err.label
                 for err in self.errors
                 # Some API operations may have some tests before they have an error
-                if err.phase == PhaseName.UNIT_TESTING
+                if err.phase in [PhaseName.EXAMPLES, PhaseName.COVERAGE, PhaseName.FUZZING]
                 and err.label not in ctx.statistic.tested_operations
                 and err.related_to_operation
             }
@@ -1258,7 +1300,7 @@ class OutputHandler(EventHandler):
             status, skip_reason = self.phases[phase]
 
             if status == Status.SKIP:
-                click.echo(_style(f"  ⏭️ {phase.value}", fg="yellow"), nl=False)
+                click.echo(_style(f"  ⏭  {phase.value}", fg="yellow"), nl=False)
                 if skip_reason:
                     click.echo(_style(f" ({skip_reason.value})", fg="yellow"))
                 else:
@@ -1395,7 +1437,7 @@ class OutputHandler(EventHandler):
         assert self.stateful_tests_manager is None
         if self.errors:
             display_section_name("ERRORS")
-            errors = sorted(self.errors, key=lambda r: (r.phase.value, r.label))
+            errors = sorted(self.errors, key=lambda r: (r.phase.value, r.label, r.info.title))
             for error in errors:
                 display_section_name(error.label, "_", fg="red")
                 click.echo(error.info.format(bold=lambda x: click.style(x, bold=True)))
@@ -1406,7 +1448,7 @@ class OutputHandler(EventHandler):
                 )
             )
         display_failures(ctx)
-        if self.warnings.missing_auth:
+        if self.warnings.missing_auth or self.warnings.only_4xx_responses:
             self.display_warnings()
         if GLOBAL_EXPERIMENTS.enabled:
             self.display_experiments()
@@ -1426,10 +1468,21 @@ class OutputHandler(EventHandler):
         if self.errors:
             self.display_errors_summary()
 
-        if self.warnings.missing_auth:
-            affected = sum(len(operations) for operations in self.warnings.missing_auth.values())
+        if self.warnings.missing_auth or self.warnings.only_4xx_responses:
             click.echo(_style("Warnings:", bold=True))
-            click.echo(_style(f"  ⚠️ Missing authentication: {bold(str(affected))}", fg="yellow"))
+
+            if self.warnings.missing_auth:
+                affected = sum(len(operations) for operations in self.warnings.missing_auth.values())
+                click.echo(_style(f"  ⚠️ Missing authentication: {bold(str(affected))}", fg="yellow"))
+
+            if self.warnings.only_4xx_responses:
+                count = len(self.warnings.only_4xx_responses)
+                suffix = "" if count == 1 else "s"
+                click.echo(
+                    _style(f"  ⚠️ Schemathesis configuration: {bold(str(count))}", fg="yellow"),
+                    nl=False,
+                )
+                click.echo(_style(f" operation{suffix} returned only 4xx responses during unit tests", fg="yellow"))
             click.echo()
 
         if ctx.summary_lines:
@@ -1448,14 +1501,41 @@ TOO_MANY_RESPONSES_WARNING_TEMPLATE = (
 TOO_MANY_RESPONSES_THRESHOLD = 0.9
 
 
-def has_too_many_responses_with_status(interactions: Iterable[Interaction], status_code: int) -> bool:
-    matched = 0
+@dataclass
+class StatusCodeStatistic:
+    """Statistics about HTTP status codes in a scenario."""
+
+    counts: dict[int, int]
+    total: int
+
+    __slots__ = ("counts", "total")
+
+    def ratio_for(self, status_code: int) -> float:
+        """Calculate the ratio of responses with the given status code."""
+        if self.total == 0:
+            return 0.0
+        return self.counts.get(status_code, 0) / self.total
+
+    def should_warn_about_only_4xx(self) -> bool:
+        """Check if an operation should be warned about (only 4xx responses, excluding auth)."""
+        if self.total == 0:
+            return False
+        # Don't duplicate auth warnings
+        if set(self.counts.keys()) <= {401, 403}:
+            return False
+        # At this point we know we only have 4xx responses
+        return True
+
+
+def aggregate_status_codes(interactions: Iterable[Interaction]) -> StatusCodeStatistic:
+    """Analyze status codes from interactions."""
+    counts: dict[int, int] = {}
     total = 0
+
     for interaction in interactions:
         if interaction.response is not None:
-            if interaction.response.status_code == status_code:
-                matched += 1
+            status = interaction.response.status_code
+            counts[status] = counts.get(status, 0) + 1
             total += 1
-    if not total:
-        return False
-    return matched / total >= TOO_MANY_RESPONSES_THRESHOLD
+
+    return StatusCodeStatistic(counts=counts, total=total)

schemathesis/cli/commands/run/hypothesis.py

@@ -3,41 +3,15 @@ from __future__ import annotations
 from enum import Enum, unique
 from typing import TYPE_CHECKING, Any
 
-import click
-
 if TYPE_CHECKING:
     import hypothesis
 
-PHASES_INVALID_USAGE_MESSAGE = "Can't use `--hypothesis-phases` and `--hypothesis-no-phases` simultaneously"
 HYPOTHESIS_IN_MEMORY_DATABASE_IDENTIFIER = ":memory:"
 
 # Importing Hypothesis is expensive, hence we re-create the enums we need in CLI commands definitions
 # Hypothesis is stable, hence it should not be a problem and adding new variants should not be automatic
 
 
-@unique
-class Phase(str, Enum):
-    explicit = "explicit"  #: controls whether explicit examples are run.
-    reuse = "reuse"  #: controls whether previous examples will be reused.
-    generate = "generate"  #: controls whether new examples will be generated.
-    target = "target"  #: controls whether examples will be mutated for targeting.
-    # The `explain` phase is not supported
-
-    def as_hypothesis(self) -> hypothesis.Phase:
-        from hypothesis import Phase
-
-        return Phase[self.name]
-
-    @staticmethod
-    def filter_from_all(variants: list[Phase], no_shrink: bool) -> list[hypothesis.Phase]:
-        from hypothesis import Phase
-
-        phases = set(Phase) - {Phase.explain} - set(variants)
-        if no_shrink:
-            return list(phases - {Phase.shrink})
-        return list(phases)
-
-
 @unique
 class HealthCheck(str, Enum):
     # We remove not relevant checks
@@ -65,25 +39,13 @@ def prepare_health_checks(
     return [entry for health_check in hypothesis_suppress_health_check for entry in health_check.as_hypothesis()]
 
 
-def prepare_phases(
-    hypothesis_phases: list[Phase] | None,
-    hypothesis_no_phases: list[Phase] | None,
-    no_shrink: bool = False,
-) -> list[hypothesis.Phase] | None:
-    from hypothesis import Phase as HypothesisPhase
-
-    if hypothesis_phases is not None and hypothesis_no_phases is not None:
-        raise click.UsageError(PHASES_INVALID_USAGE_MESSAGE)
-    if hypothesis_phases:
-        phases = [phase.as_hypothesis() for phase in hypothesis_phases]
-        if not no_shrink:
-            phases.append(HypothesisPhase.shrink)
-        return phases
-    elif hypothesis_no_phases:
-        return Phase.filter_from_all(hypothesis_no_phases, no_shrink)
-    elif no_shrink:
-        return Phase.filter_from_all([], no_shrink)
-    return None
+def prepare_phases(no_shrink: bool = False) -> list[hypothesis.Phase] | None:
+    from hypothesis import Phase
+
+    phases = set(Phase) - {Phase.explain}
+    if no_shrink:
+        return list(phases - {Phase.shrink})
+    return list(phases)
 
 
 def prepare_settings(

schemathesis/core/__init__.py

@@ -16,6 +16,8 @@ class SpecificationFeature(str, enum.Enum):
     """Features that Schemathesis can provide for different specifications."""
 
     STATEFUL_TESTING = "stateful_testing"
+    COVERAGE = "coverage_tests"
+    EXAMPLES = "example_tests"
 
 
 @dataclass
@@ -39,7 +41,11 @@ class Specification:
     def supports_feature(self, feature: SpecificationFeature) -> bool:
         """Check if Schemathesis supports a given feature for this specification."""
         if self.kind == SpecificationKind.OPENAPI:
-            return feature in {SpecificationFeature.STATEFUL_TESTING}
+            return feature in {
+                SpecificationFeature.STATEFUL_TESTING,
+                SpecificationFeature.COVERAGE,
+                SpecificationFeature.EXAMPLES,
+            }
         return False
 
 

schemathesis/engine/config.py

@@ -25,14 +25,14 @@ def _default_hypothesis_settings() -> hypothesis.settings:
 class ExecutionConfig:
     """Configuration for test execution."""
 
-    phases: list[PhaseName] = field(default_factory=lambda: [PhaseName.UNIT_TESTING, PhaseName.STATEFUL_TESTING])
+    phases: list[PhaseName] = field(default_factory=PhaseName.defaults)
     checks: list[CheckFunction] = field(default_factory=lambda: [not_a_server_error])
     targets: list[TargetFunction] = field(default_factory=list)
     hypothesis_settings: hypothesis.settings = field(default_factory=_default_hypothesis_settings)
     generation: GenerationConfig = field(default_factory=GenerationConfig)
     max_failures: int | None = None
     unique_inputs: bool = False
-    no_failfast: bool = False
+    continue_on_failure: bool = False
     seed: int | None = None
     workers_num: int = 1
 

schemathesis/engine/core.py

@@ -34,7 +34,17 @@ class Engine:
         """Create execution plan based on configuration."""
         phases = [
             self.get_phase_config(PhaseName.PROBING, is_supported=True, requires_links=False),
-            self.get_phase_config(PhaseName.UNIT_TESTING, is_supported=True, requires_links=False),
+            self.get_phase_config(
+                PhaseName.EXAMPLES,
+                is_supported=self.schema.specification.supports_feature(SpecificationFeature.EXAMPLES),
+                requires_links=False,
+            ),
+            self.get_phase_config(
+                PhaseName.COVERAGE,
+                is_supported=self.schema.specification.supports_feature(SpecificationFeature.COVERAGE),
+                requires_links=False,
+            ),
+            self.get_phase_config(PhaseName.FUZZING, is_supported=True, requires_links=False),
             self.get_phase_config(
                 PhaseName.STATEFUL_TESTING,
                 is_supported=self.schema.specification.supports_feature(SpecificationFeature.STATEFUL_TESTING),

schemathesis/engine/events.py

@@ -209,6 +209,13 @@ class NonFatalError(EngineEvent):
         self.label = label
         self.related_to_operation = related_to_operation
 
+    def __eq__(self, other: object) -> bool:
+        assert isinstance(other, NonFatalError)
+        return self.label == other.label and type(self.value) is type(other.value)
+
+    def __hash__(self) -> int:
+        return hash((self.label, type(self.value)))
+
 
 @dataclass
 class FatalError(EngineEvent):

schemathesis/engine/phases/__init__.py

@@ -14,14 +14,22 @@ class PhaseName(enum.Enum):
     """Available execution phases."""
 
     PROBING = "API probing"
-    UNIT_TESTING = "Unit testing"
-    STATEFUL_TESTING = "Stateful testing"
+    EXAMPLES = "Examples"
+    COVERAGE = "Coverage"
+    FUZZING = "Fuzzing"
+    STATEFUL_TESTING = "Stateful"
+
+    @classmethod
+    def defaults(cls) -> list[PhaseName]:
+        return [PhaseName.EXAMPLES, PhaseName.COVERAGE, PhaseName.FUZZING, PhaseName.STATEFUL_TESTING]
 
     @classmethod
     def from_str(cls, value: str) -> PhaseName:
         return {
             "probing": cls.PROBING,
-            "unit": cls.UNIT_TESTING,
+            "examples": cls.EXAMPLES,
+            "coverage": cls.COVERAGE,
+            "fuzzing": cls.FUZZING,
             "stateful": cls.STATEFUL_TESTING,
         }[value.lower()]
 
@@ -60,7 +68,11 @@ def execute(ctx: EngineContext, phase: Phase) -> EventGenerator:
 
         if phase.name == PhaseName.PROBING:
             yield from probes.execute(ctx, phase)
-        elif phase.name == PhaseName.UNIT_TESTING:
+        elif phase.name == PhaseName.EXAMPLES:
+            yield from unit.execute(ctx, phase)
+        elif phase.name == PhaseName.COVERAGE:
+            yield from unit.execute(ctx, phase)
+        elif phase.name == PhaseName.FUZZING:
             yield from unit.execute(ctx, phase)
         elif phase.name == PhaseName.STATEFUL_TESTING:
             yield from stateful.execute(ctx, phase)