schemathesis 4.0.0a4__py3-none-any.whl → 4.0.0a5__py3-none-any.whl

schemathesis/cli/commands/run/__init__.py

@@ -15,7 +15,6 @@ from schemathesis.cli.commands.run.filters import FilterArguments, with_filters
 from schemathesis.cli.commands.run.hypothesis import (
     HYPOTHESIS_IN_MEMORY_DATABASE_IDENTIFIER,
     HealthCheck,
-    Phase,
     prepare_health_checks,
     prepare_phases,
     prepare_settings,
@@ -44,7 +43,7 @@ from schemathesis.specs.openapi.checks import *  # noqa: F401, F403
 
 COLOR_OPTIONS_INVALID_USAGE_MESSAGE = "Can't use `--no-color` and `--force-color` simultaneously"
 
-DEFAULT_PHASES = ("unit", "stateful")
+DEFAULT_PHASES = ("examples", "coverage", "fuzzing", "stateful")
 
 
 @click.argument("schema", type=str)  # type: ignore[misc]
@@ -76,7 +75,7 @@ DEFAULT_PHASES = ("unit", "stateful")
 @grouped_option(
     "--phases",
     help="A comma-separated list of test phases to run",
-    type=CsvChoice(["unit", "stateful"]),
+    type=CsvChoice(DEFAULT_PHASES),
     default=",".join(DEFAULT_PHASES),
     metavar="",
 )
@@ -117,15 +116,6 @@ DEFAULT_PHASES = ("unit", "stateful")
     show_default=True,
     metavar="",
 )
-@grouped_option(
-    "-x",
-    "--exitfirst",
-    "exit_first",
-    is_flag=True,
-    default=False,
-    help="Terminate the test suite immediately upon the first failure or error encountered",
-    show_default=True,
-)
 @grouped_option(
     "--max-failures",
     "max_failures",
@@ -133,6 +123,14 @@ DEFAULT_PHASES = ("unit", "stateful")
     help="Terminate the test suite after reaching a specified number of failures or errors",
     show_default=True,
 )
+@grouped_option(
+    "--continue-on-failure",
+    "continue_on_failure",
+    help="Continue executing all test cases within a scenario, even after encountering failures",
+    is_flag=True,
+    default=False,
+    metavar="",
+)
 @grouped_option(
     "--max-response-time",
     help="Maximum allowed API response time in seconds",
@@ -307,15 +305,6 @@ DEFAULT_PHASES = ("unit", "stateful")
     multiple=True,
     metavar="FEATURES",
 )
-@grouped_option(
-    "--experimental-no-failfast",
-    "no_failfast",
-    help="Continue testing an API operation after a failure is found",
-    is_flag=True,
-    default=False,
-    metavar="",
-    envvar="SCHEMATHESIS_EXPERIMENTAL_NO_FAILFAST",
-)
 @grouped_option(
     "--experimental-missing-required-header-allowed-statuses",
     "missing_required_header_allowed_statuses",
@@ -407,7 +396,7 @@ DEFAULT_PHASES = ("unit", "stateful")
     default=None,
     callback=validation.reduce_list,
     show_default=True,
-    metavar="TARGET",
+    metavar="METRIC",
 )
 @grouped_option(
     "--generation-with-security-parameters",
@@ -486,19 +475,6 @@ DEFAULT_PHASES = ("unit", "stateful")
     type=str,
     callback=validation.validate_set_path,
 )
-@group("Hypothesis engine options")
-@grouped_option(
-    "--hypothesis-phases",
-    help="Testing phases to execute",
-    type=CsvEnumChoice(Phase),
-    metavar="",
-)
-@grouped_option(
-    "--hypothesis-no-phases",
-    help="Testing phases to exclude from execution",
-    type=CsvEnumChoice(Phase),
-    metavar="",
-)
 @group("Global options")
 @grouped_option("--no-color", help="Disable ANSI color escape codes", type=bool, is_flag=True)
 @grouped_option("--force-color", help="Explicitly tells to enable ANSI color escape codes", type=bool, is_flag=True)
@@ -513,7 +489,6 @@ def run(
     set_cookie: dict[str, str],
     set_path: dict[str, str],
     experiments: list,
-    no_failfast: bool,
     missing_required_header_allowed_statuses: list[str],
     positive_data_acceptance_allowed_statuses: list[str],
     negative_data_rejection_allowed_statuses: list[str],
@@ -521,8 +496,8 @@ def run(
     excluded_check_names: Sequence[str],
     max_response_time: float | None = None,
     phases: Sequence[str] = DEFAULT_PHASES,
-    exit_first: bool = False,
     max_failures: int | None = None,
+    continue_on_failure: bool = False,
     include_path: Sequence[str] = (),
     include_path_regex: str | None = None,
     include_method: Sequence[str] = (),
@@ -565,8 +540,6 @@ def run(
     output_sanitize: bool = True,
     output_truncate: bool = True,
     contrib_openapi_fill_missing_examples: bool = False,
-    hypothesis_phases: list[Phase] | None = None,
-    hypothesis_no_phases: list[Phase] | None = None,
     generation_modes: tuple[GenerationMode, ...] = DEFAULT_GENERATOR_MODES,
     generation_seed: int | None = None,
     generation_max_examples: int | None = None,
@@ -593,7 +566,7 @@ def run(
 
     validation.validate_schema(schema, base_url)
 
-    _hypothesis_phases = prepare_phases(hypothesis_phases, hypothesis_no_phases, generation_no_shrink)
+    _hypothesis_phases = prepare_phases(generation_no_shrink)
     _hypothesis_suppress_health_check = prepare_health_checks(suppress_health_check)
 
     for experiment in experiments:
@@ -640,9 +613,6 @@ def run(
         max_response_time=max_response_time,
     ).into()
 
-    if exit_first and max_failures is None:
-        max_failures = 1
-
     report_config = None
     if report_formats or report_junit_path or report_vcr_path or report_har_path:
         report_config = ReportConfig(
@@ -687,7 +657,7 @@ def run(
                     with_security_parameters=generation_with_security_parameters,
                 ),
                 max_failures=max_failures,
-                no_failfast=no_failfast,
+                continue_on_failure=continue_on_failure,
                 unique_inputs=generation_unique_inputs,
                 seed=seed,
                 workers_num=workers_num,

schemathesis/cli/commands/run/checks.py

@@ -69,10 +69,9 @@ class CheckArguments:
             checks_config[_max_response_time] = MaxResponseTimeConfig(self.max_response_time)
             selected_checks.append(_max_response_time)
 
-        if experimental.COVERAGE_PHASE.is_enabled:
-            from schemathesis.specs.openapi.checks import unsupported_method
+        from schemathesis.specs.openapi.checks import unsupported_method
 
-            selected_checks.append(unsupported_method)
+        selected_checks.append(unsupported_method)
 
         # Exclude checks based on their names
         selected_checks = [check for check in selected_checks if check.__name__ not in self.excluded_check_names]

schemathesis/cli/commands/run/context.py

@@ -20,8 +20,9 @@ if TYPE_CHECKING:
 class Statistic:
     """Running statistics about test execution."""
 
-    outcomes: dict[Status, int]
     failures: dict[str, dict[str, GroupedFailures]]
+    # Track first case_id where each unique failure was found
+    unique_failures_map: dict[Failure, str]
 
     extraction_failures: set[ExtractionFailure]
 
@@ -32,8 +33,8 @@ class Statistic:
     cases_without_checks: int
 
     __slots__ = (
-        "outcomes",
         "failures",
+        "unique_failures_map",
         "extraction_failures",
         "tested_operations",
         "total_cases",
@@ -42,15 +43,15 @@ class Statistic:
     )
 
     def __init__(self) -> None:
-        self.outcomes = {}
         self.failures = {}
+        self.unique_failures_map = {}
         self.extraction_failures = set()
         self.tested_operations = set()
         self.total_cases = 0
         self.cases_with_failures = 0
         self.cases_without_checks = 0
 
-    def record_checks(self, recorder: ScenarioRecorder) -> None:
+    def on_scenario_finished(self, recorder: ScenarioRecorder) -> None:
         """Update statistics and store failures from a new batch of checks."""
         from schemathesis.generation.stateful.state_machine import ExtractionFailure
 
@@ -80,20 +81,32 @@ class Statistic:
 
             self.tested_operations.add(case.value.operation.label)
             has_failures = False
-            for check in checks:
-                response = recorder.interactions[case_id].response
+            current_case_failures = []
+            last_failure_info = None
 
-                # Collect failures
+            for check in checks:
                 if check.failure_info is not None:
-                    has_failures = True
-                    if case_id not in failures:
-                        failures[case_id] = GroupedFailures(
-                            case_id=case_id,
-                            code_sample=check.failure_info.code_sample,
-                            failures=[],
-                            response=response,
-                        )
-                    failures[case_id].failures.append(check.failure_info.failure)
+                    failure = check.failure_info.failure
+
+                    # Check if this is a new unique failure
+                    if failure not in self.unique_failures_map:
+                        last_failure_info = check.failure_info
+                        self.unique_failures_map[failure] = case_id
+                        current_case_failures.append(failure)
+                        has_failures = True
+                    else:
+                        # This failure was already seen - skip it
+                        continue
+
+            if current_case_failures:
+                assert last_failure_info is not None
+                failures[case_id] = GroupedFailures(
+                    case_id=case_id,
+                    code_sample=last_failure_info.code_sample,
+                    failures=current_case_failures,
+                    response=recorder.interactions[case_id].response,
+                )
+
             if has_failures:
                 self.cases_with_failures += 1
 
@@ -178,7 +191,7 @@ class ExecutionContext:
 
     def on_event(self, event: events.EngineEvent) -> None:
         if isinstance(event, events.ScenarioFinished):
-            self.statistic.record_checks(event.recorder)
+            self.statistic.on_scenario_finished(event.recorder)
         elif isinstance(event, events.NonFatalError) or (
             isinstance(event, events.PhaseFinished)
             and event.phase.is_enabled

schemathesis/cli/commands/run/executor.py

@@ -125,6 +125,7 @@ def initialize_handlers(config: RunConfig) -> list[EventHandler]:
             seed=config.engine.execution.seed,
             rate_limit=config.rate_limit,
             wait_for_schema=config.wait_for_schema,
+            engine_config=config.engine,
             report_config=config.report,
         )
     )