schemathesis 4.0.0a3__py3-none-any.whl → 4.0.0a5__py3-none-any.whl

schemathesis/cli/__init__.py

@@ -5,7 +5,7 @@ from schemathesis.cli.commands.run.context import ExecutionContext
 from schemathesis.cli.commands.run.events import LoadingFinished, LoadingStarted
 from schemathesis.cli.commands.run.executor import handler
 from schemathesis.cli.commands.run.handlers import EventHandler
-from schemathesis.cli.ext.groups import GROUPS
+from schemathesis.cli.ext.groups import GROUPS, OptionGroup
 
 __all__ = [
     "schemathesis",
@@ -22,7 +22,7 @@ __all__ = [
 def add_group(name: str, *, index: int | None = None) -> Group:
     """Add a custom options group to `st run`."""
     if index is not None:
-        GROUPS.insert(index, name)
+        GROUPS[name] = OptionGroup(name=name, order=index)
     else:
-        GROUPS.append(name)
+        GROUPS[name] = OptionGroup(name=name)
     return Group(name)

schemathesis/cli/commands/run/__init__.py

@@ -1,24 +1,25 @@
 from __future__ import annotations
 
+from pathlib import Path
 from random import Random
 from typing import Any, Sequence
 
 import click
+from click.utils import LazyFile
 
 from schemathesis import contrib, experimental
 from schemathesis.checks import CHECKS
 from schemathesis.cli.commands.run import executor, validation
 from schemathesis.cli.commands.run.checks import CheckArguments
 from schemathesis.cli.commands.run.filters import FilterArguments, with_filters
-from schemathesis.cli.commands.run.handlers.cassettes import CassetteConfig, CassetteFormat
 from schemathesis.cli.commands.run.hypothesis import (
     HYPOTHESIS_IN_MEMORY_DATABASE_IDENTIFIER,
     HealthCheck,
-    Phase,
     prepare_health_checks,
     prepare_phases,
     prepare_settings,
 )
+from schemathesis.cli.commands.run.reports import DEFAULT_REPORT_DIRECTORY, ReportConfig, ReportFormat
 from schemathesis.cli.constants import DEFAULT_WORKERS, MAX_WORKERS, MIN_WORKERS
 from schemathesis.cli.core import ensure_color
 from schemathesis.cli.ext.groups import group, grouped_option
@@ -42,32 +43,21 @@ from schemathesis.specs.openapi.checks import *  # noqa: F401, F403
 
 COLOR_OPTIONS_INVALID_USAGE_MESSAGE = "Can't use `--no-color` and `--force-color` simultaneously"
 
-DEFAULT_PHASES = ("unit", "stateful")
+DEFAULT_PHASES = ("examples", "coverage", "fuzzing", "stateful")
 
 
 @click.argument("schema", type=str)  # type: ignore[misc]
 @group("Options")
 @grouped_option(
-    "--phases",
-    help="A comma-separated list of test phases to run",
-    type=CsvChoice(["unit", "stateful"]),
-    default=",".join(DEFAULT_PHASES),
-    metavar="",
-)
-@grouped_option(
-    "--base-url",
-    "-b",
-    help="Base URL of the API, required when schema is provided as a file",
+    "--url",
+    "-u",
+    "base_url",
+    help="API base URL (required for file-based schemas)",
+    metavar="URL",
     type=str,
     callback=validation.validate_base_url,
     envvar="SCHEMATHESIS_BASE_URL",
 )
-@grouped_option(
-    "--suppress-health-check",
-    help="A comma-separated list of Schemathesis health checks to disable",
-    type=CsvEnumChoice(HealthCheck),
-    metavar="",
-)
 @grouped_option(
     "--workers",
     "-w",
@@ -82,6 +72,26 @@ DEFAULT_PHASES = ("unit", "stateful")
     callback=validation.convert_workers,
     metavar="",
 )
+@grouped_option(
+    "--phases",
+    help="A comma-separated list of test phases to run",
+    type=CsvChoice(DEFAULT_PHASES),
+    default=",".join(DEFAULT_PHASES),
+    metavar="",
+)
+@grouped_option(
+    "--suppress-health-check",
+    help="A comma-separated list of Schemathesis health checks to disable",
+    type=CsvEnumChoice(HealthCheck),
+    metavar="",
+)
+@grouped_option(
+    "--wait-for-schema",
+    help="Maximum duration, in seconds, to wait for the API schema to become available. Disabled by default",
+    type=click.FloatRange(1.0),
+    default=None,
+    envvar="SCHEMATHESIS_WAIT_FOR_SCHEMA",
+)
 @group("API validation options")
 @grouped_option(
     "--checks",
@@ -107,62 +117,66 @@ DEFAULT_PHASES = ("unit", "stateful")
     metavar="",
 )
 @grouped_option(
-    "--max-response-time",
-    help="Time limit in seconds for API response times. The test will fail if a response time exceeds this limit",
-    type=click.FloatRange(min=0.0, min_open=True),
+    "--max-failures",
+    "max_failures",
+    type=click.IntRange(min=1),
+    help="Terminate the test suite after reaching a specified number of failures or errors",
+    show_default=True,
 )
 @grouped_option(
-    "-x",
-    "--exitfirst",
-    "exit_first",
+    "--continue-on-failure",
+    "continue_on_failure",
+    help="Continue executing all test cases within a scenario, even after encountering failures",
     is_flag=True,
     default=False,
-    help="Terminate the test suite immediately upon the first failure or error encountered",
-    show_default=True,
+    metavar="",
 )
 @grouped_option(
-    "--max-failures",
-    "max_failures",
-    type=click.IntRange(min=1),
-    help="Terminate the test suite after reaching a specified number of failures or errors",
-    show_default=True,
+    "--max-response-time",
+    help="Maximum allowed API response time in seconds",
+    type=click.FloatRange(min=0.0, min_open=True),
+    metavar="SECONDS",
+)
+@group(
+    "Filtering options",
+    description=(
+        "Filter operations by path, method, name, tag, or operation-id using:\n\n"
+        "--include-TYPE VALUE          Match operations with exact VALUE\n"
+        "--include-TYPE-regex PATTERN  Match operations using regular expression\n"
+        "--exclude-TYPE VALUE          Exclude operations with exact VALUE\n"
+        "--exclude-TYPE-regex PATTERN  Exclude operations using regular expression"
+    ),
 )
-@group("Filtering options")
 @with_filters
 @grouped_option(
     "--include-by",
     "include_by",
     type=str,
-    help="Include API operations by expression",
+    metavar="EXPR",
+    help="Include using custom expression",
 )
 @grouped_option(
     "--exclude-by",
     "exclude_by",
     type=str,
-    help="Exclude API operations by expression",
+    metavar="EXPR",
+    help="Exclude using custom expression",
 )
 @grouped_option(
     "--exclude-deprecated",
-    help="Exclude deprecated API operations from testing",
+    help="Skip deprecated operations",
     is_flag=True,
     is_eager=True,
     default=False,
     show_default=True,
 )
-@group("Loader options")
-@grouped_option(
-    "--wait-for-schema",
-    help="Maximum duration, in seconds, to wait for the API schema to become available. Disabled by default",
-    type=click.FloatRange(1.0),
-    default=None,
-    envvar="SCHEMATHESIS_WAIT_FOR_SCHEMA",
-)
 @group("Network requests options")
 @grouped_option(
     "--header",
     "-H",
     "headers",
-    help=r"Add a custom HTTP header to all API requests. Format: 'Header-Name: Value'",
+    help=r"Add a custom HTTP header to all API requests",
+    metavar="NAME:VALUE",
     multiple=True,
     type=str,
     callback=validation.validate_headers,
@@ -170,29 +184,40 @@ DEFAULT_PHASES = ("unit", "stateful")
 @grouped_option(
     "--auth",
     "-a",
-    help="Provide the server authentication details in the 'USER:PASSWORD' format",
+    help="Authenticate all API requests with basic authentication",
+    metavar="USER:PASS",
     type=str,
     callback=validation.validate_auth,
 )
 @grouped_option(
-    "--request-timeout",
-    help="Timeout limit, in seconds, for each network request during tests",
-    type=click.FloatRange(min=0.0, min_open=True),
-    default=DEFAULT_RESPONSE_TIMEOUT,
-)
-@grouped_option(
-    "--request-proxy",
+    "--proxy",
+    "request_proxy",
     help="Set the proxy for all network requests",
+    metavar="URL",
     type=str,
 )
 @grouped_option(
-    "--request-tls-verify",
-    help="Configures TLS certificate verification for server requests. Can specify path to CA_BUNDLE for custom certs",
+    "--tls-verify",
+    "request_tls_verify",
+    help="Path to CA bundle for TLS verification, or 'false' to disable",
     type=str,
     default="true",
     show_default=True,
     callback=validation.convert_boolean_string,
 )
+@grouped_option(
+    "--rate-limit",
+    help="Specify a rate limit for test requests in '<limit>/<duration>' format. "
+    "Example - `100/m` for 100 requests per minute",
+    type=str,
+    callback=validation.validate_rate_limit,
+)
+@grouped_option(
+    "--request-timeout",
+    help="Timeout limit, in seconds, for each network request during tests",
+    type=click.FloatRange(min=0.0, min_open=True),
+    default=DEFAULT_RESPONSE_TIMEOUT,
+)
 @grouped_option(
     "--request-cert",
     help="File path of unencrypted client certificate for authentication. "
@@ -210,45 +235,56 @@ DEFAULT_PHASES = ("unit", "stateful")
     show_default=False,
     callback=validation.validate_request_cert_key,
 )
+@group("Output options")
 @grouped_option(
-    "--rate-limit",
-    help="Specify a rate limit for test requests in '<limit>/<duration>' format. "
-    "Example - `100/m` for 100 requests per minute",
-    type=str,
-    callback=validation.validate_rate_limit,
+    "--report",
+    "report_formats",
+    help="Generate test reports in specified formats",
+    type=CsvEnumChoice(ReportFormat),
+    is_eager=True,
+    metavar="FORMAT",
 )
-@group("Output options")
 @grouped_option(
-    "--junit-xml",
-    help="Output a JUnit-XML style report at the specified file path",
+    "--report-dir",
+    help="Directory to store all report files",
+    type=click.Path(file_okay=False, dir_okay=True),
+    default=DEFAULT_REPORT_DIRECTORY,
+    show_default=True,
+)
+@grouped_option(
+    "--report-junit-path",
+    help="Custom path for JUnit XML report",
     type=click.File("w", encoding="utf-8"),
+    is_eager=True,
 )
 @grouped_option(
-    "--cassette-path",
-    help="Save the test outcomes in a VCR-compatible format",
+    "--report-vcr-path",
+    help="Custom path for VCR cassette",
     type=click.File("w", encoding="utf-8"),
     is_eager=True,
 )
 @grouped_option(
-    "--cassette-format",
-    help="Format of the saved cassettes",
-    type=click.Choice([item.name.lower() for item in CassetteFormat]),
-    default=CassetteFormat.VCR.name.lower(),
-    callback=validation.convert_cassette_format,
-    metavar="",
+    "--report-har-path",
+    help="Custom path for HAR file",
+    type=click.File("w", encoding="utf-8"),
+    is_eager=True,
 )
 @grouped_option(
-    "--cassette-preserve-exact-body-bytes",
+    "--report-preserve-bytes",
     help="Retain exact byte sequence of payloads in cassettes, encoded as base64",
+    type=bool,
     is_flag=True,
-    callback=validation.validate_preserve_exact_body_bytes,
+    default=False,
+    callback=validation.validate_preserve_bytes,
 )
 @grouped_option(
     "--output-sanitize",
-    type=bool,
-    default=True,
+    type=str,
+    default="true",
     show_default=True,
     help="Enable or disable automatic output sanitization to obscure sensitive data",
+    metavar="BOOLEAN",
+    callback=validation.convert_boolean_string,
 )
 @grouped_option(
     "--output-truncate",
@@ -256,6 +292,7 @@ DEFAULT_PHASES = ("unit", "stateful")
     type=str,
     default="true",
     show_default=True,
+    metavar="BOOLEAN",
     callback=validation.convert_boolean_string,
 )
 @group("Experimental options")
@@ -268,15 +305,6 @@ DEFAULT_PHASES = ("unit", "stateful")
     multiple=True,
     metavar="FEATURES",
 )
-@grouped_option(
-    "--experimental-no-failfast",
-    "no_failfast",
-    help="Continue testing an API operation after a failure is found",
-    is_flag=True,
-    default=False,
-    metavar="",
-    envvar="SCHEMATHESIS_EXPERIMENTAL_NO_FAILFAST",
-)
 @grouped_option(
     "--experimental-missing-required-header-allowed-statuses",
     "missing_required_header_allowed_statuses",
@@ -306,10 +334,10 @@ DEFAULT_PHASES = ("unit", "stateful")
 )
 @group("Data generation options")
 @grouped_option(
-    "--generation-mode",
+    "--mode",
+    "-m",
     "generation_modes",
-    help="Specify the approach Schemathesis uses to generate test data. "
-    "Use 'positive' for valid data, 'negative' for invalid data, or 'all' for both",
+    help="Test data generation mode",
     type=click.Choice([item.value for item in GenerationMode] + ["all"]),
     default=GenerationMode.default().value,
     callback=validation.convert_generation_mode,
@@ -317,14 +345,23 @@ DEFAULT_PHASES = ("unit", "stateful")
     metavar="",
 )
 @grouped_option(
-    "--generation-seed",
-    help="Seed value for Schemathesis, ensuring reproducibility across test runs",
+    "--max-examples",
+    "-n",
+    "generation_max_examples",
+    help="Maximum number of test cases per API operation",
+    type=click.IntRange(1),
+)
+@grouped_option(
+    "--seed",
+    "generation_seed",
+    help="Random seed for reproducible test runs",
     type=int,
 )
 @grouped_option(
-    "--generation-max-examples",
-    help="The cap on the number of examples generated by Schemathesis for each API operation",
-    type=click.IntRange(1),
+    "--no-shrink",
+    "generation_no_shrink",
+    help="Disable test case shrinking. Makes test failures harder to debug but improves performance",
+    is_flag=True,
 )
 @grouped_option(
     "--generation-deterministic",
@@ -336,10 +373,11 @@ DEFAULT_PHASES = ("unit", "stateful")
 )
 @grouped_option(
     "--generation-allow-x00",
-    help="Whether to allow the generation of `\x00` bytes within strings",
+    help="Whether to allow the generation of 'NULL' bytes within strings",
     type=str,
     default="true",
     show_default=True,
+    metavar="BOOLEAN",
     callback=validation.convert_boolean_string,
 )
 @grouped_option(
@@ -350,15 +388,15 @@ DEFAULT_PHASES = ("unit", "stateful")
     callback=validation.validate_generation_codec,
 )
 @grouped_option(
-    "--generation-optimize",
-    "generation_optimize",
+    "--generation-maximize",
+    "generation_maximize",
     multiple=True,
     help="Guide input generation to values more likely to expose bugs via targeted property-based testing",
     type=RegistryChoice(TARGETS),
     default=None,
     callback=validation.reduce_list,
     show_default=True,
-    metavar="TARGET",
+    metavar="METRIC",
 )
 @grouped_option(
     "--generation-with-security-parameters",
@@ -437,19 +475,6 @@ DEFAULT_PHASES = ("unit", "stateful")
     type=str,
     callback=validation.validate_set_path,
 )
-@group("Hypothesis engine options")
-@grouped_option(
-    "--hypothesis-phases",
-    help="Testing phases to execute",
-    type=CsvEnumChoice(Phase),
-    metavar="",
-)
-@grouped_option(
-    "--hypothesis-no-phases",
-    help="Testing phases to exclude from execution",
-    type=CsvEnumChoice(Phase),
-    metavar="",
-)
 @group("Global options")
 @grouped_option("--no-color", help="Disable ANSI color escape codes", type=bool, is_flag=True)
 @grouped_option("--force-color", help="Explicitly tells to enable ANSI color escape codes", type=bool, is_flag=True)
@@ -464,16 +489,15 @@ def run(
     set_cookie: dict[str, str],
     set_path: dict[str, str],
     experiments: list,
-    no_failfast: bool,
     missing_required_header_allowed_statuses: list[str],
     positive_data_acceptance_allowed_statuses: list[str],
     negative_data_rejection_allowed_statuses: list[str],
     included_check_names: Sequence[str],
     excluded_check_names: Sequence[str],
-    phases: Sequence[str] = DEFAULT_PHASES,
     max_response_time: float | None = None,
-    exit_first: bool = False,
+    phases: Sequence[str] = DEFAULT_PHASES,
     max_failures: int | None = None,
+    continue_on_failure: bool = False,
     include_path: Sequence[str] = (),
     include_path_regex: str | None = None,
     include_method: Sequence[str] = (),
@@ -499,27 +523,27 @@ def run(
     exclude_deprecated: bool = False,
     workers_num: int = DEFAULT_WORKERS,
     base_url: str | None = None,
+    wait_for_schema: float | None = None,
+    rate_limit: str | None = None,
     suppress_health_check: list[HealthCheck] | None = None,
     request_timeout: int | None = None,
     request_tls_verify: bool = True,
     request_cert: str | None = None,
     request_cert_key: str | None = None,
     request_proxy: str | None = None,
-    junit_xml: click.utils.LazyFile | None = None,
-    cassette_path: click.utils.LazyFile | None = None,
-    cassette_format: CassetteFormat = CassetteFormat.VCR,
-    cassette_preserve_exact_body_bytes: bool = False,
-    wait_for_schema: float | None = None,
-    rate_limit: str | None = None,
+    report_formats: list[ReportFormat] | None = None,
+    report_dir: Path = DEFAULT_REPORT_DIRECTORY,
+    report_junit_path: LazyFile | None = None,
+    report_vcr_path: LazyFile | None = None,
+    report_har_path: LazyFile | None = None,
+    report_preserve_bytes: bool = False,
     output_sanitize: bool = True,
     output_truncate: bool = True,
     contrib_openapi_fill_missing_examples: bool = False,
-    hypothesis_phases: list[Phase] | None = None,
-    hypothesis_no_phases: list[Phase] | None = None,
     generation_modes: tuple[GenerationMode, ...] = DEFAULT_GENERATOR_MODES,
     generation_seed: int | None = None,
     generation_max_examples: int | None = None,
-    generation_optimize: Sequence[str] | None = None,
+    generation_maximize: Sequence[str] | None = None,
     generation_deterministic: bool | None = None,
     generation_database: str | None = None,
     generation_unique_inputs: bool = False,
@@ -527,6 +551,7 @@ def run(
     generation_graphql_allow_null: bool = True,
     generation_with_security_parameters: bool = True,
     generation_codec: str = "utf-8",
+    generation_no_shrink: bool = False,
     force_color: bool = False,
     no_color: bool = False,
     **__kwargs: Any,
@@ -541,7 +566,7 @@ def run(
 
     validation.validate_schema(schema, base_url)
 
-    _hypothesis_phases = prepare_phases(hypothesis_phases, hypothesis_no_phases)
+    _hypothesis_phases = prepare_phases(generation_no_shrink)
     _hypothesis_suppress_health_check = prepare_health_checks(suppress_health_check)
 
     for experiment in experiments:
@@ -588,16 +613,16 @@ def run(
         max_response_time=max_response_time,
     ).into()
 
-    if exit_first and max_failures is None:
-        max_failures = 1
-
-    cassette_config = None
-    if cassette_path is not None:
-        cassette_config = CassetteConfig(
-            path=cassette_path,
-            format=cassette_format,
+    report_config = None
+    if report_formats or report_junit_path or report_vcr_path or report_har_path:
+        report_config = ReportConfig(
+            formats=report_formats,
+            directory=Path(report_dir),
+            junit_path=report_junit_path if report_junit_path else None,
+            vcr_path=report_vcr_path if report_vcr_path else None,
+            har_path=report_har_path if report_har_path else None,
+            preserve_bytes=report_preserve_bytes,
             sanitize_output=output_sanitize,
-            preserve_exact_body_bytes=cassette_preserve_exact_body_bytes,
         )
 
     # Use the same seed for all tests unless `derandomize=True` is used
@@ -616,7 +641,7 @@ def run(
             execution=ExecutionConfig(
                 phases=phases_,
                 checks=selected_checks,
-                targets=TARGETS.get_by_names(generation_optimize or []),
+                targets=TARGETS.get_by_names(generation_maximize or []),
                 hypothesis_settings=prepare_settings(
                     database=generation_database,
                     derandomize=generation_deterministic,
@@ -632,7 +657,7 @@ def run(
                     with_security_parameters=generation_with_security_parameters,
                 ),
                 max_failures=max_failures,
-                no_failfast=no_failfast,
+                continue_on_failure=continue_on_failure,
                 unique_inputs=generation_unique_inputs,
                 seed=seed,
                 workers_num=workers_num,
@@ -654,8 +679,7 @@ def run(
         wait_for_schema=wait_for_schema,
         rate_limit=rate_limit,
         output=OutputConfig(sanitize=output_sanitize, truncate=output_truncate),
-        cassette=cassette_config,
-        junit_xml=junit_xml,
+        report=report_config,
         args=ctx.args,
         params=ctx.params,
     )

schemathesis/cli/commands/run/checks.py

@@ -69,10 +69,9 @@ class CheckArguments:
             checks_config[_max_response_time] = MaxResponseTimeConfig(self.max_response_time)
             selected_checks.append(_max_response_time)
 
-        if experimental.COVERAGE_PHASE.is_enabled:
-            from schemathesis.specs.openapi.checks import unsupported_method
+        from schemathesis.specs.openapi.checks import unsupported_method
 
-            selected_checks.append(unsupported_method)
+        selected_checks.append(unsupported_method)
 
         # Exclude checks based on their names
         selected_checks = [check for check in selected_checks if check.__name__ not in self.excluded_check_names]