schemathesis 4.0.0a2__py3-none-any.whl → 4.0.0a4__py3-none-any.whl

schemathesis/cli/__init__.py

@@ -1,17 +1,28 @@
 from __future__ import annotations
 
 from schemathesis.cli.commands import Group, run, schemathesis
+from schemathesis.cli.commands.run.context import ExecutionContext
+from schemathesis.cli.commands.run.events import LoadingFinished, LoadingStarted
 from schemathesis.cli.commands.run.executor import handler
 from schemathesis.cli.commands.run.handlers import EventHandler
-from schemathesis.cli.ext.groups import GROUPS
+from schemathesis.cli.ext.groups import GROUPS, OptionGroup
 
-__all__ = ["schemathesis", "run", "EventHandler", "add_group", "handler"]
+__all__ = [
+    "schemathesis",
+    "run",
+    "EventHandler",
+    "ExecutionContext",
+    "LoadingStarted",
+    "LoadingFinished",
+    "add_group",
+    "handler",
+]
 
 
 def add_group(name: str, *, index: int | None = None) -> Group:
     """Add a custom options group to `st run`."""
     if index is not None:
-        GROUPS.insert(index, name)
+        GROUPS[name] = OptionGroup(name=name, order=index)
     else:
-        GROUPS.append(name)
+        GROUPS[name] = OptionGroup(name=name)
     return Group(name)

schemathesis/cli/commands/run/__init__.py

@@ -1,16 +1,17 @@
 from __future__ import annotations
 
+from pathlib import Path
 from random import Random
 from typing import Any, Sequence
 
 import click
+from click.utils import LazyFile
 
 from schemathesis import contrib, experimental
 from schemathesis.checks import CHECKS
 from schemathesis.cli.commands.run import executor, validation
 from schemathesis.cli.commands.run.checks import CheckArguments
 from schemathesis.cli.commands.run.filters import FilterArguments, with_filters
-from schemathesis.cli.commands.run.handlers.cassettes import CassetteConfig, CassetteFormat
 from schemathesis.cli.commands.run.hypothesis import (
     HYPOTHESIS_IN_MEMORY_DATABASE_IDENTIFIER,
     HealthCheck,
@@ -19,6 +20,7 @@ from schemathesis.cli.commands.run.hypothesis import (
     prepare_phases,
     prepare_settings,
 )
+from schemathesis.cli.commands.run.reports import DEFAULT_REPORT_DIRECTORY, ReportConfig, ReportFormat
 from schemathesis.cli.constants import DEFAULT_WORKERS, MAX_WORKERS, MIN_WORKERS
 from schemathesis.cli.core import ensure_color
 from schemathesis.cli.ext.groups import group, grouped_option
@@ -48,26 +50,15 @@ DEFAULT_PHASES = ("unit", "stateful")
 @click.argument("schema", type=str)  # type: ignore[misc]
 @group("Options")
 @grouped_option(
-    "--phases",
-    help="A comma-separated list of test phases to run",
-    type=CsvChoice(["unit", "stateful"]),
-    default=",".join(DEFAULT_PHASES),
-    metavar="",
-)
-@grouped_option(
-    "--base-url",
-    "-b",
-    help="Base URL of the API, required when schema is provided as a file",
+    "--url",
+    "-u",
+    "base_url",
+    help="API base URL (required for file-based schemas)",
+    metavar="URL",
     type=str,
     callback=validation.validate_base_url,
     envvar="SCHEMATHESIS_BASE_URL",
 )
-@grouped_option(
-    "--suppress-health-check",
-    help="A comma-separated list of Schemathesis health checks to disable",
-    type=CsvEnumChoice(HealthCheck),
-    metavar="",
-)
 @grouped_option(
     "--workers",
     "-w",
@@ -82,6 +73,26 @@ DEFAULT_PHASES = ("unit", "stateful")
     callback=validation.convert_workers,
     metavar="",
 )
+@grouped_option(
+    "--phases",
+    help="A comma-separated list of test phases to run",
+    type=CsvChoice(["unit", "stateful"]),
+    default=",".join(DEFAULT_PHASES),
+    metavar="",
+)
+@grouped_option(
+    "--suppress-health-check",
+    help="A comma-separated list of Schemathesis health checks to disable",
+    type=CsvEnumChoice(HealthCheck),
+    metavar="",
+)
+@grouped_option(
+    "--wait-for-schema",
+    help="Maximum duration, in seconds, to wait for the API schema to become available. Disabled by default",
+    type=click.FloatRange(1.0),
+    default=None,
+    envvar="SCHEMATHESIS_WAIT_FOR_SCHEMA",
+)
 @group("API validation options")
 @grouped_option(
     "--checks",
@@ -106,11 +117,6 @@ DEFAULT_PHASES = ("unit", "stateful")
     show_default=True,
     metavar="",
 )
-@grouped_option(
-    "--max-response-time",
-    help="Time limit in seconds for API response times. The test will fail if a response time exceeds this limit",
-    type=click.FloatRange(min=0.0, min_open=True),
-)
 @grouped_option(
     "-x",
     "--exitfirst",
@@ -127,42 +133,52 @@ DEFAULT_PHASES = ("unit", "stateful")
     help="Terminate the test suite after reaching a specified number of failures or errors",
     show_default=True,
 )
-@group("Filtering options")
+@grouped_option(
+    "--max-response-time",
+    help="Maximum allowed API response time in seconds",
+    type=click.FloatRange(min=0.0, min_open=True),
+    metavar="SECONDS",
+)
+@group(
+    "Filtering options",
+    description=(
+        "Filter operations by path, method, name, tag, or operation-id using:\n\n"
+        "--include-TYPE VALUE          Match operations with exact VALUE\n"
+        "--include-TYPE-regex PATTERN  Match operations using regular expression\n"
+        "--exclude-TYPE VALUE          Exclude operations with exact VALUE\n"
+        "--exclude-TYPE-regex PATTERN  Exclude operations using regular expression"
+    ),
+)
 @with_filters
 @grouped_option(
     "--include-by",
     "include_by",
     type=str,
-    help="Include API operations by expression",
+    metavar="EXPR",
+    help="Include using custom expression",
 )
 @grouped_option(
     "--exclude-by",
     "exclude_by",
     type=str,
-    help="Exclude API operations by expression",
+    metavar="EXPR",
+    help="Exclude using custom expression",
 )
 @grouped_option(
     "--exclude-deprecated",
-    help="Exclude deprecated API operations from testing",
+    help="Skip deprecated operations",
     is_flag=True,
     is_eager=True,
     default=False,
     show_default=True,
 )
-@group("Loader options")
-@grouped_option(
-    "--wait-for-schema",
-    help="Maximum duration, in seconds, to wait for the API schema to become available. Disabled by default",
-    type=click.FloatRange(1.0),
-    default=None,
-    envvar="SCHEMATHESIS_WAIT_FOR_SCHEMA",
-)
 @group("Network requests options")
 @grouped_option(
     "--header",
     "-H",
     "headers",
-    help=r"Add a custom HTTP header to all API requests. Format: 'Header-Name: Value'",
+    help=r"Add a custom HTTP header to all API requests",
+    metavar="NAME:VALUE",
     multiple=True,
     type=str,
     callback=validation.validate_headers,
@@ -170,29 +186,40 @@ DEFAULT_PHASES = ("unit", "stateful")
 @grouped_option(
     "--auth",
     "-a",
-    help="Provide the server authentication details in the 'USER:PASSWORD' format",
+    help="Authenticate all API requests with basic authentication",
+    metavar="USER:PASS",
     type=str,
     callback=validation.validate_auth,
 )
 @grouped_option(
-    "--request-timeout",
-    help="Timeout limit, in seconds, for each network request during tests",
-    type=click.FloatRange(min=0.0, min_open=True),
-    default=DEFAULT_RESPONSE_TIMEOUT,
-)
-@grouped_option(
-    "--request-proxy",
+    "--proxy",
+    "request_proxy",
     help="Set the proxy for all network requests",
+    metavar="URL",
     type=str,
 )
 @grouped_option(
-    "--request-tls-verify",
-    help="Configures TLS certificate verification for server requests. Can specify path to CA_BUNDLE for custom certs",
+    "--tls-verify",
+    "request_tls_verify",
+    help="Path to CA bundle for TLS verification, or 'false' to disable",
     type=str,
     default="true",
     show_default=True,
     callback=validation.convert_boolean_string,
 )
+@grouped_option(
+    "--rate-limit",
+    help="Specify a rate limit for test requests in '<limit>/<duration>' format. "
+    "Example - `100/m` for 100 requests per minute",
+    type=str,
+    callback=validation.validate_rate_limit,
+)
+@grouped_option(
+    "--request-timeout",
+    help="Timeout limit, in seconds, for each network request during tests",
+    type=click.FloatRange(min=0.0, min_open=True),
+    default=DEFAULT_RESPONSE_TIMEOUT,
+)
 @grouped_option(
     "--request-cert",
     help="File path of unencrypted client certificate for authentication. "
@@ -210,45 +237,56 @@ DEFAULT_PHASES = ("unit", "stateful")
     show_default=False,
     callback=validation.validate_request_cert_key,
 )
+@group("Output options")
 @grouped_option(
-    "--rate-limit",
-    help="Specify a rate limit for test requests in '<limit>/<duration>' format. "
-    "Example - `100/m` for 100 requests per minute",
-    type=str,
-    callback=validation.validate_rate_limit,
+    "--report",
+    "report_formats",
+    help="Generate test reports in specified formats",
+    type=CsvEnumChoice(ReportFormat),
+    is_eager=True,
+    metavar="FORMAT",
 )
-@group("Output options")
 @grouped_option(
-    "--junit-xml",
-    help="Output a JUnit-XML style report at the specified file path",
+    "--report-dir",
+    help="Directory to store all report files",
+    type=click.Path(file_okay=False, dir_okay=True),
+    default=DEFAULT_REPORT_DIRECTORY,
+    show_default=True,
+)
+@grouped_option(
+    "--report-junit-path",
+    help="Custom path for JUnit XML report",
     type=click.File("w", encoding="utf-8"),
+    is_eager=True,
 )
 @grouped_option(
-    "--cassette-path",
-    help="Save the test outcomes in a VCR-compatible format",
+    "--report-vcr-path",
+    help="Custom path for VCR cassette",
     type=click.File("w", encoding="utf-8"),
     is_eager=True,
 )
 @grouped_option(
-    "--cassette-format",
-    help="Format of the saved cassettes",
-    type=click.Choice([item.name.lower() for item in CassetteFormat]),
-    default=CassetteFormat.VCR.name.lower(),
-    callback=validation.convert_cassette_format,
-    metavar="",
+    "--report-har-path",
+    help="Custom path for HAR file",
+    type=click.File("w", encoding="utf-8"),
+    is_eager=True,
 )
 @grouped_option(
-    "--cassette-preserve-exact-body-bytes",
+    "--report-preserve-bytes",
     help="Retain exact byte sequence of payloads in cassettes, encoded as base64",
+    type=bool,
     is_flag=True,
-    callback=validation.validate_preserve_exact_body_bytes,
+    default=False,
+    callback=validation.validate_preserve_bytes,
 )
 @grouped_option(
     "--output-sanitize",
-    type=bool,
-    default=True,
+    type=str,
+    default="true",
     show_default=True,
     help="Enable or disable automatic output sanitization to obscure sensitive data",
+    metavar="BOOLEAN",
+    callback=validation.convert_boolean_string,
 )
 @grouped_option(
     "--output-truncate",
@@ -256,6 +294,7 @@ DEFAULT_PHASES = ("unit", "stateful")
     type=str,
     default="true",
     show_default=True,
+    metavar="BOOLEAN",
     callback=validation.convert_boolean_string,
 )
 @group("Experimental options")
@@ -306,10 +345,10 @@ DEFAULT_PHASES = ("unit", "stateful")
 )
 @group("Data generation options")
 @grouped_option(
-    "--generation-mode",
+    "--mode",
+    "-m",
     "generation_modes",
-    help="Specify the approach Schemathesis uses to generate test data. "
-    "Use 'positive' for valid data, 'negative' for invalid data, or 'all' for both",
+    help="Test data generation mode",
     type=click.Choice([item.value for item in GenerationMode] + ["all"]),
     default=GenerationMode.default().value,
     callback=validation.convert_generation_mode,
@@ -317,14 +356,23 @@ DEFAULT_PHASES = ("unit", "stateful")
     metavar="",
 )
 @grouped_option(
-    "--generation-seed",
-    help="Seed value for Schemathesis, ensuring reproducibility across test runs",
+    "--max-examples",
+    "-n",
+    "generation_max_examples",
+    help="Maximum number of test cases per API operation",
+    type=click.IntRange(1),
+)
+@grouped_option(
+    "--seed",
+    "generation_seed",
+    help="Random seed for reproducible test runs",
     type=int,
 )
 @grouped_option(
-    "--generation-max-examples",
-    help="The cap on the number of examples generated by Schemathesis for each API operation",
-    type=click.IntRange(1),
+    "--no-shrink",
+    "generation_no_shrink",
+    help="Disable test case shrinking. Makes test failures harder to debug but improves performance",
+    is_flag=True,
 )
 @grouped_option(
     "--generation-deterministic",
@@ -336,10 +384,11 @@ DEFAULT_PHASES = ("unit", "stateful")
 )
 @grouped_option(
     "--generation-allow-x00",
-    help="Whether to allow the generation of `\x00` bytes within strings",
+    help="Whether to allow the generation of 'NULL' bytes within strings",
     type=str,
     default="true",
     show_default=True,
+    metavar="BOOLEAN",
     callback=validation.convert_boolean_string,
 )
 @grouped_option(
@@ -350,8 +399,8 @@ DEFAULT_PHASES = ("unit", "stateful")
     callback=validation.validate_generation_codec,
 )
 @grouped_option(
-    "--generation-optimize",
-    "generation_optimize",
+    "--generation-maximize",
+    "generation_maximize",
     multiple=True,
     help="Guide input generation to values more likely to expose bugs via targeted property-based testing",
     type=RegistryChoice(TARGETS),
@@ -470,8 +519,8 @@ def run(
     negative_data_rejection_allowed_statuses: list[str],
     included_check_names: Sequence[str],
     excluded_check_names: Sequence[str],
-    phases: Sequence[str] = DEFAULT_PHASES,
     max_response_time: float | None = None,
+    phases: Sequence[str] = DEFAULT_PHASES,
     exit_first: bool = False,
     max_failures: int | None = None,
     include_path: Sequence[str] = (),
@@ -499,18 +548,20 @@ def run(
     exclude_deprecated: bool = False,
     workers_num: int = DEFAULT_WORKERS,
     base_url: str | None = None,
+    wait_for_schema: float | None = None,
+    rate_limit: str | None = None,
     suppress_health_check: list[HealthCheck] | None = None,
     request_timeout: int | None = None,
     request_tls_verify: bool = True,
     request_cert: str | None = None,
     request_cert_key: str | None = None,
     request_proxy: str | None = None,
-    junit_xml: click.utils.LazyFile | None = None,
-    cassette_path: click.utils.LazyFile | None = None,
-    cassette_format: CassetteFormat = CassetteFormat.VCR,
-    cassette_preserve_exact_body_bytes: bool = False,
-    wait_for_schema: float | None = None,
-    rate_limit: str | None = None,
+    report_formats: list[ReportFormat] | None = None,
+    report_dir: Path = DEFAULT_REPORT_DIRECTORY,
+    report_junit_path: LazyFile | None = None,
+    report_vcr_path: LazyFile | None = None,
+    report_har_path: LazyFile | None = None,
+    report_preserve_bytes: bool = False,
     output_sanitize: bool = True,
     output_truncate: bool = True,
     contrib_openapi_fill_missing_examples: bool = False,
@@ -519,7 +570,7 @@ def run(
     generation_modes: tuple[GenerationMode, ...] = DEFAULT_GENERATOR_MODES,
     generation_seed: int | None = None,
     generation_max_examples: int | None = None,
-    generation_optimize: Sequence[str] | None = None,
+    generation_maximize: Sequence[str] | None = None,
     generation_deterministic: bool | None = None,
     generation_database: str | None = None,
     generation_unique_inputs: bool = False,
@@ -527,6 +578,7 @@ def run(
     generation_graphql_allow_null: bool = True,
     generation_with_security_parameters: bool = True,
     generation_codec: str = "utf-8",
+    generation_no_shrink: bool = False,
     force_color: bool = False,
     no_color: bool = False,
     **__kwargs: Any,
@@ -541,7 +593,7 @@ def run(
 
     validation.validate_schema(schema, base_url)
 
-    _hypothesis_phases = prepare_phases(hypothesis_phases, hypothesis_no_phases)
+    _hypothesis_phases = prepare_phases(hypothesis_phases, hypothesis_no_phases, generation_no_shrink)
     _hypothesis_suppress_health_check = prepare_health_checks(suppress_health_check)
 
     for experiment in experiments:
@@ -591,13 +643,16 @@ def run(
     if exit_first and max_failures is None:
         max_failures = 1
 
-    cassette_config = None
-    if cassette_path is not None:
-        cassette_config = CassetteConfig(
-            path=cassette_path,
-            format=cassette_format,
+    report_config = None
+    if report_formats or report_junit_path or report_vcr_path or report_har_path:
+        report_config = ReportConfig(
+            formats=report_formats,
+            directory=Path(report_dir),
+            junit_path=report_junit_path if report_junit_path else None,
+            vcr_path=report_vcr_path if report_vcr_path else None,
+            har_path=report_har_path if report_har_path else None,
+            preserve_bytes=report_preserve_bytes,
             sanitize_output=output_sanitize,
-            preserve_exact_body_bytes=cassette_preserve_exact_body_bytes,
         )
 
     # Use the same seed for all tests unless `derandomize=True` is used
@@ -616,7 +671,7 @@ def run(
             execution=ExecutionConfig(
                 phases=phases_,
                 checks=selected_checks,
-                targets=TARGETS.get_by_names(generation_optimize or []),
+                targets=TARGETS.get_by_names(generation_maximize or []),
                 hypothesis_settings=prepare_settings(
                     database=generation_database,
                     derandomize=generation_deterministic,
@@ -654,8 +709,7 @@ def run(
         wait_for_schema=wait_for_schema,
         rate_limit=rate_limit,
         output=OutputConfig(sanitize=output_sanitize, truncate=output_truncate),
-        cassette=cassette_config,
-        junit_xml=junit_xml,
+        report=report_config,
         args=ctx.args,
         params=ctx.params,
     )

schemathesis/cli/commands/run/context.py

@@ -1,13 +1,19 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field
-from typing import Generator
+from typing import TYPE_CHECKING, Generator
 
 from schemathesis.core.failures import Failure
 from schemathesis.core.output import OutputConfig
+from schemathesis.core.result import Err, Ok
+from schemathesis.core.transforms import UNRESOLVABLE
 from schemathesis.core.transport import Response
 from schemathesis.engine import Status, events
-from schemathesis.engine.recorder import ScenarioRecorder
+from schemathesis.engine.recorder import CaseNode, ScenarioRecorder
+from schemathesis.generation.case import Case
+
+if TYPE_CHECKING:
+    from schemathesis.generation.stateful.state_machine import ExtractionFailure
 
 
 @dataclass
@@ -17,6 +23,8 @@ class Statistic:
     outcomes: dict[Status, int]
     failures: dict[str, dict[str, GroupedFailures]]
 
+    extraction_failures: set[ExtractionFailure]
+
     tested_operations: set[str]
 
     total_cases: int
@@ -26,6 +34,7 @@ class Statistic:
     __slots__ = (
         "outcomes",
         "failures",
+        "extraction_failures",
         "tested_operations",
         "total_cases",
         "cases_with_failures",
@@ -35,6 +44,7 @@ class Statistic:
     def __init__(self) -> None:
         self.outcomes = {}
         self.failures = {}
+        self.extraction_failures = set()
         self.tested_operations = set()
         self.total_cases = 0
         self.cases_with_failures = 0
@@ -42,10 +52,25 @@ class Statistic:
 
     def record_checks(self, recorder: ScenarioRecorder) -> None:
         """Update statistics and store failures from a new batch of checks."""
+        from schemathesis.generation.stateful.state_machine import ExtractionFailure
+
         failures = self.failures.get(recorder.label, {})
 
         self.total_cases += len(recorder.cases)
 
+        extraction_failures = set()
+
+        def collect_history(node: CaseNode, response: Response) -> list[tuple[Case, Response]]:
+            history = [(node.value, response)]
+            current = node
+            while current.parent_id is not None:
+                current_response = recorder.find_response(case_id=current.parent_id)
+                # We need a response to get there, so it should be present
+                assert current_response is not None
+                current = recorder.cases[current.parent_id]
+                history.append((current.value, current_response))
+            return history
+
         for case_id, case in recorder.cases.items():
             checks = recorder.checks.get(case_id, [])
 
@@ -71,11 +96,56 @@ class Statistic:
                     failures[case_id].failures.append(check.failure_info.failure)
             if has_failures:
                 self.cases_with_failures += 1
+
+            if case.transition is None:
+                continue
+            transition = case.transition
+            parent = recorder.cases[transition.parent_id]
+            response = recorder.find_response(case_id=parent.value.id)
+            # We need a response to get there, so it should be present
+            assert response is not None
+
+            for params in transition.parameters.values():
+                for parameter, extracted in params.items():
+                    if isinstance(extracted.value, Ok) and extracted.value.ok() is UNRESOLVABLE:
+                        history = collect_history(parent, response)
+                        extraction_failures.add(
+                            ExtractionFailure(
+                                id=transition.id,
+                                case_id=case_id,
+                                source=parent.value.operation.label,
+                                target=case.value.operation.label,
+                                parameter_name=parameter,
+                                expression=extracted.definition,
+                                history=history,
+                                response=response,
+                                error=None,
+                            )
+                        )
+                    elif isinstance(extracted.value, Err):
+                        history = collect_history(parent, response)
+                        extraction_failures.add(
+                            ExtractionFailure(
+                                id=transition.id,
+                                case_id=case_id,
+                                source=parent.value.operation.label,
+                                target=case.value.operation.label,
+                                parameter_name=parameter,
+                                expression=extracted.definition,
+                                history=history,
+                                response=response,
+                                error=extracted.value.err(),
+                            )
+                        )
+
         if failures:
             for group in failures.values():
                 group.failures = sorted(set(group.failures))
             self.failures[recorder.label] = failures
 
+        if extraction_failures:
+            self.extraction_failures.update(extraction_failures)
+
 
 @dataclass
 class GroupedFailures:

schemathesis/cli/commands/run/events.py

@@ -16,10 +16,28 @@ class LoadingStarted(events.EngineEvent):
 
 
 class LoadingFinished(events.EngineEvent):
-    __slots__ = ("id", "timestamp", "location", "duration", "base_url", "specification", "statistic")
+    __slots__ = (
+        "id",
+        "timestamp",
+        "location",
+        "duration",
+        "base_url",
+        "base_path",
+        "specification",
+        "statistic",
+        "schema",
+    )
 
     def __init__(
-        self, location: str, start_time: float, base_url: str, specification: Specification, statistic: ApiStatistic
+        self,
+        *,
+        location: str,
+        start_time: float,
+        base_url: str,
+        base_path: str,
+        specification: Specification,
+        statistic: ApiStatistic,
+        schema: dict,
     ) -> None:
         self.id = uuid.uuid4()
         self.timestamp = time.time()
@@ -28,3 +46,5 @@ class LoadingFinished(events.EngineEvent):
         self.base_url = base_url
         self.specification = specification
         self.statistic = statistic
+        self.schema = schema
+        self.base_path = base_path