schemathesis 4.0.0a3__py3-none-any.whl → 4.0.0a4__py3-none-any.whl

schemathesis/cli/commands/run/executor.py

@@ -10,10 +10,11 @@ from schemathesis.cli.commands.run.context import ExecutionContext
 from schemathesis.cli.commands.run.events import LoadingFinished, LoadingStarted
 from schemathesis.cli.commands.run.handlers import display_handler_error
 from schemathesis.cli.commands.run.handlers.base import EventHandler
-from schemathesis.cli.commands.run.handlers.cassettes import CassetteConfig, CassetteWriter
+from schemathesis.cli.commands.run.handlers.cassettes import CassetteWriter
 from schemathesis.cli.commands.run.handlers.junitxml import JunitXMLHandler
 from schemathesis.cli.commands.run.handlers.output import OutputHandler
 from schemathesis.cli.commands.run.loaders import AutodetectConfig, load_schema
+from schemathesis.cli.commands.run.reports import ReportConfig, ReportFormat
 from schemathesis.cli.ext.fs import open_file
 from schemathesis.core.errors import LoaderError
 from schemathesis.core.output import OutputConfig
@@ -43,8 +44,7 @@ class RunConfig:
     wait_for_schema: float | None
     rate_limit: str | None
     output: OutputConfig
-    junit_xml: click.utils.LazyFile | None
-    cassette: CassetteConfig | None
+    report: ReportConfig | None
     args: list[str]
     params: dict[str, Any]
 
@@ -93,27 +93,47 @@ def into_event_stream(config: RunConfig) -> EventGenerator:
         yield FatalError(exception=exc)
 
 
-def _execute(event_stream: EventGenerator, config: RunConfig) -> None:
+def initialize_handlers(config: RunConfig) -> list[EventHandler]:
+    """Create event handlers based on run configuration."""
     handlers: list[EventHandler] = []
-    if config.junit_xml is not None:
-        open_file(config.junit_xml)
-        handlers.append(JunitXMLHandler(config.junit_xml))
-    if config.cassette is not None:
-        open_file(config.cassette.path)
-        handlers.append(CassetteWriter(config=config.cassette))
+
+    if config.report is not None:
+        if ReportFormat.JUNIT in config.report.formats:
+            path = config.report.get_path(ReportFormat.JUNIT)
+            open_file(path)
+            handlers.append(JunitXMLHandler(path))
+
+        for format in (ReportFormat.VCR, ReportFormat.HAR):
+            if format in config.report.formats:
+                path = config.report.get_path(format)
+                open_file(path)
+                handlers.append(
+                    CassetteWriter(
+                        format=format,
+                        path=path,
+                        sanitize_output=config.report.sanitize_output,
+                        preserve_bytes=config.report.preserve_bytes,
+                    )
+                )
+
     for custom_handler in CUSTOM_HANDLERS:
         handlers.append(custom_handler(*config.args, **config.params))
+
     handlers.append(
         OutputHandler(
             workers_num=config.engine.execution.workers_num,
             seed=config.engine.execution.seed,
             rate_limit=config.rate_limit,
             wait_for_schema=config.wait_for_schema,
-            cassette_config=config.cassette,
-            junit_xml_file=config.junit_xml.name if config.junit_xml is not None else None,
+            report_config=config.report,
         )
     )
 
+    return handlers
+
+
+def _execute(event_stream: EventGenerator, config: RunConfig) -> None:
+    handlers = initialize_handlers(config)
     ctx = ExecutionContext(output_config=config.output, seed=config.engine.execution.seed)
 
     def shutdown() -> None:

schemathesis/cli/commands/run/filters.py

@@ -27,6 +27,7 @@ def _with_filter(*, by: str, mode: Literal["include", "exclude"], modifier: Lite
         help=help_text,
         type=str,
         multiple=modifier is None,
+        hidden=True,
     )
 
 

schemathesis/cli/commands/run/handlers/cassettes.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import datetime
-import enum
 import json
 import sys
 import threading
@@ -11,11 +10,12 @@ from queue import Queue
 from typing import IO, Callable, Iterator
 from urllib.parse import parse_qsl, urlparse
 
-import click
 import harfile
+from click.utils import LazyFile
 
 from schemathesis.cli.commands.run.context import ExecutionContext
 from schemathesis.cli.commands.run.handlers.base import EventHandler
+from schemathesis.cli.commands.run.reports import ReportFormat
 from schemathesis.core.output.sanitization import sanitize_url, sanitize_value
 from schemathesis.core.transforms import deepclone
 from schemathesis.core.transport import Response
@@ -28,43 +28,26 @@ from schemathesis.generation.meta import CoveragePhaseData
 WRITER_WORKER_JOIN_TIMEOUT = 1
 
 
-class CassetteFormat(str, enum.Enum):
-    """Type of the cassette."""
-
-    VCR = "vcr"
-    HAR = "har"
-
-    @classmethod
-    def from_str(cls, value: str) -> CassetteFormat:
-        try:
-            return cls[value.upper()]
-        except KeyError:
-            available_formats = ", ".join(cls)
-            raise ValueError(
-                f"Invalid value for cassette format: {value}. Available formats: {available_formats}"
-            ) from None
-
-
-@dataclass
-class CassetteConfig:
-    path: click.utils.LazyFile
-    format: CassetteFormat = CassetteFormat.VCR
-    preserve_exact_body_bytes: bool = False
-    sanitize_output: bool = True
-
-
 @dataclass
 class CassetteWriter(EventHandler):
     """Write network interactions to a cassette."""
 
-    config: CassetteConfig
+    format: ReportFormat
+    path: LazyFile
+    sanitize_output: bool = True
+    preserve_bytes: bool = False
     queue: Queue = field(default_factory=Queue)
     worker: threading.Thread = field(init=False)
 
     def __post_init__(self) -> None:
-        kwargs = {"config": self.config, "queue": self.queue}
+        kwargs = {
+            "path": self.path,
+            "sanitize_output": self.sanitize_output,
+            "preserve_bytes": self.preserve_bytes,
+            "queue": self.queue,
+        }
         writer: Callable
-        if self.config.format == CassetteFormat.HAR:
+        if self.format == ReportFormat.HAR:
             writer = har_writer
         else:
             writer = vcr_writer
@@ -120,7 +103,7 @@ def get_command_representation() -> str:
     return f"st {args}"
 
 
-def vcr_writer(config: CassetteConfig, queue: Queue) -> None:
+def vcr_writer(path: LazyFile, sanitize_output: bool, preserve_bytes: bool, queue: Queue) -> None:
     """Write YAML to a file in an incremental manner.
 
     This implementation doesn't use `pyyaml` package and composes YAML manually as string due to the following reasons:
@@ -131,12 +114,12 @@ def vcr_writer(config: CassetteConfig, queue: Queue) -> None:
         providing tags, anchors to have incremental writing, with primitive types it is much simpler.
     """
     current_id = 1
-    stream = config.path.open()
+    stream = path.open()
 
     def format_header_values(values: list[str]) -> str:
         return "\n".join(f"      - {json.dumps(v)}" for v in values)
 
-    if config.sanitize_output:
+    if sanitize_output:
 
         def format_headers(headers: dict[str, list[str]]) -> str:
             headers = deepclone(headers)
@@ -162,7 +145,7 @@ def vcr_writer(config: CassetteConfig, queue: Queue) -> None:
   checks:
 {items}"""
 
-    if config.preserve_exact_body_bytes:
+    if preserve_bytes:
 
         def format_request_body(output: IO, request: Request) -> None:
             if request.encoded_body is not None:
@@ -283,7 +266,7 @@ http_interactions:"""
                 else:
                     stream.write("null")
 
-                if config.sanitize_output:
+                if sanitize_output:
                     uri = sanitize_url(interaction.request.uri)
                 else:
                     uri = interaction.request.uri
@@ -321,7 +304,7 @@ http_interactions:"""
                 current_id += 1
         else:
             break
-    config.path.close()
+    path.close()
 
 
 def write_double_quoted(stream: IO, text: str | None) -> None:
@@ -367,13 +350,13 @@ def write_double_quoted(stream: IO, text: str | None) -> None:
     stream.write('"')
 
 
-def har_writer(config: CassetteConfig, queue: Queue) -> None:
-    with harfile.open(config.path) as har:
+def har_writer(path: LazyFile, sanitize_output: bool, preserve_bytes: bool, queue: Queue) -> None:
+    with harfile.open(path) as har:
         while True:
             item = queue.get()
             if isinstance(item, Process):
                 for interaction in item.recorder.interactions.values():
-                    if config.sanitize_output:
+                    if sanitize_output:
                         uri = sanitize_url(interaction.request.uri)
                     else:
                         uri = interaction.request.uri
@@ -382,7 +365,7 @@ def har_writer(config: CassetteConfig, queue: Queue) -> None:
                         post_data = harfile.PostData(
                             mimeType=interaction.request.headers.get("Content-Type", [""])[0],
                             text=interaction.request.encoded_body
-                            if config.preserve_exact_body_bytes
+                            if preserve_bytes
                             else interaction.request.body.decode("utf-8", "replace"),
                         )
                     else:
@@ -393,16 +376,14 @@ def har_writer(config: CassetteConfig, queue: Queue) -> None:
                             size=interaction.response.body_size or 0,
                             mimeType=content_type,
                             text=interaction.response.encoded_body
-                            if config.preserve_exact_body_bytes
+                            if preserve_bytes
                             else interaction.response.content.decode("utf-8", "replace")
                             if interaction.response.content is not None
                             else None,
-                            encoding="base64"
-                            if interaction.response.content is not None and config.preserve_exact_body_bytes
-                            else None,
+                            encoding="base64" if interaction.response.content is not None and preserve_bytes else None,
                         )
                         http_version = f"HTTP/{interaction.response.http_version}"
-                        if config.sanitize_output:
+                        if sanitize_output:
                             headers = deepclone(interaction.response.headers)
                             sanitize_value(headers)
                         else:
@@ -424,7 +405,7 @@ def har_writer(config: CassetteConfig, queue: Queue) -> None:
                         time = 0
                         http_version = ""
 
-                    if config.sanitize_output:
+                    if sanitize_output:
                         headers = deepclone(interaction.request.headers)
                         sanitize_value(headers)
                     else:

schemathesis/cli/commands/run/handlers/junitxml.py

@@ -25,7 +25,7 @@ class JunitXMLHandler(EventHandler):
             test_case.elapsed_sec += event.elapsed_time
             if event.status == Status.FAILURE:
                 add_failure(test_case, ctx.statistic.failures[label].values(), ctx)
-            elif event.status == Status.SKIP:
+            elif event.status == Status.SKIP and event.skip_reason is not None:
                 test_case.add_skipped_info(output=event.skip_reason)
         elif isinstance(event, events.NonFatalError):
             test_case = self.get_or_create_test_case(event.label)

schemathesis/cli/commands/run/handlers/output.py

@@ -1,8 +1,10 @@
 from __future__ import annotations
 
 import os
+import textwrap
 import time
 from dataclasses import dataclass, field
+from json.decoder import JSONDecodeError
 from types import GeneratorType
 from typing import TYPE_CHECKING, Any, Generator, Iterable
 
@@ -11,11 +13,12 @@ import click
 from schemathesis.cli.commands.run.context import ExecutionContext, GroupedFailures
 from schemathesis.cli.commands.run.events import LoadingFinished, LoadingStarted
 from schemathesis.cli.commands.run.handlers.base import EventHandler
-from schemathesis.cli.commands.run.handlers.cassettes import CassetteConfig
+from schemathesis.cli.commands.run.reports import ReportConfig, ReportFormat
 from schemathesis.cli.constants import ISSUE_TRACKER_URL
 from schemathesis.cli.core import get_terminal_width
 from schemathesis.core.errors import LoaderError, LoaderErrorKind, format_exception, split_traceback
 from schemathesis.core.failures import MessageBlock, Severity, format_failures
+from schemathesis.core.output import prepare_response_payload
 from schemathesis.core.result import Err, Ok
 from schemathesis.core.version import SCHEMATHESIS_VERSION
 from schemathesis.engine import Status, events
@@ -32,6 +35,8 @@ if TYPE_CHECKING:
     from rich.progress import Progress, TaskID
     from rich.text import Text
 
+    from schemathesis.generation.stateful.state_machine import ExtractionFailure
+
 IO_ENCODING = os.getenv("PYTHONIOENCODING", "utf-8")
 DISCORD_LINK = "https://discord.gg/R9ASRAmHnA"
 
@@ -100,7 +105,7 @@ def display_failures_for_single_test(ctx: ExecutionContext, label: str, checks:
 
 
 VERIFY_URL_SUGGESTION = "Verify that the URL points directly to the Open API schema or GraphQL endpoint"
-DISABLE_SSL_SUGGESTION = f"Bypass SSL verification with {bold('`--request-tls-verify=false`')}."
+DISABLE_SSL_SUGGESTION = f"Bypass SSL verification with {bold('`--tls-verify=false`')}."
 LOADER_ERROR_SUGGESTIONS = {
     # SSL-specific connection issue
     LoaderErrorKind.CONNECTION_SSL: DISABLE_SSL_SUGGESTION,
@@ -379,6 +384,7 @@ class UnitTestProgressManager:
 
     def __init__(
         self,
+        *,
         console: Console,
         title: str,
         total: int,
@@ -437,6 +443,7 @@ class UnitTestProgressManager:
             Status.FAILURE: 0,
             Status.SKIP: 0,
             Status.ERROR: 0,
+            Status.INTERRUPTED: 0,
         }
         self._update_stats_display()
 
@@ -453,8 +460,8 @@ class UnitTestProgressManager:
             parts.append(f"❌ {self.stats[Status.FAILURE]:{width}d} failed")
         if self.stats[Status.ERROR]:
             parts.append(f"🚫 {self.stats[Status.ERROR]:{width}d} errors")
-        if self.stats[Status.SKIP]:
-            parts.append(f"⏭️ {self.stats[Status.SKIP]:{width}d} skipped")
+        if self.stats[Status.SKIP] or self.stats[Status.INTERRUPTED]:
+            parts.append(f"⏭️ {self.stats[Status.SKIP] + self.stats[Status.INTERRUPTED]:{width}d} skipped")
         return "  ".join(parts)
 
     def _update_stats_display(self) -> None:
@@ -789,8 +796,7 @@ class OutputHandler(EventHandler):
 
     statistic: ApiStatistic | None = None
     skip_reasons: list[str] = field(default_factory=list)
-    cassette_config: CassetteConfig | None = None
-    junit_xml_file: str | None = None
+    report_config: ReportConfig | None = None
     warnings: WarningData = field(default_factory=WarningData)
     errors: list[events.NonFatalError] = field(default_factory=list)
     phases: dict[PhaseName, tuple[Status, PhaseSkipReason | None]] = field(
@@ -865,7 +871,8 @@ class OutputHandler(EventHandler):
 
         table.add_row("Base URL:", event.base_url)
         table.add_row("Specification:", event.specification.name)
-        table.add_row("Operations:", str(event.statistic.operations.total))
+        statistic = event.statistic.operations
+        table.add_row("Operations:", f"{statistic.selected} selected / {statistic.total} total")
 
         message = Padding(table, BLOCK_PADDING)
         self.console.print(message)
@@ -892,7 +899,7 @@ class OutputHandler(EventHandler):
         self.unit_tests_manager = UnitTestProgressManager(
             console=self.console,
             title="Unit tests",
-            total=self.statistic.operations.total,
+            total=self.statistic.operations.selected,
         )
         self.unit_tests_manager.start()
 
@@ -1165,6 +1172,54 @@ class OutputHandler(EventHandler):
         )
         click.echo()
 
+    def display_stateful_failures(self, ctx: ExecutionContext) -> None:
+        display_section_name("Stateful tests")
+
+        click.echo("\nFailed to extract data from response:")
+
+        grouped: dict[str, list[ExtractionFailure]] = {}
+        for failure in ctx.statistic.extraction_failures:
+            grouped.setdefault(failure.id, []).append(failure)
+
+        for idx, (transition_id, failures) in enumerate(grouped.items(), 1):
+            for failure in failures:
+                click.echo(f"\n    {idx}. Test Case ID: {failure.case_id}\n")
+                click.echo(f"    {transition_id}")
+
+                indent = "        "
+                if failure.error:
+                    if isinstance(failure.error, JSONDecodeError):
+                        click.echo(f"\n{indent}Failed to parse JSON from response")
+                    else:
+                        click.echo(f"\n{indent}{failure.error.__class__.__name__}: {failure.error}")
+                else:
+                    description = (
+                        f"\n{indent}Could not resolve parameter `{failure.parameter_name}` via `{failure.expression}`"
+                    )
+                    prefix = "$response.body"
+                    if failure.expression.startswith(prefix):
+                        description += f"\n{indent}Path `{failure.expression[len(prefix) :]}` not found in response"
+                    click.echo(description)
+
+                click.echo()
+
+                for case, response in reversed(failure.history):
+                    curl = case.as_curl_command(headers=dict(response.request.headers), verify=response.verify)
+                    click.echo(f"{indent}[{response.status_code}] {curl}")
+
+                response = failure.response
+
+                if response.content is None or not response.content:
+                    click.echo(f"\n{indent}<EMPTY>")
+                else:
+                    try:
+                        payload = prepare_response_payload(response.text, config=ctx.output_config)
+                        click.echo(textwrap.indent(f"\n{payload}", prefix=indent))
+                    except UnicodeDecodeError:
+                        click.echo(f"\n{indent}<BINARY>")
+
+        click.echo()
+
     def display_api_operations(self, ctx: ExecutionContext) -> None:
         assert self.statistic is not None
         click.echo(_style("API Operations:", bold=True))
@@ -1313,14 +1368,13 @@ class OutputHandler(EventHandler):
         display_section_name(message, fg=color)
 
     def display_reports(self) -> None:
-        reports = []
-        if self.cassette_config is not None:
-            format_name = self.cassette_config.format.name.upper()
-            reports.append((format_name, self.cassette_config.path.name))
-        if self.junit_xml_file is not None:
-            reports.append(("JUnit XML", self.junit_xml_file))
-
-        if reports:
+        if self.report_config is not None:
+            reports = [
+                (format.value.upper(), self.report_config.get_path(format).name)
+                for format in ReportFormat
+                if format in self.report_config.formats
+            ]
+
             click.echo(_style("Reports:", bold=True))
             for report_type, path in reports:
                 click.echo(_style(f"  - {report_type}: {path}"))
@@ -1356,6 +1410,8 @@ class OutputHandler(EventHandler):
             self.display_warnings()
         if GLOBAL_EXPERIMENTS.enabled:
             self.display_experiments()
+        if ctx.statistic.extraction_failures:
+            self.display_stateful_failures(ctx)
         display_section_name("SUMMARY")
         click.echo()
 

schemathesis/cli/commands/run/hypothesis.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from enum import IntEnum, unique
+from enum import Enum, unique
 from typing import TYPE_CHECKING, Any
 
 import click
@@ -16,12 +16,11 @@ HYPOTHESIS_IN_MEMORY_DATABASE_IDENTIFIER = ":memory:"
 
 
 @unique
-class Phase(IntEnum):
-    explicit = 0  #: controls whether explicit examples are run.
-    reuse = 1  #: controls whether previous examples will be reused.
-    generate = 2  #: controls whether new examples will be generated.
-    target = 3  #: controls whether examples will be mutated for targeting.
-    shrink = 4  #: controls whether examples will be shrunk.
+class Phase(str, Enum):
+    explicit = "explicit"  #: controls whether explicit examples are run.
+    reuse = "reuse"  #: controls whether previous examples will be reused.
+    generate = "generate"  #: controls whether new examples will be generated.
+    target = "target"  #: controls whether examples will be mutated for targeting.
     # The `explain` phase is not supported
 
     def as_hypothesis(self) -> hypothesis.Phase:
@@ -30,20 +29,23 @@ class Phase(IntEnum):
         return Phase[self.name]
 
     @staticmethod
-    def filter_from_all(variants: list[Phase]) -> list[hypothesis.Phase]:
+    def filter_from_all(variants: list[Phase], no_shrink: bool) -> list[hypothesis.Phase]:
         from hypothesis import Phase
 
-        return list(set(Phase) - {Phase.explain} - set(variants))
+        phases = set(Phase) - {Phase.explain} - set(variants)
+        if no_shrink:
+            return list(phases - {Phase.shrink})
+        return list(phases)
 
 
 @unique
-class HealthCheck(IntEnum):
+class HealthCheck(str, Enum):
     # We remove not relevant checks
-    data_too_large = 1
-    filter_too_much = 2
-    too_slow = 3
-    large_base_example = 7
-    all = 8
+    data_too_large = "data_too_large"
+    filter_too_much = "filter_too_much"
+    too_slow = "too_slow"
+    large_base_example = "large_base_example"
+    all = "all"
 
     def as_hypothesis(self) -> list[hypothesis.HealthCheck]:
         from hypothesis import HealthCheck
@@ -64,14 +66,23 @@ def prepare_health_checks(
 
 
 def prepare_phases(
-    hypothesis_phases: list[Phase] | None, hypothesis_no_phases: list[Phase] | None
+    hypothesis_phases: list[Phase] | None,
+    hypothesis_no_phases: list[Phase] | None,
+    no_shrink: bool = False,
 ) -> list[hypothesis.Phase] | None:
+    from hypothesis import Phase as HypothesisPhase
+
     if hypothesis_phases is not None and hypothesis_no_phases is not None:
         raise click.UsageError(PHASES_INVALID_USAGE_MESSAGE)
     if hypothesis_phases:
-        return [phase.as_hypothesis() for phase in hypothesis_phases]
-    if hypothesis_no_phases:
-        return Phase.filter_from_all(hypothesis_no_phases)
+        phases = [phase.as_hypothesis() for phase in hypothesis_phases]
+        if not no_shrink:
+            phases.append(HypothesisPhase.shrink)
+        return phases
+    elif hypothesis_no_phases:
+        return Phase.filter_from_all(hypothesis_no_phases, no_shrink)
+    elif no_shrink:
+        return Phase.filter_from_all([], no_shrink)
     return None
 
 

schemathesis/cli/commands/run/reports.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+from enum import Enum
+from pathlib import Path
+
+from click.utils import LazyFile
+
+DEFAULT_REPORT_DIRECTORY = Path("./schemathesis-report")
+
+
+class ReportFormat(str, Enum):
+    """Available report formats."""
+
+    JUNIT = "junit"
+    VCR = "vcr"
+    HAR = "har"
+
+    @property
+    def extension(self) -> str:
+        """File extension for this format."""
+        return {
+            self.JUNIT: "xml",
+            self.VCR: "yaml",
+            self.HAR: "json",
+        }[self]
+
+
+class ReportConfig:
+    """Configuration for test report generation."""
+
+    __slots__ = (
+        "formats",
+        "directory",
+        "junit_path",
+        "vcr_path",
+        "har_path",
+        "preserve_bytes",
+        "sanitize_output",
+    )
+
+    def __init__(
+        self,
+        formats: list[ReportFormat] | None = None,
+        directory: Path = DEFAULT_REPORT_DIRECTORY,
+        *,
+        junit_path: LazyFile | None = None,
+        vcr_path: LazyFile | None = None,
+        har_path: LazyFile | None = None,
+        preserve_bytes: bool = False,
+        sanitize_output: bool = True,
+    ) -> None:
+        self.formats = formats or []
+        # Auto-enable formats when paths are specified
+        if junit_path and ReportFormat.JUNIT not in self.formats:
+            self.formats.append(ReportFormat.JUNIT)
+        if vcr_path and ReportFormat.VCR not in self.formats:
+            self.formats.append(ReportFormat.VCR)
+        if har_path and ReportFormat.HAR not in self.formats:
+            self.formats.append(ReportFormat.HAR)
+        self.directory = directory
+        self.junit_path = junit_path
+        self.vcr_path = vcr_path
+        self.har_path = har_path
+        self.preserve_bytes = preserve_bytes
+        self.sanitize_output = sanitize_output
+
+    def get_path(self, format: ReportFormat) -> LazyFile:
+        """Get the final path for a specific format."""
+        custom_path = getattr(self, f"{format.value}_path")
+        if custom_path is not None:
+            return custom_path
+        return LazyFile(self.directory / f"{format.value}.{format.extension}", mode="w", encoding="utf-8")