schemathesis 4.0.0a3__py3-none-any.whl → 4.0.0a5__py3-none-any.whl

schemathesis/cli/commands/run/context.py

@@ -1,21 +1,30 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field
-from typing import Generator
+from typing import TYPE_CHECKING, Generator
 
 from schemathesis.core.failures import Failure
 from schemathesis.core.output import OutputConfig
+from schemathesis.core.result import Err, Ok
+from schemathesis.core.transforms import UNRESOLVABLE
 from schemathesis.core.transport import Response
 from schemathesis.engine import Status, events
-from schemathesis.engine.recorder import ScenarioRecorder
+from schemathesis.engine.recorder import CaseNode, ScenarioRecorder
+from schemathesis.generation.case import Case
+
+if TYPE_CHECKING:
+    from schemathesis.generation.stateful.state_machine import ExtractionFailure
 
 
 @dataclass
 class Statistic:
     """Running statistics about test execution."""
 
-    outcomes: dict[Status, int]
     failures: dict[str, dict[str, GroupedFailures]]
+    # Track first case_id where each unique failure was found
+    unique_failures_map: dict[Failure, str]
+
+    extraction_failures: set[ExtractionFailure]
 
     tested_operations: set[str]
 
@@ -24,8 +33,9 @@ class Statistic:
     cases_without_checks: int
 
     __slots__ = (
-        "outcomes",
         "failures",
+        "unique_failures_map",
+        "extraction_failures",
         "tested_operations",
         "total_cases",
         "cases_with_failures",
@@ -33,19 +43,35 @@ class Statistic:
     )
 
     def __init__(self) -> None:
-        self.outcomes = {}
         self.failures = {}
+        self.unique_failures_map = {}
+        self.extraction_failures = set()
         self.tested_operations = set()
         self.total_cases = 0
         self.cases_with_failures = 0
         self.cases_without_checks = 0
 
-    def record_checks(self, recorder: ScenarioRecorder) -> None:
+    def on_scenario_finished(self, recorder: ScenarioRecorder) -> None:
         """Update statistics and store failures from a new batch of checks."""
+        from schemathesis.generation.stateful.state_machine import ExtractionFailure
+
         failures = self.failures.get(recorder.label, {})
 
         self.total_cases += len(recorder.cases)
 
+        extraction_failures = set()
+
+        def collect_history(node: CaseNode, response: Response) -> list[tuple[Case, Response]]:
+            history = [(node.value, response)]
+            current = node
+            while current.parent_id is not None:
+                current_response = recorder.find_response(case_id=current.parent_id)
+                # We need a response to get there, so it should be present
+                assert current_response is not None
+                current = recorder.cases[current.parent_id]
+                history.append((current.value, current_response))
+            return history
+
         for case_id, case in recorder.cases.items():
             checks = recorder.checks.get(case_id, [])
 
@@ -55,27 +81,84 @@ class Statistic:
 
             self.tested_operations.add(case.value.operation.label)
             has_failures = False
-            for check in checks:
-                response = recorder.interactions[case_id].response
+            current_case_failures = []
+            last_failure_info = None
 
-                # Collect failures
+            for check in checks:
                 if check.failure_info is not None:
-                    has_failures = True
-                    if case_id not in failures:
-                        failures[case_id] = GroupedFailures(
-                            case_id=case_id,
-                            code_sample=check.failure_info.code_sample,
-                            failures=[],
-                            response=response,
-                        )
-                    failures[case_id].failures.append(check.failure_info.failure)
+                    failure = check.failure_info.failure
+
+                    # Check if this is a new unique failure
+                    if failure not in self.unique_failures_map:
+                        last_failure_info = check.failure_info
+                        self.unique_failures_map[failure] = case_id
+                        current_case_failures.append(failure)
+                        has_failures = True
+                    else:
+                        # This failure was already seen - skip it
+                        continue
+
+            if current_case_failures:
+                assert last_failure_info is not None
+                failures[case_id] = GroupedFailures(
+                    case_id=case_id,
+                    code_sample=last_failure_info.code_sample,
+                    failures=current_case_failures,
+                    response=recorder.interactions[case_id].response,
+                )
+
             if has_failures:
                 self.cases_with_failures += 1
+
+            if case.transition is None:
+                continue
+            transition = case.transition
+            parent = recorder.cases[transition.parent_id]
+            response = recorder.find_response(case_id=parent.value.id)
+            # We need a response to get there, so it should be present
+            assert response is not None
+
+            for params in transition.parameters.values():
+                for parameter, extracted in params.items():
+                    if isinstance(extracted.value, Ok) and extracted.value.ok() is UNRESOLVABLE:
+                        history = collect_history(parent, response)
+                        extraction_failures.add(
+                            ExtractionFailure(
+                                id=transition.id,
+                                case_id=case_id,
+                                source=parent.value.operation.label,
+                                target=case.value.operation.label,
+                                parameter_name=parameter,
+                                expression=extracted.definition,
+                                history=history,
+                                response=response,
+                                error=None,
+                            )
+                        )
+                    elif isinstance(extracted.value, Err):
+                        history = collect_history(parent, response)
+                        extraction_failures.add(
+                            ExtractionFailure(
+                                id=transition.id,
+                                case_id=case_id,
+                                source=parent.value.operation.label,
+                                target=case.value.operation.label,
+                                parameter_name=parameter,
+                                expression=extracted.definition,
+                                history=history,
+                                response=response,
+                                error=extracted.value.err(),
+                            )
+                        )
+
         if failures:
             for group in failures.values():
                 group.failures = sorted(set(group.failures))
             self.failures[recorder.label] = failures
 
+        if extraction_failures:
+            self.extraction_failures.update(extraction_failures)
+
 
 @dataclass
 class GroupedFailures:
@@ -108,7 +191,7 @@ class ExecutionContext:
 
     def on_event(self, event: events.EngineEvent) -> None:
         if isinstance(event, events.ScenarioFinished):
-            self.statistic.record_checks(event.recorder)
+            self.statistic.on_scenario_finished(event.recorder)
         elif isinstance(event, events.NonFatalError) or (
             isinstance(event, events.PhaseFinished)
             and event.phase.is_enabled

schemathesis/cli/commands/run/executor.py

@@ -10,10 +10,11 @@ from schemathesis.cli.commands.run.context import ExecutionContext
 from schemathesis.cli.commands.run.events import LoadingFinished, LoadingStarted
 from schemathesis.cli.commands.run.handlers import display_handler_error
 from schemathesis.cli.commands.run.handlers.base import EventHandler
-from schemathesis.cli.commands.run.handlers.cassettes import CassetteConfig, CassetteWriter
+from schemathesis.cli.commands.run.handlers.cassettes import CassetteWriter
 from schemathesis.cli.commands.run.handlers.junitxml import JunitXMLHandler
 from schemathesis.cli.commands.run.handlers.output import OutputHandler
 from schemathesis.cli.commands.run.loaders import AutodetectConfig, load_schema
+from schemathesis.cli.commands.run.reports import ReportConfig, ReportFormat
 from schemathesis.cli.ext.fs import open_file
 from schemathesis.core.errors import LoaderError
 from schemathesis.core.output import OutputConfig
@@ -43,8 +44,7 @@ class RunConfig:
     wait_for_schema: float | None
     rate_limit: str | None
     output: OutputConfig
-    junit_xml: click.utils.LazyFile | None
-    cassette: CassetteConfig | None
+    report: ReportConfig | None
     args: list[str]
     params: dict[str, Any]
 
@@ -93,27 +93,48 @@ def into_event_stream(config: RunConfig) -> EventGenerator:
         yield FatalError(exception=exc)
 
 
-def _execute(event_stream: EventGenerator, config: RunConfig) -> None:
+def initialize_handlers(config: RunConfig) -> list[EventHandler]:
+    """Create event handlers based on run configuration."""
     handlers: list[EventHandler] = []
-    if config.junit_xml is not None:
-        open_file(config.junit_xml)
-        handlers.append(JunitXMLHandler(config.junit_xml))
-    if config.cassette is not None:
-        open_file(config.cassette.path)
-        handlers.append(CassetteWriter(config=config.cassette))
+
+    if config.report is not None:
+        if ReportFormat.JUNIT in config.report.formats:
+            path = config.report.get_path(ReportFormat.JUNIT)
+            open_file(path)
+            handlers.append(JunitXMLHandler(path))
+
+        for format in (ReportFormat.VCR, ReportFormat.HAR):
+            if format in config.report.formats:
+                path = config.report.get_path(format)
+                open_file(path)
+                handlers.append(
+                    CassetteWriter(
+                        format=format,
+                        path=path,
+                        sanitize_output=config.report.sanitize_output,
+                        preserve_bytes=config.report.preserve_bytes,
+                    )
+                )
+
     for custom_handler in CUSTOM_HANDLERS:
         handlers.append(custom_handler(*config.args, **config.params))
+
     handlers.append(
         OutputHandler(
             workers_num=config.engine.execution.workers_num,
             seed=config.engine.execution.seed,
             rate_limit=config.rate_limit,
             wait_for_schema=config.wait_for_schema,
-            cassette_config=config.cassette,
-            junit_xml_file=config.junit_xml.name if config.junit_xml is not None else None,
+            engine_config=config.engine,
+            report_config=config.report,
         )
     )
 
+    return handlers
+
+
+def _execute(event_stream: EventGenerator, config: RunConfig) -> None:
+    handlers = initialize_handlers(config)
     ctx = ExecutionContext(output_config=config.output, seed=config.engine.execution.seed)
 
     def shutdown() -> None:

schemathesis/cli/commands/run/filters.py

@@ -27,6 +27,7 @@ def _with_filter(*, by: str, mode: Literal["include", "exclude"], modifier: Lite
         help=help_text,
         type=str,
         multiple=modifier is None,
+        hidden=True,
     )
 
 

schemathesis/cli/commands/run/handlers/cassettes.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import datetime
-import enum
 import json
 import sys
 import threading
@@ -11,11 +10,12 @@ from queue import Queue
 from typing import IO, Callable, Iterator
 from urllib.parse import parse_qsl, urlparse
 
-import click
 import harfile
+from click.utils import LazyFile
 
 from schemathesis.cli.commands.run.context import ExecutionContext
 from schemathesis.cli.commands.run.handlers.base import EventHandler
+from schemathesis.cli.commands.run.reports import ReportFormat
 from schemathesis.core.output.sanitization import sanitize_url, sanitize_value
 from schemathesis.core.transforms import deepclone
 from schemathesis.core.transport import Response
@@ -28,43 +28,26 @@ from schemathesis.generation.meta import CoveragePhaseData
 WRITER_WORKER_JOIN_TIMEOUT = 1
 
 
-class CassetteFormat(str, enum.Enum):
-    """Type of the cassette."""
-
-    VCR = "vcr"
-    HAR = "har"
-
-    @classmethod
-    def from_str(cls, value: str) -> CassetteFormat:
-        try:
-            return cls[value.upper()]
-        except KeyError:
-            available_formats = ", ".join(cls)
-            raise ValueError(
-                f"Invalid value for cassette format: {value}. Available formats: {available_formats}"
-            ) from None
-
-
-@dataclass
-class CassetteConfig:
-    path: click.utils.LazyFile
-    format: CassetteFormat = CassetteFormat.VCR
-    preserve_exact_body_bytes: bool = False
-    sanitize_output: bool = True
-
-
 @dataclass
 class CassetteWriter(EventHandler):
     """Write network interactions to a cassette."""
 
-    config: CassetteConfig
+    format: ReportFormat
+    path: LazyFile
+    sanitize_output: bool = True
+    preserve_bytes: bool = False
     queue: Queue = field(default_factory=Queue)
     worker: threading.Thread = field(init=False)
 
     def __post_init__(self) -> None:
-        kwargs = {"config": self.config, "queue": self.queue}
+        kwargs = {
+            "path": self.path,
+            "sanitize_output": self.sanitize_output,
+            "preserve_bytes": self.preserve_bytes,
+            "queue": self.queue,
+        }
         writer: Callable
-        if self.config.format == CassetteFormat.HAR:
+        if self.format == ReportFormat.HAR:
             writer = har_writer
         else:
             writer = vcr_writer
@@ -120,7 +103,7 @@ def get_command_representation() -> str:
     return f"st {args}"
 
 
-def vcr_writer(config: CassetteConfig, queue: Queue) -> None:
+def vcr_writer(path: LazyFile, sanitize_output: bool, preserve_bytes: bool, queue: Queue) -> None:
     """Write YAML to a file in an incremental manner.
 
     This implementation doesn't use `pyyaml` package and composes YAML manually as string due to the following reasons:
@@ -131,12 +114,12 @@ def vcr_writer(config: CassetteConfig, queue: Queue) -> None:
         providing tags, anchors to have incremental writing, with primitive types it is much simpler.
     """
     current_id = 1
-    stream = config.path.open()
+    stream = path.open()
 
     def format_header_values(values: list[str]) -> str:
         return "\n".join(f"      - {json.dumps(v)}" for v in values)
 
-    if config.sanitize_output:
+    if sanitize_output:
 
         def format_headers(headers: dict[str, list[str]]) -> str:
             headers = deepclone(headers)
@@ -162,7 +145,7 @@ def vcr_writer(config: CassetteConfig, queue: Queue) -> None:
   checks:
 {items}"""
 
-    if config.preserve_exact_body_bytes:
+    if preserve_bytes:
 
         def format_request_body(output: IO, request: Request) -> None:
             if request.encoded_body is not None:
@@ -283,7 +266,7 @@ http_interactions:"""
                 else:
                     stream.write("null")
 
-                if config.sanitize_output:
+                if sanitize_output:
                     uri = sanitize_url(interaction.request.uri)
                 else:
                     uri = interaction.request.uri
@@ -321,7 +304,7 @@ http_interactions:"""
                 current_id += 1
         else:
             break
-    config.path.close()
+    path.close()
 
 
 def write_double_quoted(stream: IO, text: str | None) -> None:
@@ -367,13 +350,13 @@ def write_double_quoted(stream: IO, text: str | None) -> None:
     stream.write('"')
 
 
-def har_writer(config: CassetteConfig, queue: Queue) -> None:
-    with harfile.open(config.path) as har:
+def har_writer(path: LazyFile, sanitize_output: bool, preserve_bytes: bool, queue: Queue) -> None:
+    with harfile.open(path) as har:
         while True:
             item = queue.get()
             if isinstance(item, Process):
                 for interaction in item.recorder.interactions.values():
-                    if config.sanitize_output:
+                    if sanitize_output:
                         uri = sanitize_url(interaction.request.uri)
                     else:
                         uri = interaction.request.uri
@@ -382,7 +365,7 @@ def har_writer(config: CassetteConfig, queue: Queue) -> None:
                         post_data = harfile.PostData(
                             mimeType=interaction.request.headers.get("Content-Type", [""])[0],
                             text=interaction.request.encoded_body
-                            if config.preserve_exact_body_bytes
+                            if preserve_bytes
                             else interaction.request.body.decode("utf-8", "replace"),
                         )
                     else:
@@ -393,16 +376,14 @@ def har_writer(config: CassetteConfig, queue: Queue) -> None:
                             size=interaction.response.body_size or 0,
                             mimeType=content_type,
                             text=interaction.response.encoded_body
-                            if config.preserve_exact_body_bytes
+                            if preserve_bytes
                             else interaction.response.content.decode("utf-8", "replace")
                             if interaction.response.content is not None
                             else None,
-                            encoding="base64"
-                            if interaction.response.content is not None and config.preserve_exact_body_bytes
-                            else None,
+                            encoding="base64" if interaction.response.content is not None and preserve_bytes else None,
                         )
                         http_version = f"HTTP/{interaction.response.http_version}"
-                        if config.sanitize_output:
+                        if sanitize_output:
                             headers = deepclone(interaction.response.headers)
                             sanitize_value(headers)
                         else:
@@ -424,7 +405,7 @@ def har_writer(config: CassetteConfig, queue: Queue) -> None:
                         time = 0
                         http_version = ""
 
-                    if config.sanitize_output:
+                    if sanitize_output:
                         headers = deepclone(interaction.request.headers)
                         sanitize_value(headers)
                     else:

schemathesis/cli/commands/run/handlers/junitxml.py

@@ -25,7 +25,7 @@ class JunitXMLHandler(EventHandler):
             test_case.elapsed_sec += event.elapsed_time
             if event.status == Status.FAILURE:
                 add_failure(test_case, ctx.statistic.failures[label].values(), ctx)
-            elif event.status == Status.SKIP:
+            elif event.status == Status.SKIP and event.skip_reason is not None:
                 test_case.add_skipped_info(output=event.skip_reason)
         elif isinstance(event, events.NonFatalError):
             test_case = self.get_or_create_test_case(event.label)