schemathesis 3.26.2__py3-none-any.whl → 3.27.1__py3-none-any.whl

schemathesis/__init__.py

@@ -3,7 +3,7 @@ from typing import Any
 
 from . import auths, checks, experimental, contrib, fixups, graphql, hooks, runner, serializers, targets  # noqa: E402
 from ._lazy_import import lazy_import
-from .generation import DataGenerationMethod, GenerationConfig  # noqa: E402
+from .generation import DataGenerationMethod, GenerationConfig, HeaderConfig  # noqa: E402
 from .constants import SCHEMATHESIS_VERSION  # noqa: E402
 from .models import Case  # noqa: E402
 from .specs import openapi  # noqa: E402

schemathesis/cli/__init__.py

@@ -1769,6 +1769,6 @@ def after_init_cli_run_handlers(
 def process_call_kwargs(context: HookContext, case: Case, kwargs: dict[str, Any]) -> None:
     """Called before every network call in CLI tests.
 
-    Aims to modify the argument passed to `case.call` / `case.call_wsgi` / `case.call_asgi`.
+    Aims to modify the argument passed to `case.call`.
     Note that you need to modify `kwargs` in-place.
     """

schemathesis/cli/__main__.py

@@ -0,0 +1,4 @@
+if __name__ == "__main__":
+    import schemathesis.cli
+
+    schemathesis.cli.schemathesis()

schemathesis/cli/junitxml.py

@@ -1,5 +1,8 @@
 from __future__ import annotations
+
+import base64
 import platform
+import textwrap
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING
 
@@ -7,12 +10,14 @@ from junit_xml import TestCase, TestSuite, to_xml_report_file
 
 from ..models import Status
 from ..runner import events
-from ..runner.serialization import deduplicate_failures
+from ..runner.serialization import SerializedCheck, SerializedError
+from ..exceptions import prepare_response_payload, RuntimeErrorType
 from .handlers import EventHandler
-
+from .reporting import group_by_case, TEST_CASE_ID_TITLE, split_traceback, get_runtime_error_suggestion
 
 if TYPE_CHECKING:
     from click.utils import LazyFile
+
     from .context import ExecutionContext
 
 
@@ -29,15 +34,79 @@ class JunitXMLHandler(EventHandler):
                 allow_multiple_subelements=True,
             )
             if event.status == Status.failure:
-                checks = deduplicate_failures(event.result.checks)
-                for idx, check in enumerate(checks, 1):
-                    # `check.message` is always not empty for events with `failure` status
-                    test_case.add_failure_info(message=f"{idx}. {check.message}")
-            if event.status == Status.error:
-                test_case.add_error_info(
-                    message=event.result.errors[-1].exception, output=event.result.errors[-1].exception_with_traceback
-                )
+                for idx, (code_sample, group) in enumerate(
+                    group_by_case(event.result.checks, context.code_sample_style), 1
+                ):
+                    checks = sorted(group, key=lambda c: c.name != "not_a_server_error")
+                    test_case.add_failure_info(message=build_failure_message(idx, code_sample, checks))
+            elif event.status == Status.error:
+                test_case.add_error_info(message=build_error_message(context, event.result.errors[-1]))
+            elif event.status == Status.skip:
+                test_case.add_skipped_info(message=event.result.skip_reason)
             self.test_cases.append(test_case)
         if isinstance(event, events.Finished):
             test_suites = [TestSuite("schemathesis", test_cases=self.test_cases, hostname=platform.node())]
             to_xml_report_file(file_descriptor=self.file_handle, test_suites=test_suites, prettyprint=True)
+
+
+def build_failure_message(idx: int, code_sample: str, checks: list[SerializedCheck]) -> str:
+    from ..transports.responses import get_reason
+
+    message = ""
+    for check_idx, check in enumerate(checks):
+        if check_idx == 0:
+            message += f"{idx}. {TEST_CASE_ID_TITLE}: {check.example.id}\n"
+        message += f"\n- {check.title}\n"
+        formatted_message = check.formatted_message
+        if formatted_message:
+            message += f"\n{formatted_message}\n"
+        if check_idx + 1 == len(checks):
+            if check.response is not None:
+                status_code = check.response.status_code
+                reason = get_reason(status_code)
+                message += f"\n[{check.response.status_code}] {reason}:\n"
+                response_body = check.response.body
+                if response_body is not None:
+                    if not response_body:
+                        message += "\n    <EMPTY>\n"
+                    else:
+                        encoding = check.response.encoding or "utf8"
+                        try:
+                            payload = base64.b64decode(response_body).decode(encoding)
+                            payload = prepare_response_payload(payload)
+                            payload = textwrap.indent(f"\n`{payload}`\n", prefix="    ")
+                            message += payload
+                        except UnicodeDecodeError:
+                            message += "\n    <BINARY>\n"
+
+    message += f"\nReproduce with: \n\n    {code_sample}"
+    return message
+
+
+def build_error_message(context: ExecutionContext, error: SerializedError) -> str:
+    message = ""
+    if error.title:
+        if error.type == RuntimeErrorType.SCHEMA_GENERIC:
+            message = "Schema Error\n"
+        else:
+            message = f"{error.title}\n"
+        if error.message:
+            message += f"\n{error.message}\n"
+    elif error.message:
+        message = error.message
+    else:
+        message = error.exception
+    if error.extras:
+        extras = error.extras
+    elif context.show_trace and error.type.has_useful_traceback:
+        extras = split_traceback(error.exception_with_traceback)
+    else:
+        extras = []
+    if extras:
+        message += "\n"
+    for extra in extras:
+        message += f"    {extra}\n"
+    suggestion = get_runtime_error_suggestion(error.type, bold=str)
+    if suggestion is not None:
+        message += f"\nTip: {suggestion}"
+    return message

schemathesis/cli/output/default.py

@@ -7,14 +7,12 @@ import shutil
 import textwrap
 import time
 from importlib import metadata
-from itertools import groupby
 from queue import Queue
 from typing import Any, Generator, cast, TYPE_CHECKING
 
 import click
 
 from ... import service
-from ...code_samples import CodeSampleStyle
 from ...constants import (
     DISCORD_LINK,
     FALSE_VALUES,
@@ -37,10 +35,11 @@ from ...models import Status
 from ...runner import events
 from ...runner.events import InternalErrorType, SchemaErrorType
 from ...runner.probes import ProbeOutcome
-from ...runner.serialization import SerializedCheck, SerializedError, SerializedTestResult, deduplicate_failures
+from ...runner.serialization import SerializedError, SerializedTestResult
 from ...service.models import AnalysisSuccess, UnknownExtension, ErrorState
 from ..context import ExecutionContext, FileReportContext, ServiceReportContext
 from ..handlers import EventHandler
+from ..reporting import group_by_case, TEST_CASE_ID_TITLE, split_traceback, get_runtime_error_suggestion
 
 if TYPE_CHECKING:
     import requests
@@ -256,11 +255,11 @@ def _display_error(context: ExecutionContext, error: SerializedError) -> bool:
     if error.extras:
         extras = error.extras
     elif context.show_trace and error.type.has_useful_traceback:
-        extras = _split_traceback(error.exception_with_traceback)
+        extras = split_traceback(error.exception_with_traceback)
     else:
         extras = []
     _display_extras(extras)
-    suggestion = RUNTIME_ERROR_SUGGESTIONS.get(error.type)
+    suggestion = get_runtime_error_suggestion(error.type)
     _maybe_display_tip(suggestion)
     return display_full_traceback_message(error)
 
@@ -279,9 +278,6 @@ def display_failures(context: ExecutionContext, event: events.Finished) -> None:
         display_failures_for_single_test(context, result)
 
 
-TEST_CASE_ID_TITLE = "Test Case ID"
-
-
 def display_failures_for_single_test(context: ExecutionContext, result: SerializedTestResult) -> None:
     """Display a failure for a single method / path."""
     from ...transports.responses import get_reason
@@ -297,18 +293,9 @@ def display_failures_for_single_test(context: ExecutionContext, result: Serializ
         for check_idx, check in enumerate(checks):
             if check_idx == 0:
                 click.secho(f"{idx}. {TEST_CASE_ID_TITLE}: {check.example.id}", bold=True)
-            if check.context is not None:
-                title = check.context.title
-                if check.context.message:
-                    message = check.context.message
-                else:
-                    message = None
-            else:
-                title = f"Custom check failed: `{check.name}`"
-                message = check.message
-            click.secho(f"\n- {title}", fg="red", bold=True)
+            click.secho(f"\n- {check.title}", fg="red", bold=True)
+            message = check.formatted_message
             if message:
-                message = textwrap.indent(message, prefix="    ")
                 click.secho(f"\n{message}", fg="red")
             if check_idx + 1 == len(checks):
                 if check.response is not None:
@@ -316,9 +303,8 @@ def display_failures_for_single_test(context: ExecutionContext, result: Serializ
                     reason = get_reason(status_code)
                     response = bold(f"[{check.response.status_code}] {reason}")
                     click.echo(f"\n{response}:")
-
                     response_body = check.response.body
-                    if check.response is not None and response_body is not None:
+                    if response_body is not None:
                         if not response_body:
                             click.echo("\n    <EMPTY>")
                         else:
@@ -336,26 +322,6 @@ def display_failures_for_single_test(context: ExecutionContext, result: Serializ
         )
 
 
-def group_by_case(
-    checks: list[SerializedCheck], code_sample_style: CodeSampleStyle
-) -> Generator[tuple[str, Generator[SerializedCheck, None, None]], None, None]:
-    checks = deduplicate_failures(checks)
-    checks = sorted(checks, key=lambda c: _by_unique_code_sample(c, code_sample_style))
-    yield from groupby(checks, lambda c: _by_unique_code_sample(c, code_sample_style))
-
-
-def _by_unique_code_sample(check: SerializedCheck, code_sample_style: CodeSampleStyle) -> str:
-    request_body = base64.b64decode(check.example.body).decode() if check.example.body is not None else None
-    return code_sample_style.generate(
-        method=check.example.method,
-        url=check.example.url,
-        body=request_body,
-        headers=check.example.headers,
-        verify=check.example.verify,
-        extra_headers=check.example.extra_headers,
-    )
-
-
 def display_application_logs(context: ExecutionContext, event: events.Finished) -> None:
     """Print logs captured during the application run."""
     if not event.has_logs:
@@ -436,7 +402,7 @@ def display_analysis(context: ExecutionContext) -> None:
             title = "Network Error"
         else:
             traceback = format_exception(exception, True)
-            extras = _split_traceback(traceback)
+            extras = split_traceback(traceback)
             title = "Internal Error"
             message = f"We apologize for the inconvenience. This appears to be an internal issue.\nPlease, consider reporting the following details to our issue tracker:\n\n  {ISSUE_TRACKER_URL}"
             suggestion = "Please update your CLI to the latest version and try again."
@@ -719,10 +685,6 @@ def should_skip_suggestion(context: ExecutionContext, event: events.InternalErro
     return event.subtype == SchemaErrorType.CONNECTION_OTHER and context.wait_for_schema is not None
 
 
-def _split_traceback(traceback: str) -> list[str]:
-    return [entry for entry in traceback.splitlines() if entry]
-
-
 def _display_extras(extras: list[str]) -> None:
     if extras:
         click.echo()
@@ -743,7 +705,7 @@ def display_internal_error(context: ExecutionContext, event: events.InternalErro
     if event.type == InternalErrorType.SCHEMA:
         extras = event.extras
     elif context.show_trace:
-        extras = _split_traceback(event.exception_with_traceback)
+        extras = split_traceback(event.exception_with_traceback)
     else:
         extras = [event.exception]
     _display_extras(extras)

schemathesis/cli/reporting.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+import base64
+from itertools import groupby
+from typing import Callable, Generator
+
+import click
+
+from ..code_samples import CodeSampleStyle
+from ..exceptions import RuntimeErrorType
+from ..runner.serialization import SerializedCheck, deduplicate_failures
+
+TEST_CASE_ID_TITLE = "Test Case ID"
+
+
+def group_by_case(
+    checks: list[SerializedCheck], code_sample_style: CodeSampleStyle
+) -> Generator[tuple[str, Generator[SerializedCheck, None, None]], None, None]:
+    checks = deduplicate_failures(checks)
+    checks = sorted(checks, key=lambda c: _by_unique_code_sample(c, code_sample_style))
+    yield from groupby(checks, lambda c: _by_unique_code_sample(c, code_sample_style))
+
+
+def _by_unique_code_sample(check: SerializedCheck, code_sample_style: CodeSampleStyle) -> str:
+    request_body = base64.b64decode(check.example.body).decode() if check.example.body is not None else None
+    return code_sample_style.generate(
+        method=check.example.method,
+        url=check.example.url,
+        body=request_body,
+        headers=check.example.headers,
+        verify=check.example.verify,
+        extra_headers=check.example.extra_headers,
+    )
+
+
+def split_traceback(traceback: str) -> list[str]:
+    return [entry for entry in traceback.splitlines() if entry]
+
+
+def bold(option: str) -> str:
+    return click.style(option, bold=True)
+
+
+def get_runtime_error_suggestion(error_type: RuntimeErrorType, bold: Callable[[str], str] = bold) -> str | None:
+    DISABLE_SSL_SUGGESTION = f"Bypass SSL verification with {bold('`--request-tls-verify=false`')}."
+    DISABLE_SCHEMA_VALIDATION_SUGGESTION = (
+        f"Bypass validation using {bold('`--validate-schema=false`')}. Caution: May cause unexpected errors."
+    )
+
+    def _format_health_check_suggestion(label: str) -> str:
+        return f"Bypass this health check using {bold(f'`--hypothesis-suppress-health-check={label}`')}."
+
+    RUNTIME_ERROR_SUGGESTIONS = {
+        RuntimeErrorType.CONNECTION_SSL: DISABLE_SSL_SUGGESTION,
+        RuntimeErrorType.HYPOTHESIS_DEADLINE_EXCEEDED: (
+            f"Adjust the deadline using {bold('`--hypothesis-deadline=MILLIS`')} or "
+            f"disable with {bold('`--hypothesis-deadline=None`')}."
+        ),
+        RuntimeErrorType.HYPOTHESIS_UNSATISFIABLE: "Examine the schema for inconsistencies and consider simplifying it.",
+        RuntimeErrorType.SCHEMA_BODY_IN_GET_REQUEST: DISABLE_SCHEMA_VALIDATION_SUGGESTION,
+        RuntimeErrorType.SCHEMA_INVALID_REGULAR_EXPRESSION: "Ensure your regex is compatible with Python's syntax.\n"
+        "For guidance, visit: https://docs.python.org/3/library/re.html",
+        RuntimeErrorType.HYPOTHESIS_UNSUPPORTED_GRAPHQL_SCALAR: "Define a custom strategy for it.\n"
+        "For guidance, visit: https://schemathesis.readthedocs.io/en/stable/graphql.html#custom-scalars",
+        RuntimeErrorType.HYPOTHESIS_HEALTH_CHECK_DATA_TOO_LARGE: _format_health_check_suggestion("data_too_large"),
+        RuntimeErrorType.HYPOTHESIS_HEALTH_CHECK_FILTER_TOO_MUCH: _format_health_check_suggestion("filter_too_much"),
+        RuntimeErrorType.HYPOTHESIS_HEALTH_CHECK_TOO_SLOW: _format_health_check_suggestion("too_slow"),
+        RuntimeErrorType.HYPOTHESIS_HEALTH_CHECK_LARGE_BASE_EXAMPLE: _format_health_check_suggestion(
+            "large_base_example"
+        ),
+    }
+    return RUNTIME_ERROR_SUGGESTIONS.get(error_type)

schemathesis/generation/__init__.py

@@ -1,8 +1,11 @@
 from __future__ import annotations
 import random
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from enum import Enum
-from typing import Union, Iterable
+from typing import Union, Iterable, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from hypothesis.strategies import SearchStrategy
 
 
 class DataGenerationMethod(str, Enum):
@@ -58,6 +61,13 @@ def generate_random_case_id(length: int = 6) -> str:
     return output
 
 
+@dataclass
+class HeaderConfig:
+    """Configuration for generating headers."""
+
+    strategy: SearchStrategy[str] | None = None
+
+
 @dataclass
 class GenerationConfig:
     """Holds various configuration options relevant for data generation."""
@@ -66,3 +76,5 @@ class GenerationConfig:
     allow_x00: bool = True
     # Generate strings using the given codec
     codec: str | None = "utf-8"
+    # Header generation configuration
+    headers: HeaderConfig = field(default_factory=HeaderConfig)