schemathesis 3.19.7__py3-none-any.whl → 3.20.1__py3-none-any.whl

schemathesis/cli/callbacks.py

@@ -10,17 +10,37 @@ import hypothesis
 from click.types import LazyFile  # type: ignore
 from requests import PreparedRequest, RequestException
 
-from .. import exceptions, throttling, utils
-from ..constants import CodeSampleStyle, DataGenerationMethod
+from .. import exceptions, experimental, throttling, utils
+from ..code_samples import CodeSampleStyle
+from ..constants import DataGenerationMethod
 from ..service.hosts import get_temporary_hosts_file
 from ..stateful import Stateful
 from ..types import PathLike
 from .constants import DEFAULT_WORKERS
 
+INVALID_DERANDOMIZE_MESSAGE = (
+    "`--hypothesis-derandomize` implies no database, so passing `--hypothesis-database` too is invalid."
+)
 MISSING_CASSETTE_PATH_ARGUMENT_MESSAGE = (
     'Missing argument, "--cassette-path" should be specified as well if you use "--cassette-preserve-exact-body-bytes".'
 )
 INVALID_SCHEMA_MESSAGE = "Invalid SCHEMA, must be a valid URL, file path or an API name from Schemathesis.io."
+FILE_DOES_NOT_EXIST_MESSAGE = "The specified file does not exist. Please provide a valid path to an existing file."
+INVALID_BASE_URL_MESSAGE = (
+    "The provided base URL is invalid. This URL serves as a prefix for all API endpoints you want to test. "
+    "Make sure it is a properly formatted URL."
+)
+MISSING_BASE_URL_MESSAGE = "The `--base-url` option is required when specifying a schema via a file."
+APPLICATION_FORMAT_MESSAGE = """Unable to import application from the provided module.
+The `--app` option should follow this format:
+
+    module_path:variable_name
+
+- `module_path`: A path to an importable Python module.
+- `variable_name`: The name of the application variable within that module.
+
+Example: `st run --app=your_module:app ...`"""
+MISSING_REQUEST_CERT_MESSAGE = "The `--request-cert` option must be specified if `--request-cert-key` is used."
 
 
 @enum.unique
@@ -47,7 +67,7 @@ def parse_schema_kind(schema: str, app: Optional[str]) -> SchemaInputKind:
         raise click.UsageError(INVALID_SCHEMA_MESSAGE)
     if netloc:
         return SchemaInputKind.URL
-    if utils.file_exists(schema):
+    if utils.file_exists(schema) or utils.is_filename(schema):
         return SchemaInputKind.PATH
     if app is not None:
         return SchemaInputKind.APP_PATH
@@ -69,7 +89,11 @@ def validate_schema(
     if kind == SchemaInputKind.PATH:
         # Base URL is required if it is not a dry run
         if app is None and base_url is None and not dry_run:
-            raise click.UsageError('Missing argument, "--base-url" is required for SCHEMA specified by file.')
+            if not utils.file_exists(schema):
+                message = FILE_DOES_NOT_EXIST_MESSAGE
+            else:
+                message = MISSING_BASE_URL_MESSAGE
+            raise click.UsageError(message)
     if kind == SchemaInputKind.NAME:
         if api_name is not None:
             raise click.UsageError(f"Got unexpected extra argument ({api_name})")
@@ -86,9 +110,9 @@ def validate_base_url(ctx: click.core.Context, param: click.core.Parameter, raw_
     try:
         netloc = urlparse(raw_value).netloc
     except ValueError as exc:
-        raise click.UsageError("Invalid base URL") from exc
+        raise click.UsageError(INVALID_BASE_URL_MESSAGE) from exc
     if raw_value and not netloc:
-        raise click.UsageError("Invalid base URL")
+        raise click.UsageError(INVALID_BASE_URL_MESSAGE)
     return raw_value
 
 
@@ -104,14 +128,6 @@ def validate_rate_limit(
         raise click.UsageError(exc.args[0]) from exc
 
 
-APPLICATION_FORMAT_MESSAGE = (
-    "Can not import application from the given module!\n"
-    "The `--app` option value should be in format:\n\n    path:variable\n\n"
-    "where `path` is an importable path to a Python module,\n"
-    "and `variable` is a variable name inside that module."
-)
-
-
 def validate_app(ctx: click.core.Context, param: click.core.Parameter, raw_value: Optional[str]) -> Optional[str]:
     if raw_value is None:
         return raw_value
@@ -139,9 +155,7 @@ def validate_hypothesis_database(
     if raw_value is None:
         return raw_value
     if ctx.params.get("hypothesis_derandomize"):
-        raise click.UsageError(
-            "--hypothesis-derandomize implies no database, so passing --hypothesis-database too is invalid."
-        )
+        raise click.UsageError(INVALID_DERANDOMIZE_MESSAGE)
     return raw_value
 
 
@@ -195,7 +209,7 @@ def validate_request_cert_key(
     ctx: click.core.Context, param: click.core.Parameter, raw_value: Optional[str]
 ) -> Optional[str]:
     if raw_value is not None and "request_cert" not in ctx.params:
-        raise click.UsageError('Missing argument, "--request-cert" should be specified as well.')
+        raise click.UsageError(MISSING_REQUEST_CERT_MESSAGE)
     return raw_value
 
 
@@ -219,6 +233,16 @@ def convert_stateful(ctx: click.core.Context, param: click.core.Parameter, value
     return Stateful[value]
 
 
+def convert_experimental(
+    ctx: click.core.Context, param: click.core.Parameter, value: Tuple[str, ...]
+) -> List[experimental.Experiment]:
+    return [
+        feature
+        for feature in experimental.GLOBAL_EXPERIMENTS.available
+        if feature.name in value or os.getenv(feature.env_var, "").lower() in TRUE_VALUES
+    ]
+
+
 def convert_checks(ctx: click.core.Context, param: click.core.Parameter, value: Tuple[List[str]]) -> List[str]:
     return sum(value, [])
 

schemathesis/cli/context.py

@@ -6,7 +6,7 @@ from typing import List, Optional, Union
 
 import hypothesis
 
-from ..constants import CodeSampleStyle
+from ..code_samples import CodeSampleStyle
 from ..runner.serialization import SerializedTestResult
 
 
@@ -31,6 +31,7 @@ class ExecutionContext:
     workers_num: int = 1
     rate_limit: Optional[str] = None
     show_errors_tracebacks: bool = False
+    wait_for_schema: Optional[float] = None
     validate_schema: bool = True
     operations_processed: int = 0
     # It is set in runtime, from a `Initialized` event

schemathesis/cli/output/default.py

@@ -11,18 +11,22 @@ import requests
 
 from ... import service
 from ..._compat import metadata
-from ...constants import DISCORD_LINK, FLAKY_FAILURE_MESSAGE, CodeSampleStyle, __version__
+from ...constants import (
+    DISCORD_LINK,
+    FLAKY_FAILURE_MESSAGE,
+    REPORT_SUGGESTION_ENV_VAR,
+    SCHEMATHESIS_TEST_CASE_HEADER,
+    __version__,
+)
+from ...experimental import GLOBAL_EXPERIMENTS
 from ...models import Response, Status
 from ...runner import events
+from ...runner.events import InternalErrorType, SchemaErrorType
 from ...runner.serialization import SerializedCase, SerializedError, SerializedTestResult, deduplicate_failures
+from ..callbacks import FALSE_VALUES
 from ..context import ExecutionContext, FileReportContext, ServiceReportContext
 from ..handlers import EventHandler
 
-DISABLE_SCHEMA_VALIDATION_MESSAGE = (
-    "\nYou can disable input schema validation with --validate-schema=false "
-    "command-line option\nIn this case, Schemathesis cannot guarantee proper"
-    " behavior during the test run"
-)
 ISSUE_TRACKER_URL = (
     "https://github.com/schemathesis/schemathesis/issues/new?"
     "labels=Status%3A+Review+Needed%2C+Type%3A+Bug&template=bug_report.md&title=%5BBUG%5D"
@@ -140,8 +144,7 @@ def display_errors(context: ExecutionContext, event: events.Finished) -> None:
             "Add this option to your command line parameters to see full tracebacks: --show-errors-tracebacks", fg="red"
         )
     click.secho(
-        f"\nIf you need assistance in solving this, feel free to join our Discord server and report it:\n\n"
-        f"  {DISCORD_LINK}\n",
+        f"\nNeed more help?\n" f"    Join our Discord server: {DISCORD_LINK}",
         fg="red",
     )
 
@@ -150,7 +153,7 @@ def display_single_error(context: ExecutionContext, result: SerializedTestResult
     display_subsection(result)
     should_display_full_traceback_message = False
     for error in result.errors:
-        should_display_full_traceback_message |= _display_error(context, error, result.seed)
+        should_display_full_traceback_message |= _display_error(context, error)
     return should_display_full_traceback_message
 
 
@@ -162,24 +165,20 @@ def display_generic_errors(context: ExecutionContext, errors: List[SerializedErr
 
 def display_full_traceback_message(exception: str) -> bool:
     # Some errors should not trigger the message that suggests to show full tracebacks to the user
-    return not exception.startswith("DeadlineExceeded")
+    return not exception.startswith(("DeadlineExceeded", "OperationSchemaError"))
 
 
-def _display_error(context: ExecutionContext, error: SerializedError, seed: Optional[int] = None) -> bool:
+def _display_error(context: ExecutionContext, error: SerializedError) -> bool:
     if context.show_errors_tracebacks:
         message = error.exception_with_traceback
     else:
         message = error.exception
-    if error.exception.startswith("InvalidSchema") and context.validate_schema:
-        message += DISABLE_SCHEMA_VALIDATION_MESSAGE + "\n"
     if error.exception.startswith("DeadlineExceeded"):
         message += (
             "Consider extending the deadline with the `--hypothesis-deadline` CLI option.\n"
             "You can disable it completely with `--hypothesis-deadline=None`.\n"
         )
     click.secho(message, fg="red")
-    if error.example is not None:
-        display_example(context, error.example, seed=seed)
     return display_full_traceback_message(error.exception)
 
 
@@ -239,10 +238,20 @@ def display_example(
     if response is not None and response.body is not None:
         payload = base64.b64decode(response.body).decode(response.encoding or "utf8", errors="replace")
         click.secho(f"Response status: {response.status_code}\nResponse payload: `{payload}`\n", fg="red")
-    if context.code_sample_style == CodeSampleStyle.python:
-        click.secho(f"Run this Python code to reproduce this failure: \n\n    {case.requests_code}\n", fg="red")
-    if context.code_sample_style == CodeSampleStyle.curl:
-        click.secho(f"Run this cURL command to reproduce this failure: \n\n    {case.curl_code}\n", fg="red")
+    request_body = base64.b64decode(case.body).decode() if case.body is not None else None
+    code_sample = context.code_sample_style.generate(
+        method=case.method,
+        url=case.url,
+        body=request_body,
+        headers=case.headers,
+        verify=case.verify,
+        extra_headers=case.extra_headers,
+    )
+    click.secho(
+        f"Run this {context.code_sample_style.verbose_name} to reproduce this failure: \n\n    {code_sample}\n",
+        fg="red",
+    )
+    click.secho(f"{SCHEMATHESIS_TEST_CASE_HEADER}: {case.id}\n", fg="red")
     if seed is not None:
         click.secho(f"Or add this option to your command line parameters: --hypothesis-seed={seed}", fg="red")
 
@@ -295,6 +304,22 @@ def display_statistic(context: ExecutionContext, event: events.Finished) -> None
             for warning in event.warnings:
                 click.secho(f"  - {warning}", fg="yellow")
 
+    if len(GLOBAL_EXPERIMENTS.enabled) > 0:
+        click.secho("\nExperimental Features:", bold=True)
+        for experiment in sorted(GLOBAL_EXPERIMENTS.enabled, key=lambda e: e.name):
+            click.secho(f"  - {experiment.verbose_name}: {experiment.description}")
+            click.secho(f"    Feedback: {experiment.discussion_url}")
+        click.echo()
+        click.echo(
+            "Your feedback is crucial for experimental features. "
+            "Please visit the provided URL(s) to share your thoughts."
+        )
+
+    if event.failed_count > 0:
+        click.echo(
+            f"\n{bold('Note')}: The '{SCHEMATHESIS_TEST_CASE_HEADER}' header can be used to correlate test cases with server logs for debugging."
+        )
+
     if context.report is not None and not context.is_interrupted:
         if isinstance(context.report, FileReportContext):
             click.echo()
@@ -304,6 +329,9 @@ def display_statistic(context: ExecutionContext, event: events.Finished) -> None
             click.echo()
             handle_service_integration(context.report)
     else:
+        env_var = os.getenv(REPORT_SUGGESTION_ENV_VAR)
+        if env_var is not None and env_var.lower() in FALSE_VALUES:
+            return
         click.echo()
         category = click.style("Hint", bold=True)
         click.echo(
@@ -354,7 +382,8 @@ def display_report_metadata(meta: service.Metadata) -> None:
 def display_service_error(event: service.Error) -> None:
     """Show information about an error during communication with Schemathesis.io."""
     if isinstance(event.exception, requests.HTTPError):
-        status_code = event.exception.response.status_code
+        response = cast(requests.Response, event.exception.response)
+        status_code = response.status_code
         click.secho(f"Schemathesis.io responded with HTTP {status_code}", fg="red")
         if 500 <= status_code <= 599:
             # Server error, should be resolved soon
@@ -447,23 +476,62 @@ def display_check_result(check_name: str, results: Dict[Union[str, Status], int]
     click.echo(template.format(check_name, f"{success} / {total} passed", click.style(verdict, fg=color, bold=True)))
 
 
+VERIFY_URL_SUGGESTION = "Verify that the URL points directly to the Open API schema"
+
+
+def bold(option: str) -> str:
+    return click.style(option, bold=True)
+
+
+SCHEMA_ERROR_SUGGESTIONS = {
+    # SSL-specific connection issue
+    SchemaErrorType.CONNECTION_SSL: f"Bypass SSL verification with {bold('`--request-tls-verify=false`')}.",
+    # Other connection problems
+    SchemaErrorType.CONNECTION_OTHER: f"Use {bold('`--wait-for-schema=NUM`')} to wait up to NUM seconds for schema availability.",
+    # Response issues
+    SchemaErrorType.UNEXPECTED_CONTENT_TYPE: VERIFY_URL_SUGGESTION,
+    SchemaErrorType.HTTP_FORBIDDEN: "Verify your API keys or authentication headers.",
+    SchemaErrorType.HTTP_NOT_FOUND: VERIFY_URL_SUGGESTION,
+    # OpenAPI specification issues
+    SchemaErrorType.OPEN_API_UNSPECIFIED_VERSION: f"Include the version in the schema or manually set it with {bold('`--force-schema-version`')}.",
+    SchemaErrorType.OPEN_API_UNSUPPORTED_VERSION: f"Proceed with {bold('`--force-schema-version`')}. Caution: May not be fully supported.",
+    SchemaErrorType.OPEN_API_INVALID_SCHEMA: f"Bypass validation using {bold('`--validate-schema=false`')}. Caution: May cause unexpected errors.",
+    # YAML specific issues
+    SchemaErrorType.YAML_NUMERIC_STATUS_CODES: "Convert numeric status codes to strings.",
+    SchemaErrorType.YAML_NON_STRING_KEYS: "Convert non-string keys to strings.",
+    # Unclassified
+    SchemaErrorType.UNCLASSIFIED: f"If you suspect this is a Schemathesis issue and the schema is valid, please report it and include the schema if you can: {ISSUE_TRACKER_URL}",
+}
+
+
+def should_skip_suggestion(context: ExecutionContext, event: events.InternalError) -> bool:
+    return event.subtype == SchemaErrorType.CONNECTION_OTHER and context.wait_for_schema is not None
+
+
 def display_internal_error(context: ExecutionContext, event: events.InternalError) -> None:
-    click.secho(event.message, fg="red")
+    click.secho(event.title, fg="red", bold=True)
     click.echo()
-    if event.exception:
-        if context.show_errors_tracebacks:
-            message = (
-                f"Error: {event.exception_with_traceback}\n"
-                f"Please, consider reporting the traceback below it to our issue tracker: {ISSUE_TRACKER_URL}"
-            )
+    click.secho(event.message)
+    if event.type == InternalErrorType.SCHEMA:
+        extras = event.extras
+    elif context.show_errors_tracebacks:
+        extras = [entry for entry in event.exception_with_traceback.splitlines() if entry]
+    else:
+        extras = [event.exception]
+    if extras:
+        click.echo()
+    for extra in extras:
+        click.secho(f"    {extra}")
+    if not should_skip_suggestion(context, event):
+        if event.type == InternalErrorType.SCHEMA and isinstance(event.subtype, SchemaErrorType):
+            suggestion = SCHEMA_ERROR_SUGGESTIONS.get(event.subtype)
+        elif context.show_errors_tracebacks:
+            suggestion = f"Please consider reporting the traceback above to our issue tracker: {ISSUE_TRACKER_URL}."
         else:
-            message = (
-                f"Error: {event.exception}\n"
-                f"Add this option to your command line parameters to see full tracebacks: --show-errors-tracebacks\n"
-            )
-        if event.exception_type == "schemathesis.exceptions.SchemaLoadingError":
-            message += "\n" + DISABLE_SCHEMA_VALIDATION_MESSAGE
-        click.secho(message, fg="red")
+            suggestion = f"To see full tracebacks, add {bold('`--show-errors-tracebacks`')} to your CLI options"
+        # Display suggestion if any
+        if suggestion is not None:
+            click.secho(f"\n{click.style('Tip:', bold=True, fg='green')} {suggestion}")
 
 
 def handle_initialized(context: ExecutionContext, event: events.Initialized) -> None:

schemathesis/cli/sanitization.py

@@ -0,0 +1,15 @@
+from dataclasses import dataclass
+
+from ..runner import events
+from ..sanitization import sanitize_serialized_check, sanitize_serialized_interaction
+from .handlers import EventHandler, ExecutionContext
+
+
+@dataclass
+class SanitizationHandler(EventHandler):
+    def handle_event(self, context: ExecutionContext, event: events.ExecutionEvent) -> None:
+        if isinstance(event, events.AfterExecution):
+            for check in event.result.checks:
+                sanitize_serialized_check(check)
+            for interaction in event.result.interactions:
+                sanitize_serialized_interaction(interaction)

schemathesis/code_samples.py

@@ -0,0 +1,141 @@
+from enum import Enum
+from shlex import quote
+from typing import Optional, Union
+
+from requests.structures import CaseInsensitiveDict
+from requests.utils import default_headers
+
+from .constants import SCHEMATHESIS_TEST_CASE_HEADER, DataGenerationMethod
+from .types import Headers
+
+DEFAULT_DATA_GENERATION_METHODS = (DataGenerationMethod.default(),)
+# These headers are added automatically by Schemathesis or `requests`.
+# Do not show them in code samples to make them more readable
+EXCLUDED_HEADERS = CaseInsensitiveDict(
+    {
+        "Content-Length": None,
+        "Transfer-Encoding": None,
+        SCHEMATHESIS_TEST_CASE_HEADER: None,
+        **default_headers(),
+    }
+)
+
+
+class CodeSampleStyle(str, Enum):
+    """Controls the style of code samples for failure reproduction."""
+
+    python = "python"
+    curl = "curl"
+
+    @property
+    def verbose_name(self) -> str:
+        return {
+            self.curl: "cURL command",
+            self.python: "Python code",
+        }[self]
+
+    @classmethod
+    def default(cls) -> "CodeSampleStyle":
+        return cls.curl
+
+    @classmethod
+    def from_str(cls, value: str) -> "CodeSampleStyle":
+        try:
+            return cls[value]
+        except KeyError:
+            available_styles = ", ".join(cls)
+            raise ValueError(
+                f"Invalid value for code sample style: {value}. Available styles: {available_styles}"
+            ) from None
+
+    def generate(
+        self,
+        *,
+        method: str,
+        url: str,
+        body: Optional[Union[str, bytes]],
+        headers: Optional[Headers],
+        verify: bool,
+        extra_headers: Optional[Headers] = None,
+    ) -> str:
+        """Generate a code snippet for making HTTP requests."""
+        handlers = {
+            self.curl: _generate_curl,
+            self.python: _generate_requests,
+        }
+        return handlers[self](
+            method=method, url=url, body=body, headers=_filter_headers(headers, extra_headers), verify=verify
+        )
+
+
+def _filter_headers(headers: Optional[Headers], extra: Optional[Headers] = None) -> Headers:
+    headers = headers.copy() if headers else {}
+    if extra is not None:
+        for key, value in extra.items():
+            if key not in EXCLUDED_HEADERS:
+                headers[key] = value
+    return headers
+
+
+def _generate_curl(
+    *,
+    method: str,
+    url: str,
+    body: Optional[Union[str, bytes]],
+    headers: Headers,
+    verify: bool,
+) -> str:
+    """Create a cURL command to reproduce an HTTP request."""
+    command = f"curl -X {method}"
+    for key, value in headers.items():
+        header = f"{key}: {value}"
+        command += f" -H {quote(header)}"
+    if body:
+        if isinstance(body, bytes):
+            body = body.decode("utf-8", errors="replace")
+        command += f" -d {quote(body)}"
+    if not verify:
+        command += " --insecure"
+    return f"{command} {quote(url)}"
+
+
+def _generate_requests(
+    *,
+    method: str,
+    url: str,
+    body: Optional[Union[str, bytes]],
+    headers: Headers,
+    verify: bool,
+) -> str:
+    """Create a Python code to reproduce an HTTP request."""
+    url = _escape_single_quotes(url)
+    command = f"requests.{method.lower()}('{url}'"
+    if body:
+        command += f", data={repr(body)}"
+    if headers:
+        command += f", headers={repr(headers)}"
+    if not verify:
+        command += ", verify=False"
+    command += ")"
+    return command
+
+
+def _escape_single_quotes(url: str) -> str:
+    """Escape single quotes in a string, so it is usable as in generated Python code.
+
+    The usual ``str.replace`` is not suitable as it may convert already escaped quotes to not-escaped.
+    """
+    result = []
+    escape = False
+    for char in url:
+        if escape:
+            result.append(char)
+            escape = False
+        elif char == "\\":
+            result.append(char)
+            escape = True
+        elif char == "'":
+            result.append("\\'")
+        else:
+            result.append(char)
+    return "".join(result)

schemathesis/constants.py

@@ -34,9 +34,6 @@ SERIALIZERS_SUGGESTION_MESSAGE = (
     "and Schemathesis will be able to make API calls with this media type. \n"
     "See https://schemathesis.readthedocs.io/en/stable/how.html#payload-serialization for more information."
 )
-USE_WAIT_FOR_SCHEMA_SUGGESTION_MESSAGE = (
-    "You can use `--wait-for-schema=NUM` to wait for a maximum of NUM seconds on the API schema availability."
-)
 FLAKY_FAILURE_MESSAGE = "[FLAKY] Schemathesis was not able to reliably reproduce this failure"
 BOM_MARK = "\ufeff"
 WAIT_FOR_SCHEMA_INTERVAL = 0.05
@@ -44,6 +41,7 @@ HOOKS_MODULE_ENV_VAR = "SCHEMATHESIS_HOOKS"
 API_NAME_ENV_VAR = "SCHEMATHESIS_API_NAME"
 BASE_URL_ENV_VAR = "SCHEMATHESIS_BASE_URL"
 WAIT_FOR_SCHEMA_ENV_VAR = "SCHEMATHESIS_WAIT_FOR_SCHEMA"
+REPORT_SUGGESTION_ENV_VAR = "SCHEMATHESIS_REPORT_SUGGESTION"
 
 
 class DataGenerationMethod(str, Enum):
@@ -74,24 +72,3 @@ class DataGenerationMethod(str, Enum):
 
 
 DEFAULT_DATA_GENERATION_METHODS = (DataGenerationMethod.default(),)
-
-
-class CodeSampleStyle(str, Enum):
-    """Controls the style of code samples for failure reproduction."""
-
-    python = "python"
-    curl = "curl"
-
-    @classmethod
-    def default(cls) -> "CodeSampleStyle":
-        return cls.curl
-
-    @classmethod
-    def from_str(cls, value: str) -> "CodeSampleStyle":
-        try:
-            return cls[value]
-        except KeyError:
-            available_styles = ", ".join(cls)
-            raise ValueError(
-                f"Invalid value for code sample style: {value}. Available styles: {available_styles}"
-            ) from None