schemathesis 3.25.5__py3-none-any.whl → 4.0.0a1__py3-none-any.whl

schemathesis/engine/phases/unit/_executor.py

@@ -0,0 +1,321 @@
+from __future__ import annotations
+
+import time
+import unittest
+import uuid
+from typing import TYPE_CHECKING, Callable, Iterable
+from warnings import WarningMessage, catch_warnings
+
+import requests
+from hypothesis.errors import InvalidArgument
+from hypothesis_jsonschema._canonicalise import HypothesisRefResolutionError
+from jsonschema.exceptions import SchemaError as JsonSchemaError
+from jsonschema.exceptions import ValidationError
+
+from schemathesis.checks import CheckContext, CheckFunction, run_checks
+from schemathesis.core.compat import BaseExceptionGroup
+from schemathesis.core.control import SkipTest
+from schemathesis.core.errors import (
+    SERIALIZERS_SUGGESTION_MESSAGE,
+    InternalError,
+    InvalidHeadersExample,
+    InvalidRegexPattern,
+    InvalidRegexType,
+    InvalidSchema,
+    MalformedMediaType,
+    SerializationNotPossible,
+)
+from schemathesis.core.failures import Failure, FailureGroup
+from schemathesis.core.transport import Response
+from schemathesis.engine import Status, events
+from schemathesis.engine.context import EngineContext
+from schemathesis.engine.errors import (
+    DeadlineExceeded,
+    UnexpectedError,
+    UnsupportedRecursiveReference,
+    deduplicate_errors,
+)
+from schemathesis.engine.phases import PhaseName
+from schemathesis.engine.recorder import ScenarioRecorder
+from schemathesis.generation import targets
+from schemathesis.generation.case import Case
+from schemathesis.generation.hypothesis.builder import (
+    InvalidHeadersExampleMark,
+    InvalidRegexMark,
+    NonSerializableMark,
+    UnsatisfiableExampleMark,
+)
+from schemathesis.generation.hypothesis.reporting import ignore_hypothesis_output
+
+if TYPE_CHECKING:
+    from schemathesis.schemas import APIOperation
+
+
+def run_test(
+    *, operation: APIOperation, test_function: Callable, ctx: EngineContext, suite_id: uuid.UUID
+) -> events.EventGenerator:
+    """A single test run with all error handling needed."""
+    import hypothesis.errors
+
+    # To simplify connecting `before` and `after` events in external systems
+    scenario_started = events.ScenarioStarted(label=operation.label, phase=PhaseName.UNIT_TESTING, suite_id=suite_id)
+    yield scenario_started
+    errors: list[Exception] = []
+    skip_reason = None
+    test_start_time = time.monotonic()
+    recorder = ScenarioRecorder(label=operation.label)
+
+    def non_fatal_error(error: Exception) -> events.NonFatalError:
+        return events.NonFatalError(
+            error=error, phase=PhaseName.UNIT_TESTING, label=operation.label, related_to_operation=True
+        )
+
+    try:
+        setup_hypothesis_database_key(test_function, operation)
+        with catch_warnings(record=True) as warnings, ignore_hypothesis_output():
+            test_function(ctx=ctx, errors=errors, recorder=recorder)
+        # Test body was not executed at all - Hypothesis did not generate any tests, but there is no error
+        status = Status.SUCCESS
+    except (SkipTest, unittest.case.SkipTest) as exc:
+        status = Status.SKIP
+        skip_reason = {"Hypothesis has been told to run no examples for this test.": "No examples in schema"}.get(
+            str(exc), str(exc)
+        )
+    except (FailureGroup, Failure):
+        status = Status.FAILURE
+    except UnexpectedError:
+        # It could be an error in user-defined extensions, network errors or internal Schemathesis errors
+        status = Status.ERROR
+        for idx, err in enumerate(errors):
+            if isinstance(err, MalformedMediaType):
+                errors[idx] = InvalidSchema(str(err))
+    except hypothesis.errors.Flaky as exc:
+        if isinstance(exc.__cause__, hypothesis.errors.DeadlineExceeded):
+            status = Status.ERROR
+            yield non_fatal_error(DeadlineExceeded.from_exc(exc.__cause__))
+        elif isinstance(exc, hypothesis.errors.FlakyFailure) and any(
+            isinstance(subexc, hypothesis.errors.DeadlineExceeded) for subexc in exc.exceptions
+        ):
+            for sub_exc in exc.exceptions:
+                if isinstance(sub_exc, hypothesis.errors.DeadlineExceeded):
+                    yield non_fatal_error(DeadlineExceeded.from_exc(sub_exc))
+            status = Status.ERROR
+        elif errors:
+            status = Status.ERROR
+        else:
+            status = Status.FAILURE
+    except BaseExceptionGroup:
+        status = Status.ERROR
+    except hypothesis.errors.Unsatisfiable:
+        # We need more clear error message here
+        status = Status.ERROR
+        yield non_fatal_error(hypothesis.errors.Unsatisfiable("Failed to generate test cases for this API operation"))
+    except KeyboardInterrupt:
+        yield events.Interrupted(phase=PhaseName.UNIT_TESTING)
+        return
+    except AssertionError as exc:  # May come from `hypothesis-jsonschema` or `hypothesis`
+        status = Status.ERROR
+        try:
+            operation.schema.validate()
+            msg = "Unexpected error during testing of this API operation"
+            exc_msg = str(exc)
+            if exc_msg:
+                msg += f": {exc_msg}"
+            try:
+                raise InternalError(msg) from exc
+            except InternalError as exc:
+                yield non_fatal_error(exc)
+        except ValidationError as exc:
+            yield non_fatal_error(
+                InvalidSchema.from_jsonschema_error(
+                    exc,
+                    path=operation.path,
+                    method=operation.method,
+                    full_path=operation.schema.get_full_path(operation.path),
+                )
+            )
+    except HypothesisRefResolutionError:
+        status = Status.ERROR
+        yield non_fatal_error(UnsupportedRecursiveReference())
+    except InvalidArgument as exc:
+        status = Status.ERROR
+        message = get_invalid_regular_expression_message(warnings)
+        if message:
+            # `hypothesis-jsonschema` emits a warning on invalid regular expression syntax
+            yield non_fatal_error(InvalidRegexPattern.from_hypothesis_jsonschema_message(message))
+        else:
+            yield non_fatal_error(exc)
+    except hypothesis.errors.DeadlineExceeded as exc:
+        status = Status.ERROR
+        yield non_fatal_error(DeadlineExceeded.from_exc(exc))
+    except JsonSchemaError as exc:
+        status = Status.ERROR
+        yield non_fatal_error(InvalidRegexPattern.from_schema_error(exc, from_examples=False))
+    except Exception as exc:
+        status = Status.ERROR
+        # Likely a YAML parsing issue. E.g. `00:00:00.00` (without quotes) is parsed as float `0.0`
+        if str(exc) == "first argument must be string or compiled pattern":
+            yield non_fatal_error(
+                InvalidRegexType(
+                    "Invalid `pattern` value: expected a string. "
+                    "If your schema is in YAML, ensure `pattern` values are quoted",
+                )
+            )
+        else:
+            yield non_fatal_error(exc)
+    if (
+        status == Status.SUCCESS
+        and ctx.config.execution.no_failfast
+        and any(check.status == Status.FAILURE for checks in recorder.checks.values() for check in checks)
+    ):
+        status = Status.FAILURE
+    if UnsatisfiableExampleMark.is_set(test_function):
+        status = Status.ERROR
+        yield non_fatal_error(
+            hypothesis.errors.Unsatisfiable("Failed to generate test cases from examples for this API operation")
+        )
+    non_serializable = NonSerializableMark.get(test_function)
+    if non_serializable is not None and status != Status.ERROR:
+        status = Status.ERROR
+        media_types = ", ".join(non_serializable.media_types)
+        yield non_fatal_error(
+            SerializationNotPossible(
+                "Failed to generate test cases from examples for this API operation because of"
+                f" unsupported payload media types: {media_types}\n{SERIALIZERS_SUGGESTION_MESSAGE}",
+                media_types=non_serializable.media_types,
+            )
+        )
+
+    invalid_regex = InvalidRegexMark.get(test_function)
+    if invalid_regex is not None and status != Status.ERROR:
+        status = Status.ERROR
+        yield non_fatal_error(InvalidRegexPattern.from_schema_error(invalid_regex, from_examples=True))
+    invalid_headers = InvalidHeadersExampleMark.get(test_function)
+    if invalid_headers:
+        status = Status.ERROR
+        yield non_fatal_error(InvalidHeadersExample.from_headers(invalid_headers))
+    test_elapsed_time = time.monotonic() - test_start_time
+    for error in deduplicate_errors(errors):
+        yield non_fatal_error(error)
+    yield events.ScenarioFinished(
+        id=scenario_started.id,
+        suite_id=suite_id,
+        phase=PhaseName.UNIT_TESTING,
+        recorder=recorder,
+        status=status,
+        elapsed_time=test_elapsed_time,
+        skip_reason=skip_reason,
+        is_final=False,
+    )
+
+
+def setup_hypothesis_database_key(test: Callable, operation: APIOperation) -> None:
+    """Make Hypothesis use separate database entries for every API operation.
+
+    It increases the effectiveness of the Hypothesis database in the CLI.
+    """
+    # Hypothesis's function digest depends on the test function signature. To reflect it for the web API case,
+    # we use all API operation parameters in the digest.
+    extra = operation.label.encode("utf8")
+    for parameter in operation.iter_parameters():
+        extra += parameter.serialize(operation).encode("utf8")
+    test.hypothesis.inner_test._hypothesis_internal_add_digest = extra  # type: ignore
+
+
+def get_invalid_regular_expression_message(warnings: list[WarningMessage]) -> str | None:
+    for warning in warnings:
+        message = str(warning.message)
+        if "is not valid syntax for a Python regular expression" in message:
+            return message
+    return None
+
+
+def cached_test_func(f: Callable) -> Callable:
+    def wrapped(*, ctx: EngineContext, case: Case, errors: list[Exception], recorder: ScenarioRecorder) -> None:
+        try:
+            if ctx.has_to_stop:
+                raise KeyboardInterrupt
+            if ctx.config.execution.unique_inputs:
+                cached = ctx.get_cached_outcome(case)
+                if isinstance(cached, BaseException):
+                    raise cached
+                elif cached is None:
+                    return None
+                try:
+                    f(ctx=ctx, case=case, recorder=recorder)
+                except BaseException as exc:
+                    ctx.cache_outcome(case, exc)
+                    raise
+                else:
+                    ctx.cache_outcome(case, None)
+            else:
+                f(ctx=ctx, case=case, recorder=recorder)
+        except (KeyboardInterrupt, Failure):
+            raise
+        except Exception as exc:
+            errors.append(exc)
+            raise UnexpectedError from None
+
+    wrapped.__name__ = f.__name__
+
+    return wrapped
+
+
+@cached_test_func
+def test_func(*, ctx: EngineContext, case: Case, recorder: ScenarioRecorder) -> None:
+    recorder.record_case(parent_id=None, case=case)
+    try:
+        response = case.call(**ctx.transport_kwargs)
+    except (requests.Timeout, requests.ConnectionError) as error:
+        if isinstance(error.request, requests.Request):
+            recorder.record_request(case_id=case.id, request=error.request.prepare())
+        elif isinstance(error.request, requests.PreparedRequest):
+            recorder.record_request(case_id=case.id, request=error.request)
+        raise
+    recorder.record_response(case_id=case.id, response=response)
+    targets.run(ctx.config.execution.targets, case=case, response=response)
+    validate_response(
+        case=case,
+        ctx=ctx.get_check_context(recorder),
+        checks=ctx.config.execution.checks,
+        response=response,
+        no_failfast=ctx.config.execution.no_failfast,
+        recorder=recorder,
+    )
+
+
+def validate_response(
+    *,
+    case: Case,
+    ctx: CheckContext,
+    checks: Iterable[CheckFunction],
+    response: Response,
+    no_failfast: bool,
+    recorder: ScenarioRecorder,
+) -> None:
+    failures = set()
+
+    def on_failure(name: str, collected: set[Failure], failure: Failure) -> None:
+        collected.add(failure)
+        failure_data = recorder.find_failure_data(parent_id=case.id, failure=failure)
+        recorder.record_check_failure(
+            name=name,
+            case_id=failure_data.case.id,
+            code_sample=failure_data.case.as_curl_command(headers=failure_data.headers, verify=failure_data.verify),
+            failure=failure,
+        )
+
+    def on_success(name: str, _case: Case) -> None:
+        recorder.record_check_success(name=name, case_id=_case.id)
+
+    failures = run_checks(
+        case=case,
+        response=response,
+        ctx=ctx,
+        checks=checks,
+        on_failure=on_failure,
+        on_success=on_success,
+    )
+
+    if failures and not no_failfast:
+        raise FailureGroup(list(failures)) from None

schemathesis/engine/phases/unit/_pool.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+import threading
+import uuid
+from queue import Queue
+from types import TracebackType
+from typing import TYPE_CHECKING, Callable
+
+from schemathesis.core.result import Result
+
+if TYPE_CHECKING:
+    from schemathesis.engine.context import EngineContext
+
+
+class TaskProducer:
+    """Produces test tasks for workers to execute."""
+
+    def __init__(self, ctx: EngineContext) -> None:
+        self.operations = ctx.schema.get_all_operations(generation_config=ctx.config.execution.generation)
+        self.lock = threading.Lock()
+
+    def next_operation(self) -> Result | None:
+        """Get next API operation in a thread-safe manner."""
+        with self.lock:
+            return next(self.operations, None)
+
+
+class WorkerPool:
+    """Manages a pool of worker threads."""
+
+    def __init__(
+        self,
+        workers_num: int,
+        producer: TaskProducer,
+        worker_factory: Callable,
+        ctx: EngineContext,
+        suite_id: uuid.UUID,
+    ) -> None:
+        self.workers_num = workers_num
+        self.producer = producer
+        self.worker_factory = worker_factory
+        self.ctx = ctx
+        self.suite_id = suite_id
+        self.workers: list[threading.Thread] = []
+        self.events_queue: Queue = Queue()
+
+    def start(self) -> None:
+        """Start all worker threads."""
+        for i in range(self.workers_num):
+            worker = threading.Thread(
+                target=self.worker_factory,
+                kwargs={
+                    "ctx": self.ctx,
+                    "events_queue": self.events_queue,
+                    "producer": self.producer,
+                    "suite_id": self.suite_id,
+                },
+                name=f"schemathesis_{i}",
+                daemon=True,
+            )
+            self.workers.append(worker)
+            worker.start()
+
+    def stop(self) -> None:
+        """Stop all workers gracefully."""
+        for worker in self.workers:
+            worker.join()
+
+    def __enter__(self) -> WorkerPool:
+        self.start()
+        return self
+
+    def __exit__(self, ty: type[BaseException] | None, value: BaseException | None, tb: TracebackType | None) -> None:
+        self.stop()

schemathesis/engine/recorder.py

@@ -0,0 +1,241 @@
+from __future__ import annotations
+
+import base64
+import time
+import uuid
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Iterator, cast
+
+from schemathesis.core.failures import Failure
+from schemathesis.core.transport import Response
+from schemathesis.engine import Status
+from schemathesis.generation.case import Case
+
+if TYPE_CHECKING:
+    import requests
+
+
+@dataclass
+class ScenarioRecorder:
+    """Tracks and organizes all data related to a logical block of testing.
+
+    Records test cases, their hierarchy, API interactions, and results of checks performed during execution.
+    """
+
+    id: uuid.UUID
+    # Human-readable label
+    label: str
+
+    # Recorded test cases
+    cases: dict[str, CaseNode]
+    # Results of checks categorized by test case ID
+    checks: dict[str, list[CheckNode]]
+    # Network interactions by test case ID
+    interactions: dict[str, Interaction]
+
+    __slots__ = ("id", "label", "status", "roots", "cases", "checks", "interactions")
+
+    def __init__(self, *, label: str) -> None:
+        self.id = uuid.uuid4()
+        self.label = label
+        self.cases = {}
+        self.checks = {}
+        self.interactions = {}
+
+    def record_case(self, *, parent_id: str | None, case: Case) -> None:
+        """Record a test case and its relationship to a parent, if applicable."""
+        self.cases[case.id] = CaseNode(value=case, parent_id=parent_id)
+
+    def record_response(self, *, case_id: str, response: Response) -> None:
+        """Record the API response for a given test case."""
+        request = Request.from_prepared_request(response.request)
+        self.interactions[case_id] = Interaction(request=request, response=response)
+
+    def record_request(self, *, case_id: str, request: requests.PreparedRequest) -> None:
+        """Record a network-level error for a given test case."""
+        self.interactions[case_id] = Interaction(request=Request.from_prepared_request(request), response=None)
+
+    def record_check_failure(self, *, name: str, case_id: str, code_sample: str, failure: Failure) -> None:
+        """Record a failure of a check for a given test case."""
+        self.checks.setdefault(case_id, []).append(
+            CheckNode(
+                name=name,
+                status=Status.FAILURE,
+                failure_info=CheckFailureInfo(code_sample=code_sample, failure=failure),
+            )
+        )
+
+    def record_check_success(self, *, name: str, case_id: str) -> None:
+        """Record a successful pass of a check for a given test case."""
+        self.checks.setdefault(case_id, []).append(CheckNode(name=name, status=Status.SUCCESS, failure_info=None))
+
+    def find_failure_data(self, *, parent_id: str, failure: Failure) -> FailureData:
+        """Retrieve the relevant test case & interaction data for a failure.
+
+        It may happen that a failure comes from a different test case if a check generated some additional
+        test cases & interactions.
+        """
+        case_id = failure.case_id or parent_id
+        case = self.cases[case_id].value
+        request = self.interactions[case_id].request
+        response = self.interactions[case_id].response
+        assert isinstance(response, Response)
+        headers = {key: value[0] for key, value in request.headers.items()}
+        return FailureData(case=case, headers=headers, verify=response.verify)
+
+    def find_parent(self, *, case_id: str) -> Case | None:
+        """Find the parent case of a given test case, if it exists."""
+        case = self.cases.get(case_id)
+        if case is not None and case.parent_id is not None:
+            parent = self.cases.get(case.parent_id)
+            # The recorder state should always be consistent
+            assert parent is not None, "Parent does not exist"
+            return parent.value
+        return None
+
+    def find_related(self, *, case_id: str) -> Iterator[Case]:
+        """Iterate over all ancestors and their children for a given case."""
+        current_id = case_id
+        seen = {current_id}
+
+        while True:
+            current_node = self.cases.get(current_id)
+            if current_node is None or current_node.parent_id is None:
+                break
+
+            # Get all children of the parent (siblings of the current case)
+            parent_id = current_node.parent_id
+            for case_id, maybe_child in self.cases.items():
+                # If this case has the same parent and we haven't seen it yet
+                if parent_id == maybe_child.parent_id and case_id not in seen:
+                    seen.add(case_id)
+                    yield maybe_child.value
+
+            # Move up to the parent
+            current_id = parent_id
+            if current_id not in seen:
+                seen.add(current_id)
+                parent_node = self.cases.get(current_id)
+                if parent_node:
+                    yield parent_node.value
+
+    def find_response(self, *, case_id: str) -> Response | None:
+        """Retrieve the API response for a given test case, if available."""
+        interaction = self.interactions.get(case_id)
+        if interaction is None or interaction.response is None:
+            return None
+        return interaction.response
+
+
+@dataclass
+class CaseNode:
+    """Represents a test case and its parent-child relationship."""
+
+    value: Case
+    parent_id: str | None
+
+    __slots__ = ("value", "parent_id")
+
+
+@dataclass
+class CheckNode:
+    name: str
+    status: Status
+    failure_info: CheckFailureInfo | None
+
+    __slots__ = ("name", "status", "failure_info")
+
+
+@dataclass
+class CheckFailureInfo:
+    code_sample: str
+    failure: Failure
+
+    __slots__ = ("code_sample", "failure")
+
+
+def serialize_payload(payload: bytes) -> str:
+    return base64.b64encode(payload).decode()
+
+
+@dataclass(repr=False)
+class Request:
+    """Request data extracted from `Case`."""
+
+    method: str
+    uri: str
+    body: bytes | None
+    body_size: int | None
+    headers: dict[str, list[str]]
+
+    __slots__ = ("method", "uri", "body", "body_size", "headers", "_encoded_body_cache")
+
+    def __init__(
+        self,
+        method: str,
+        uri: str,
+        body: bytes | None,
+        body_size: int | None,
+        headers: dict[str, list[str]],
+    ):
+        self.method = method
+        self.uri = uri
+        self.body = body
+        self.body_size = body_size
+        self.headers = headers
+        self._encoded_body_cache: str | None = None
+
+    @classmethod
+    def from_prepared_request(cls, prepared: requests.PreparedRequest) -> Request:
+        """A prepared request version is already stored in `requests.Response`."""
+        body = prepared.body
+
+        if isinstance(body, str):
+            # can be a string for `application/x-www-form-urlencoded`
+            body = body.encode("utf-8")
+
+        # these values have `str` type at this point
+        uri = cast(str, prepared.url)
+        method = cast(str, prepared.method)
+        return cls(
+            uri=uri,
+            method=method,
+            headers={key: [value] for (key, value) in prepared.headers.items()},
+            body=body,
+            body_size=len(body) if body is not None else None,
+        )
+
+    @property
+    def encoded_body(self) -> str | None:
+        if self.body is not None:
+            if self._encoded_body_cache is None:
+                self._encoded_body_cache = serialize_payload(self.body)
+            return self._encoded_body_cache
+        return None
+
+
+@dataclass
+class Interaction:
+    """Represents a single interaction with the tested application."""
+
+    request: Request
+    response: Response | None
+    timestamp: float
+
+    __slots__ = ("request", "response", "timestamp")
+
+    def __init__(self, request: Request, response: Response | None) -> None:
+        self.request = request
+        self.response = response
+        self.timestamp = time.time()
+
+
+@dataclass
+class FailureData:
+    """Details about a test failure, including the case and its context."""
+
+    case: Case
+    headers: dict[str, str]
+    verify: bool
+
+    __slots__ = ("case", "headers", "verify")

schemathesis/errors.py

@@ -0,0 +1,31 @@
+"""Public Schemathesis errors."""
+
+from schemathesis.core.errors import IncorrectUsage as IncorrectUsage
+from schemathesis.core.errors import InternalError as InternalError
+from schemathesis.core.errors import InvalidHeadersExample as InvalidHeadersExample
+from schemathesis.core.errors import InvalidRateLimit as InvalidRateLimit
+from schemathesis.core.errors import InvalidRegexPattern as InvalidRegexPattern
+from schemathesis.core.errors import InvalidRegexType as InvalidRegexType
+from schemathesis.core.errors import InvalidSchema as InvalidSchema
+from schemathesis.core.errors import LoaderError as LoaderError
+from schemathesis.core.errors import OperationNotFound as OperationNotFound
+from schemathesis.core.errors import SchemathesisError as SchemathesisError
+from schemathesis.core.errors import SerializationError as SerializationError
+from schemathesis.core.errors import SerializationNotPossible as SerializationNotPossible
+from schemathesis.core.errors import UnboundPrefix as UnboundPrefix
+
+__all__ = [
+    "IncorrectUsage",
+    "InternalError",
+    "InvalidHeadersExample",
+    "InvalidRateLimit",
+    "InvalidRegexPattern",
+    "InvalidRegexType",
+    "InvalidSchema",
+    "LoaderError",
+    "OperationNotFound",
+    "SchemathesisError",
+    "SerializationError",
+    "SerializationNotPossible",
+    "UnboundPrefix",
+]