schemathesis 3.34.3__py3-none-any.whl → 3.35.1__py3-none-any.whl

schemathesis/_hypothesis.py

@@ -3,26 +3,28 @@
 from __future__ import annotations
 
 import asyncio
+import json
 import warnings
 from typing import Any, Callable, Generator, Mapping, Optional, Tuple
 
 import hypothesis
 from hypothesis import Phase
-from hypothesis import strategies as st
 from hypothesis.errors import HypothesisWarning, Unsatisfiable
 from hypothesis.internal.entropy import deterministic_PRNG
 from hypothesis.internal.reflection import proxies
 from jsonschema.exceptions import SchemaError
 
 from .auths import get_auth_storage_from_test
-from .constants import DEFAULT_DEADLINE
+from .constants import DEFAULT_DEADLINE, NOT_SET
 from .exceptions import OperationSchemaError, SerializationNotPossible
-from .generation import DataGenerationMethod, GenerationConfig
+from .experimental import COVERAGE_PHASE
+from .generation import DataGenerationMethod, GenerationConfig, combine_strategies, coverage, get_single_example
 from .hooks import GLOBAL_HOOK_DISPATCHER, HookContext, HookDispatcher
-from .models import APIOperation, Case
+from .models import APIOperation, Case, GenerationMetadata, TestPhase
 from .transports.content_types import parse_content_type
 from .transports.headers import has_invalid_characters, is_latin_1_encodable
-from .utils import GivenInput, combine_strategies
+from .types import NotSet
+from .utils import GivenInput
 
 # Forcefully initializes Hypothesis' global PRNG to avoid races that initilize it
 # if e.g. Schemathesis CLI is used with multiple workers
@@ -101,6 +103,8 @@ def create_test(
             wrapped_test = add_examples(
                 wrapped_test, operation, hook_dispatcher=hook_dispatcher, as_strategy_kwargs=as_strategy_kwargs
             )
+            if COVERAGE_PHASE.is_enabled:
+                wrapped_test = add_coverage(wrapped_test, operation, data_generation_methods)
     return wrapped_test
 
 
@@ -199,6 +203,79 @@ def add_examples(
     return test
 
 
+def add_coverage(
+    test: Callable, operation: APIOperation, data_generation_methods: list[DataGenerationMethod]
+) -> Callable:
+    for example in _iter_coverage_cases(operation, data_generation_methods):
+        test = hypothesis.example(case=example)(test)
+    return test
+
+
+def _iter_coverage_cases(
+    operation: APIOperation, data_generation_methods: list[DataGenerationMethod]
+) -> Generator[Case, None, None]:
+    from .specs.openapi.constants import LOCATION_TO_CONTAINER
+
+    ctx = coverage.CoverageContext(data_generation_methods=data_generation_methods)
+    meta = GenerationMetadata(
+        query=None, path_parameters=None, headers=None, cookies=None, body=None, phase=TestPhase.COVERAGE
+    )
+    generators: dict[tuple[str, str], Generator[coverage.GeneratedValue, None, None]] = {}
+    template: dict[str, Any] = {}
+    template_generation_method = DataGenerationMethod.positive
+    for parameter in operation.iter_parameters():
+        schema = parameter.as_json_schema(operation)
+        gen = coverage.cover_schema_iter(ctx, schema)
+        value = next(gen, NOT_SET)
+        if isinstance(value, NotSet):
+            continue
+        location = parameter.location
+        name = parameter.name
+        container = template.setdefault(LOCATION_TO_CONTAINER[location], {})
+        if location in ("header", "cookie") and not isinstance(value.value, str):
+            container[name] = json.dumps(value.value)
+        else:
+            container[name] = value.value
+        template_generation_method = value.data_generation_method
+        generators[(location, name)] = gen
+    if operation.body:
+        for body in operation.body:
+            schema = body.as_json_schema(operation)
+            gen = coverage.cover_schema_iter(ctx, schema)
+            value = next(gen, NOT_SET)
+            if isinstance(value, NotSet):
+                continue
+            if "body" not in template:
+                template["body"] = value.value
+                template["media_type"] = body.media_type
+            case = operation.make_case(**{**template, "body": value.value, "media_type": body.media_type})
+            case.data_generation_method = value.data_generation_method
+            case.meta = meta
+            yield case
+            for next_value in gen:
+                case = operation.make_case(**{**template, "body": next_value.value, "media_type": body.media_type})
+                case.data_generation_method = next_value.data_generation_method
+                case.meta = meta
+                yield case
+    else:
+        case = operation.make_case(**template)
+        case.data_generation_method = template_generation_method
+        case.meta = meta
+        yield case
+    for (location, name), gen in generators.items():
+        container_name = LOCATION_TO_CONTAINER[location]
+        container = template[container_name]
+        for value in gen:
+            if location in ("header", "cookie") and not isinstance(value.value, str):
+                generated = json.dumps(value.value)
+            else:
+                generated = value.value
+            case = operation.make_case(**{**template, container_name: {**container, name: generated}})
+            case.data_generation_method = value.data_generation_method
+            case.meta = meta
+            yield case
+
+
 def find_invalid_headers(headers: Mapping) -> Generator[Tuple[str, str], None, None]:
     for name, value in headers.items():
         if not is_latin_1_encodable(value) or has_invalid_characters(name, value):
@@ -248,25 +325,3 @@ def get_invalid_example_headers_mark(test: Callable) -> Optional[dict[str, str]]
 
 def add_invalid_example_header_mark(test: Callable, headers: dict[str, str]) -> None:
     test._schemathesis_invalid_example_headers = headers  # type: ignore
-
-
-def get_single_example(strategy: st.SearchStrategy[Case]) -> Case:
-    examples: list[Case] = []
-    add_single_example(strategy, examples)
-    return examples[0]
-
-
-def add_single_example(strategy: st.SearchStrategy[Case], examples: list[Case]) -> None:
-    @hypothesis.given(strategy)  # type: ignore
-    @hypothesis.settings(  # type: ignore
-        database=None,
-        max_examples=1,
-        deadline=None,
-        verbosity=hypothesis.Verbosity.quiet,
-        phases=(hypothesis.Phase.generate,),
-        suppress_health_check=list(hypothesis.HealthCheck),
-    )
-    def example_generating_inner_function(ex: Case) -> None:
-        examples.append(ex)
-
-    example_generating_inner_function()

schemathesis/cli/__init__.py

@@ -11,7 +11,7 @@ from collections import defaultdict
 from dataclasses import dataclass
 from enum import Enum
 from queue import Queue
-from typing import TYPE_CHECKING, Any, Callable, Generator, Iterable, Literal, NoReturn, Sequence, cast
+from typing import TYPE_CHECKING, Any, Callable, Generator, Iterable, Literal, NoReturn, Sequence, Type, cast
 from urllib.parse import urlparse
 
 import click
@@ -42,7 +42,7 @@ from ..internal.datetime import current_datetime
 from ..internal.output import OutputConfig
 from ..internal.validation import file_exists
 from ..loaders import load_app, load_yaml
-from ..models import APIOperation, Case, CheckFunction
+from ..models import Case, CheckFunction
 from ..runner import events, prepare_hypothesis_settings, probes
 from ..specs.graphql import loaders as gql_loaders
 from ..specs.openapi import loaders as oas_loaders
@@ -50,11 +50,12 @@ from ..stateful import Stateful
 from ..targets import Target
 from ..transports import RequestConfig
 from ..transports.auth import get_requests_auth
-from ..types import Filter, PathLike, RequestCert
+from ..types import PathLike, RequestCert
 from . import callbacks, cassettes, output
 from .constants import DEFAULT_WORKERS, MAX_WORKERS, MIN_WORKERS, HealthCheck, Phase, Verbosity
 from .context import ExecutionContext, FileReportContext, ServiceReportContext
 from .debug import DebugOutputHandler
+from .handlers import EventHandler
 from .junitxml import JunitXMLHandler
 from .options import CsvChoice, CsvEnumChoice, CustomHelpMessageChoice, NotSet, OptionalInt
 from .sanitization import SanitizationHandler
@@ -66,13 +67,13 @@ if TYPE_CHECKING:
     from ..schemas import BaseSchema
     from ..service.client import ServiceClient
     from ..specs.graphql.schemas import GraphQLSchema
-    from .handlers import EventHandler
 
 
 def _get_callable_names(items: tuple[Callable, ...]) -> tuple[str, ...]:
     return tuple(item.__name__ for item in items)
 
 
+CUSTOM_HANDLERS: list[Type[EventHandler]] = []
 CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]}
 
 DEFAULT_CHECKS_NAMES = _get_callable_names(checks_module.DEFAULT_CHECKS)
@@ -759,6 +760,7 @@ The report data, consisting of a tar gz file with multiple JSON files, is subjec
             experimental.SCHEMA_ANALYSIS.name,
             experimental.STATEFUL_TEST_RUNNER.name,
             experimental.STATEFUL_ONLY.name,
+            experimental.COVERAGE_PHASE.name,
         ]
     ),
     callback=callbacks.convert_experimental,
@@ -927,6 +929,7 @@ def run(
     schemathesis_io_telemetry: bool = True,
     hosts_file: PathLike = service.DEFAULT_HOSTS_PATH,
     force_color: bool = False,
+    **__kwargs,
 ) -> None:
     """Run tests against an API using a specified SCHEMA.
 
@@ -1212,6 +1215,7 @@ def run(
     )
     execute(
         event_stream,
+        ctx=ctx,
         hypothesis_settings=hypothesis_settings,
         workers_num=workers_num,
         rate_limit=rate_limit,
@@ -1595,6 +1599,7 @@ class OutputStyle(Enum):
 def execute(
     event_stream: Generator[events.ExecutionEvent, None, None],
     *,
+    ctx: click.Context,
     hypothesis_settings: hypothesis.settings,
     workers_num: int,
     rate_limit: str | None,
@@ -1669,6 +1674,8 @@ def execute(
                 cassette_path, format=cassette_format, preserve_exact_body_bytes=cassette_preserve_exact_body_bytes
             )
         )
+    for custom_handler in CUSTOM_HANDLERS:
+        handlers.append(custom_handler(*ctx.args, **ctx.params))
     handlers.append(get_output_handler(workers_num))
     if sanitize_output:
         handlers.insert(0, SanitizationHandler())
@@ -2000,6 +2007,20 @@ def decide_color_output(ctx: click.Context, no_color: bool, force_color: bool) -
         ctx.color = False
 
 
+def add_option(*args: Any, cls: Type = click.Option, **kwargs: Any) -> None:
+    """Add a new CLI option to `st run`."""
+    run.params.append(cls(args, **kwargs))
+
+
+def handler() -> Callable[[Type], None]:
+    """Register a new CLI event handler."""
+
+    def _wrapper(cls: Type) -> None:
+        CUSTOM_HANDLERS.append(cls)
+
+    return _wrapper
+
+
 @HookDispatcher.register_spec([HookScope.GLOBAL])
 def after_init_cli_run_handlers(
     context: HookContext, handlers: list[EventHandler], execution_context: ExecutionContext

schemathesis/cli/cassettes.py

@@ -226,6 +226,7 @@ http_interactions:"""
             for interaction in item.interactions:
                 status = interaction.status.name.upper()
                 # Body payloads are handled via separate `stream.write` calls to avoid some allocations
+                phase = f"'{interaction.phase.value}'" if interaction.phase is not None else "null"
                 stream.write(
                     f"""\n- id: '{current_id}'
   status: '{status}'
@@ -233,6 +234,7 @@ http_interactions:"""
   thread_id: {item.thread_id}
   correlation_id: '{item.correlation_id}'
   data_generation_method: '{interaction.data_generation_method.value}'
+  phase: {phase}
   elapsed: '{interaction.response.elapsed}'
   recorded_at: '{interaction.recorded_at}'
   checks:

schemathesis/cli/context.py

@@ -60,7 +60,11 @@ class ExecutionContext:
     analysis: Result[AnalysisResult, Exception] | None = None
     output_config: OutputConfig = field(default_factory=OutputConfig)
     state_machine_sink: StateMachineSink | None = None
+    summary_lines: list[str] = field(default_factory=list)
 
     @deprecated_property(removed_in="4.0", replacement="show_trace")
     def show_errors_tracebacks(self) -> bool:
         return self.show_trace
+
+    def add_summary_line(self, line: str) -> None:
+        self.summary_lines.append(line)

schemathesis/cli/handlers.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 if TYPE_CHECKING:
     from ..runner import events
@@ -8,6 +8,9 @@ if TYPE_CHECKING:
 
 
 class EventHandler:
+    def __init__(self, *args: Any, **params: Any) -> None:
+        pass
+
     def handle_event(self, context: ExecutionContext, event: events.ExecutionEvent) -> None:
         raise NotImplementedError
 

schemathesis/cli/output/default.py

@@ -850,6 +850,10 @@ def handle_finished(context: ExecutionContext, event: events.Finished) -> None:
     display_application_logs(context, event)
     display_analysis(context)
     display_statistic(context, event)
+    if context.summary_lines:
+        click.echo()
+        for line in context.summary_lines:
+            click.echo(line)
     click.echo()
     display_summary(event)
 

schemathesis/contrib/openapi/fill_missing_examples.py

@@ -18,7 +18,7 @@ def uninstall() -> None:
 
 def before_add_examples(context: HookContext, examples: list[Case]) -> None:
     if not examples and context.operation is not None:
-        from ..._hypothesis import add_single_example
+        from ...generation import add_single_example
 
         strategy = context.operation.as_strategy()
         add_single_example(strategy, examples)

schemathesis/experimental/__init__.py

@@ -93,3 +93,10 @@ STATEFUL_ONLY = GLOBAL_EXPERIMENTS.create_experiment(
     description="Run only stateful tests",
     discussion_url="https://github.com/schemathesis/schemathesis/discussions/2262",
 )
+COVERAGE_PHASE = GLOBAL_EXPERIMENTS.create_experiment(
+    name="coverage-phase",
+    verbose_name="Coverage phase",
+    env_var="COVERAGE_PHASE",
+    description="Generate covering test cases",
+    discussion_url="https://github.com/schemathesis/schemathesis/discussions/2418",
+)

schemathesis/generation/__init__.py

@@ -2,48 +2,15 @@ from __future__ import annotations
 
 import random
 from dataclasses import dataclass, field
-from enum import Enum
-from typing import TYPE_CHECKING, Iterable, Union
+from typing import TYPE_CHECKING
+
+from ._hypothesis import add_single_example, combine_strategies, get_single_example  # noqa: E402
+from ._methods import DataGenerationMethod, DataGenerationMethodInput  # noqa: E402
 
 if TYPE_CHECKING:
     from hypothesis.strategies import SearchStrategy
 
 
-class DataGenerationMethod(str, Enum):
-    """Defines what data Schemathesis generates for tests."""
-
-    # Generate data, that fits the API schema
-    positive = "positive"
-    # Doesn't fit the API schema
-    negative = "negative"
-
-    @classmethod
-    def default(cls) -> DataGenerationMethod:
-        return cls.positive
-
-    @classmethod
-    def all(cls) -> list[DataGenerationMethod]:
-        return list(DataGenerationMethod)
-
-    def as_short_name(self) -> str:
-        return {
-            DataGenerationMethod.positive: "P",
-            DataGenerationMethod.negative: "N",
-        }[self]
-
-    @property
-    def is_negative(self) -> bool:
-        return self == DataGenerationMethod.negative
-
-    @classmethod
-    def ensure_list(cls, value: DataGenerationMethodInput) -> list[DataGenerationMethod]:
-        if isinstance(value, DataGenerationMethod):
-            return [value]
-        return list(value)
-
-
-DataGenerationMethodInput = Union[DataGenerationMethod, Iterable[DataGenerationMethod]]
-
 DEFAULT_DATA_GENERATION_METHODS = (DataGenerationMethod.default(),)
 
 

schemathesis/generation/_hypothesis.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+from functools import lru_cache, reduce
+from operator import or_
+from typing import TYPE_CHECKING, TypeVar
+
+if TYPE_CHECKING:
+    from hypothesis import settings
+    from hypothesis import strategies as st
+
+
+@lru_cache()
+def default_settings() -> settings:
+    from hypothesis import HealthCheck, Phase, Verbosity, settings
+
+    return settings(
+        database=None,
+        max_examples=1,
+        deadline=None,
+        verbosity=Verbosity.quiet,
+        phases=(Phase.generate,),
+        suppress_health_check=list(HealthCheck),
+    )
+
+
+T = TypeVar("T")
+
+
+def get_single_example(strategy: st.SearchStrategy[T]) -> T:  # type: ignore[type-var]
+    examples: list[T] = []
+    add_single_example(strategy, examples)
+    return examples[0]
+
+
+def add_single_example(strategy: st.SearchStrategy[T], examples: list[T]) -> None:
+    from hypothesis import given
+
+    @given(strategy)  # type: ignore
+    @default_settings()  # type: ignore
+    def example_generating_inner_function(ex: T) -> None:
+        examples.append(ex)
+
+    example_generating_inner_function()
+
+
+def combine_strategies(strategies: list[st.SearchStrategy] | tuple[st.SearchStrategy]) -> st.SearchStrategy:
+    """Combine a list of strategies into a single one.
+
+    If the input is `[a, b, c]`, then the result is equivalent to `a | b | c`.
+    """
+    return reduce(or_, strategies[1:], strategies[0])

schemathesis/generation/_methods.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from enum import Enum
+from typing import Iterable, Union
+
+
+class DataGenerationMethod(str, Enum):
+    """Defines what data Schemathesis generates for tests."""
+
+    # Generate data, that fits the API schema
+    positive = "positive"
+    # Doesn't fit the API schema
+    negative = "negative"
+
+    @classmethod
+    def default(cls) -> DataGenerationMethod:
+        return cls.positive
+
+    @classmethod
+    def all(cls) -> list[DataGenerationMethod]:
+        return list(DataGenerationMethod)
+
+    def as_short_name(self) -> str:
+        return {
+            DataGenerationMethod.positive: "P",
+            DataGenerationMethod.negative: "N",
+        }[self]
+
+    @property
+    def is_negative(self) -> bool:
+        return self == DataGenerationMethod.negative
+
+    @classmethod
+    def ensure_list(cls, value: DataGenerationMethodInput) -> list[DataGenerationMethod]:
+        if isinstance(value, DataGenerationMethod):
+            return [value]
+        return list(value)
+
+
+DataGenerationMethodInput = Union[DataGenerationMethod, Iterable[DataGenerationMethod]]