schemathesis 3.34.3__tar.gz → 3.35.0__tar.gz

{schemathesis-3.34.3 → schemathesis-3.35.0}/.pre-commit-config.yaml

@@ -38,12 +38,12 @@ repos:
         additional_dependencies: [types-click, types-requests, types-PyYAML]
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.1
+    rev: v0.6.2
     hooks:
       - id: ruff-format
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.1
+    rev: v0.6.2
     hooks:
       - id: ruff
 

{schemathesis-3.34.3 → schemathesis-3.35.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: schemathesis
-Version: 3.34.3
+Version: 3.35.0
 Summary: Property-based testing framework for Open API and GraphQL based apps
 Project-URL: Documentation, https://schemathesis.readthedocs.io/en/stable/
 Project-URL: Changelog, https://schemathesis.readthedocs.io/en/stable/changelog.html

{schemathesis-3.34.3 → schemathesis-3.35.0}/docs/changelog.rst

@@ -1,9 +1,19 @@
 Changelog
 =========
 
-:version:`Unreleased <v3.34.3...HEAD>` - TBD
+:version:`Unreleased <v3.35.0...HEAD>` - TBD
 --------------------------------------------
 
+.. _v3.35.0:
+
+:version:`3.35.0 <v3.34.3...v3.35.0>` - 2024-08-25
+--------------------------------------------------
+
+**Added**
+
+- **EXPERIMENTAL**: New "coverage" phase in the test runner. It aims to explicitly cover common test scenarios like missing required properties, incorrect types, etc. Enable it with ``--experimental=coverage-phase``
+- Extending CLI with custom options and CLI handlers via hooks.
+
 .. _v3.34.3:
 
 :version:`3.34.3 <v3.34.2...v3.34.3>` - 2024-08-24

{schemathesis-3.34.3 → schemathesis-3.35.0}/docs/experimental.rst

@@ -112,6 +112,31 @@ To enable OpenAPI 3.1 support via environment variables, you can set:
 
 For more details, join the `GitHub Discussion #1822 <https://github.com/schemathesis/schemathesis/discussions/1822>`_.
 
+.. _coverage-phase:
+
+Coverage phase
+~~~~~~~~~~~~~~
+
+Schemathesis primarily generates randomized tests; therefore, the outcomes can vary between test runs.
+This feature enables a new test phase that aims to generate a set of examples that cover common testing patterns for each parameter or property.
+Such tests include checking boundary values, omitting required properties, etc.
+
+In CLI
+~~~~~~
+
+.. code-block:: bash
+
+    st run https://example.schemathesis.io/openapi.json --experimental=coverage-phase
+
+Using Environment Variables
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code-block:: bash
+
+    export SCHEMATHESIS_EXPERIMENTAL_COVERAGE_PHASE=true
+
+For more details, join the `GitHub Discussion #2418 <https://github.com/schemathesis/schemathesis/discussions/2418>`_.
+
 .. _stateful-test-runner:
 
 New Stateful Test Runner

{schemathesis-3.34.3 → schemathesis-3.35.0}/docs/extending.rst

@@ -374,6 +374,34 @@ Here's a GraphQL example that includes all queries:
 In these examples, the ``filter_operations`` hook skips all ``POST`` methods in Open API and all mutations in GraphQL.
 You can implement any custom logic within the ``filter_operations`` function to include or exclude specific API operations.
 
+Extending CLI
+~~~~~~~~~~~~~
+
+This example demonstrates how to add a custom CLI option and an event handler that uses it:
+
+.. code:: python
+
+    from schemathesis import cli, runner
+
+
+    cli.add_option("--custom-counter", type=int)
+
+
+    @cli.handler()
+    class EventCounter(cli.EventHandler):
+        def __init__(self, *args, **params):
+            self.counter = params["custom_counter"] or 0
+
+        def handle_event(self, context, event) -> None:
+            self.counter += 1
+            if isinstance(event, runner.events.Finished):
+                context.add_summary_line(
+                    f"Counter: {self.counter}",
+                )
+
+The ``--custom-counter`` CLI option sets the initial value for the ``EventCounter`` handler. 
+The handler increments the counter for each event and adds a summary line with the final count when the test run finishes.
+
 ``before_process_path``
 ~~~~~~~~~~~~~~~~~~~~~~~
 

{schemathesis-3.34.3 → schemathesis-3.35.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "schemathesis"
-version = "3.34.3"
+version = "3.35.0"
 description = "Property-based testing framework for Open API and GraphQL based apps"
 keywords = ["pytest", "hypothesis", "openapi", "graphql", "testing"]
 authors = [{ name = "Dmitry Dygalo", email = "dmitry@dygalo.dev" }]

{schemathesis-3.34.3 → schemathesis-3.35.0}/src/schemathesis/_hypothesis.py

@@ -8,21 +8,21 @@ from typing import Any, Callable, Generator, Mapping, Optional, Tuple
 
 import hypothesis
 from hypothesis import Phase
-from hypothesis import strategies as st
 from hypothesis.errors import HypothesisWarning, Unsatisfiable
 from hypothesis.internal.entropy import deterministic_PRNG
 from hypothesis.internal.reflection import proxies
 from jsonschema.exceptions import SchemaError
 
 from .auths import get_auth_storage_from_test
-from .constants import DEFAULT_DEADLINE
+from .constants import DEFAULT_DEADLINE, NOT_SET
 from .exceptions import OperationSchemaError, SerializationNotPossible
-from .generation import DataGenerationMethod, GenerationConfig
+from .experimental import COVERAGE_PHASE
+from .generation import DataGenerationMethod, GenerationConfig, combine_strategies, coverage, get_single_example
 from .hooks import GLOBAL_HOOK_DISPATCHER, HookContext, HookDispatcher
 from .models import APIOperation, Case
 from .transports.content_types import parse_content_type
 from .transports.headers import has_invalid_characters, is_latin_1_encodable
-from .utils import GivenInput, combine_strategies
+from .utils import GivenInput
 
 # Forcefully initializes Hypothesis' global PRNG to avoid races that initilize it
 # if e.g. Schemathesis CLI is used with multiple workers
@@ -101,6 +101,8 @@ def create_test(
             wrapped_test = add_examples(
                 wrapped_test, operation, hook_dispatcher=hook_dispatcher, as_strategy_kwargs=as_strategy_kwargs
             )
+            if COVERAGE_PHASE.is_enabled:
+                wrapped_test = add_coverage(wrapped_test, operation, data_generation_methods)
     return wrapped_test
 
 
@@ -199,6 +201,55 @@ def add_examples(
     return test
 
 
+def add_coverage(
+    test: Callable, operation: APIOperation, data_generation_methods: list[DataGenerationMethod]
+) -> Callable:
+    for example in _iter_coverage_cases(operation, data_generation_methods):
+        test = hypothesis.example(case=example)(test)
+    return test
+
+
+def _iter_coverage_cases(
+    operation: APIOperation, data_generation_methods: list[DataGenerationMethod]
+) -> Generator[Case, None, None]:
+    from .specs.openapi.constants import LOCATION_TO_CONTAINER
+
+    ctx = coverage.CoverageContext(data_generation_methods=data_generation_methods)
+    generators: dict[tuple[str, str], Generator] = {}
+    template: dict[str, Any] = {}
+    for parameter in operation.iter_parameters():
+        schema = parameter.as_json_schema(operation)
+        gen = coverage.cover_schema_iter(ctx, schema)
+        value = next(gen, NOT_SET)
+        if value is NOT_SET:
+            continue
+        location = parameter.location
+        name = parameter.name
+        container = template.setdefault(LOCATION_TO_CONTAINER[location], {})
+        container[name] = value
+        generators[(location, name)] = gen
+    if operation.body:
+        for body in operation.body:
+            schema = body.as_json_schema(operation)
+            gen = coverage.cover_schema_iter(ctx, schema)
+            value = next(gen, NOT_SET)
+            if value is NOT_SET:
+                continue
+            if "body" not in template:
+                template["body"] = value
+                template["media_type"] = body.media_type
+            yield operation.make_case(**{**template, "body": value, "media_type": body.media_type})
+            for next_value in gen:
+                yield operation.make_case(**{**template, "body": next_value, "media_type": body.media_type})
+    else:
+        yield operation.make_case(**template)
+    for (location, name), gen in generators.items():
+        container_name = LOCATION_TO_CONTAINER[location]
+        container = template[container_name]
+        for value in gen:
+            yield operation.make_case(**{**template, container_name: {**container, name: value}})
+
+
 def find_invalid_headers(headers: Mapping) -> Generator[Tuple[str, str], None, None]:
     for name, value in headers.items():
         if not is_latin_1_encodable(value) or has_invalid_characters(name, value):
@@ -248,25 +299,3 @@ def get_invalid_example_headers_mark(test: Callable) -> Optional[dict[str, str]]
 
 def add_invalid_example_header_mark(test: Callable, headers: dict[str, str]) -> None:
     test._schemathesis_invalid_example_headers = headers  # type: ignore
-
-
-def get_single_example(strategy: st.SearchStrategy[Case]) -> Case:
-    examples: list[Case] = []
-    add_single_example(strategy, examples)
-    return examples[0]
-
-
-def add_single_example(strategy: st.SearchStrategy[Case], examples: list[Case]) -> None:
-    @hypothesis.given(strategy)  # type: ignore
-    @hypothesis.settings(  # type: ignore
-        database=None,
-        max_examples=1,
-        deadline=None,
-        verbosity=hypothesis.Verbosity.quiet,
-        phases=(hypothesis.Phase.generate,),
-        suppress_health_check=list(hypothesis.HealthCheck),
-    )
-    def example_generating_inner_function(ex: Case) -> None:
-        examples.append(ex)
-
-    example_generating_inner_function()

{schemathesis-3.34.3 → schemathesis-3.35.0}/src/schemathesis/cli/__init__.py

@@ -11,7 +11,7 @@ from collections import defaultdict
 from dataclasses import dataclass
 from enum import Enum
 from queue import Queue
-from typing import TYPE_CHECKING, Any, Callable, Generator, Iterable, Literal, NoReturn, Sequence, cast
+from typing import TYPE_CHECKING, Any, Callable, Generator, Iterable, Literal, NoReturn, Sequence, Type, cast
 from urllib.parse import urlparse
 
 import click
@@ -42,7 +42,7 @@ from ..internal.datetime import current_datetime
 from ..internal.output import OutputConfig
 from ..internal.validation import file_exists
 from ..loaders import load_app, load_yaml
-from ..models import APIOperation, Case, CheckFunction
+from ..models import Case, CheckFunction
 from ..runner import events, prepare_hypothesis_settings, probes
 from ..specs.graphql import loaders as gql_loaders
 from ..specs.openapi import loaders as oas_loaders
@@ -50,11 +50,12 @@ from ..stateful import Stateful
 from ..targets import Target
 from ..transports import RequestConfig
 from ..transports.auth import get_requests_auth
-from ..types import Filter, PathLike, RequestCert
+from ..types import PathLike, RequestCert
 from . import callbacks, cassettes, output
 from .constants import DEFAULT_WORKERS, MAX_WORKERS, MIN_WORKERS, HealthCheck, Phase, Verbosity
 from .context import ExecutionContext, FileReportContext, ServiceReportContext
 from .debug import DebugOutputHandler
+from .handlers import EventHandler
 from .junitxml import JunitXMLHandler
 from .options import CsvChoice, CsvEnumChoice, CustomHelpMessageChoice, NotSet, OptionalInt
 from .sanitization import SanitizationHandler
@@ -66,13 +67,13 @@ if TYPE_CHECKING:
     from ..schemas import BaseSchema
     from ..service.client import ServiceClient
     from ..specs.graphql.schemas import GraphQLSchema
-    from .handlers import EventHandler
 
 
 def _get_callable_names(items: tuple[Callable, ...]) -> tuple[str, ...]:
     return tuple(item.__name__ for item in items)
 
 
+CUSTOM_HANDLERS: list[Type[EventHandler]] = []
 CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]}
 
 DEFAULT_CHECKS_NAMES = _get_callable_names(checks_module.DEFAULT_CHECKS)
@@ -759,6 +760,7 @@ The report data, consisting of a tar gz file with multiple JSON files, is subjec
             experimental.SCHEMA_ANALYSIS.name,
             experimental.STATEFUL_TEST_RUNNER.name,
             experimental.STATEFUL_ONLY.name,
+            experimental.COVERAGE_PHASE.name,
         ]
     ),
     callback=callbacks.convert_experimental,
@@ -927,6 +929,7 @@ def run(
     schemathesis_io_telemetry: bool = True,
     hosts_file: PathLike = service.DEFAULT_HOSTS_PATH,
     force_color: bool = False,
+    **__kwargs,
 ) -> None:
     """Run tests against an API using a specified SCHEMA.
 
@@ -1212,6 +1215,7 @@ def run(
     )
     execute(
         event_stream,
+        ctx=ctx,
         hypothesis_settings=hypothesis_settings,
         workers_num=workers_num,
         rate_limit=rate_limit,
@@ -1595,6 +1599,7 @@ class OutputStyle(Enum):
 def execute(
     event_stream: Generator[events.ExecutionEvent, None, None],
     *,
+    ctx: click.Context,
     hypothesis_settings: hypothesis.settings,
     workers_num: int,
     rate_limit: str | None,
@@ -1669,6 +1674,8 @@ def execute(
                 cassette_path, format=cassette_format, preserve_exact_body_bytes=cassette_preserve_exact_body_bytes
             )
         )
+    for custom_handler in CUSTOM_HANDLERS:
+        handlers.append(custom_handler(*ctx.args, **ctx.params))
     handlers.append(get_output_handler(workers_num))
     if sanitize_output:
         handlers.insert(0, SanitizationHandler())
@@ -2000,6 +2007,20 @@ def decide_color_output(ctx: click.Context, no_color: bool, force_color: bool) -
         ctx.color = False
 
 
+def add_option(*args: Any, cls: Type = click.Option, **kwargs: Any) -> None:
+    """Add a new CLI option to `st run`."""
+    run.params.append(cls(args, **kwargs))
+
+
+def handler() -> Callable[[Type], None]:
+    """Register a new CLI event handler."""
+
+    def _wrapper(cls: Type) -> None:
+        CUSTOM_HANDLERS.append(cls)
+
+    return _wrapper
+
+
 @HookDispatcher.register_spec([HookScope.GLOBAL])
 def after_init_cli_run_handlers(
     context: HookContext, handlers: list[EventHandler], execution_context: ExecutionContext

{schemathesis-3.34.3 → schemathesis-3.35.0}/src/schemathesis/cli/context.py

@@ -60,7 +60,11 @@ class ExecutionContext:
     analysis: Result[AnalysisResult, Exception] | None = None
     output_config: OutputConfig = field(default_factory=OutputConfig)
     state_machine_sink: StateMachineSink | None = None
+    summary_lines: list[str] = field(default_factory=list)
 
     @deprecated_property(removed_in="4.0", replacement="show_trace")
     def show_errors_tracebacks(self) -> bool:
         return self.show_trace
+
+    def add_summary_line(self, line: str) -> None:
+        self.summary_lines.append(line)

{schemathesis-3.34.3 → schemathesis-3.35.0}/src/schemathesis/cli/handlers.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 if TYPE_CHECKING:
     from ..runner import events
@@ -8,6 +8,9 @@ if TYPE_CHECKING:
 
 
 class EventHandler:
+    def __init__(self, *args: Any, **params: Any) -> None:
+        pass
+
     def handle_event(self, context: ExecutionContext, event: events.ExecutionEvent) -> None:
         raise NotImplementedError
 

{schemathesis-3.34.3 → schemathesis-3.35.0}/src/schemathesis/cli/output/default.py

@@ -850,6 +850,10 @@ def handle_finished(context: ExecutionContext, event: events.Finished) -> None:
     display_application_logs(context, event)
     display_analysis(context)
     display_statistic(context, event)
+    if context.summary_lines:
+        click.echo()
+        for line in context.summary_lines:
+            click.echo(line)
     click.echo()
     display_summary(event)
 

{schemathesis-3.34.3 → schemathesis-3.35.0}/src/schemathesis/contrib/openapi/fill_missing_examples.py

@@ -18,7 +18,7 @@ def uninstall() -> None:
 
 def before_add_examples(context: HookContext, examples: list[Case]) -> None:
     if not examples and context.operation is not None:
-        from ..._hypothesis import add_single_example
+        from ...generation import add_single_example
 
         strategy = context.operation.as_strategy()
         add_single_example(strategy, examples)

{schemathesis-3.34.3 → schemathesis-3.35.0}/src/schemathesis/experimental/__init__.py

@@ -93,3 +93,10 @@ STATEFUL_ONLY = GLOBAL_EXPERIMENTS.create_experiment(
     description="Run only stateful tests",
     discussion_url="https://github.com/schemathesis/schemathesis/discussions/2262",
 )
+COVERAGE_PHASE = GLOBAL_EXPERIMENTS.create_experiment(
+    name="coverage-phase",
+    verbose_name="Coverage phase",
+    env_var="COVERAGE_PHASE",
+    description="Generate covering test cases",
+    discussion_url="https://github.com/schemathesis/schemathesis/discussions/2418",
+)

{schemathesis-3.34.3 → schemathesis-3.35.0}/src/schemathesis/generation/__init__.py

@@ -2,48 +2,15 @@ from __future__ import annotations
 
 import random
 from dataclasses import dataclass, field
-from enum import Enum
-from typing import TYPE_CHECKING, Iterable, Union
+from typing import TYPE_CHECKING
+
+from ._hypothesis import add_single_example, combine_strategies, get_single_example  # noqa: E402
+from ._methods import DataGenerationMethod, DataGenerationMethodInput  # noqa: E402
 
 if TYPE_CHECKING:
     from hypothesis.strategies import SearchStrategy
 
 
-class DataGenerationMethod(str, Enum):
-    """Defines what data Schemathesis generates for tests."""
-
-    # Generate data, that fits the API schema
-    positive = "positive"
-    # Doesn't fit the API schema
-    negative = "negative"
-
-    @classmethod
-    def default(cls) -> DataGenerationMethod:
-        return cls.positive
-
-    @classmethod
-    def all(cls) -> list[DataGenerationMethod]:
-        return list(DataGenerationMethod)
-
-    def as_short_name(self) -> str:
-        return {
-            DataGenerationMethod.positive: "P",
-            DataGenerationMethod.negative: "N",
-        }[self]
-
-    @property
-    def is_negative(self) -> bool:
-        return self == DataGenerationMethod.negative
-
-    @classmethod
-    def ensure_list(cls, value: DataGenerationMethodInput) -> list[DataGenerationMethod]:
-        if isinstance(value, DataGenerationMethod):
-            return [value]
-        return list(value)
-
-
-DataGenerationMethodInput = Union[DataGenerationMethod, Iterable[DataGenerationMethod]]
-
 DEFAULT_DATA_GENERATION_METHODS = (DataGenerationMethod.default(),)
 
 

schemathesis-3.35.0/src/schemathesis/generation/_hypothesis.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+from functools import lru_cache, reduce
+from operator import or_
+from typing import TYPE_CHECKING, TypeVar
+
+if TYPE_CHECKING:
+    from hypothesis import settings
+    from hypothesis import strategies as st
+
+
+@lru_cache()
+def default_settings() -> settings:
+    from hypothesis import HealthCheck, Phase, Verbosity, settings
+
+    return settings(
+        database=None,
+        max_examples=1,
+        deadline=None,
+        verbosity=Verbosity.quiet,
+        phases=(Phase.generate,),
+        suppress_health_check=list(HealthCheck),
+    )
+
+
+T = TypeVar("T")
+
+
+def get_single_example(strategy: st.SearchStrategy[T]) -> T:  # type: ignore[type-var]
+    examples: list[T] = []
+    add_single_example(strategy, examples)
+    return examples[0]
+
+
+def add_single_example(strategy: st.SearchStrategy[T], examples: list[T]) -> None:
+    from hypothesis import given
+
+    @given(strategy)  # type: ignore
+    @default_settings()  # type: ignore
+    def example_generating_inner_function(ex: T) -> None:
+        examples.append(ex)
+
+    example_generating_inner_function()
+
+
+def combine_strategies(strategies: list[st.SearchStrategy] | tuple[st.SearchStrategy]) -> st.SearchStrategy:
+    """Combine a list of strategies into a single one.
+
+    If the input is `[a, b, c]`, then the result is equivalent to `a | b | c`.
+    """
+    return reduce(or_, strategies[1:], strategies[0])

schemathesis-3.35.0/src/schemathesis/generation/_methods.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from enum import Enum
+from typing import Iterable, Union
+
+
+class DataGenerationMethod(str, Enum):
+    """Defines what data Schemathesis generates for tests."""
+
+    # Generate data, that fits the API schema
+    positive = "positive"
+    # Doesn't fit the API schema
+    negative = "negative"
+
+    @classmethod
+    def default(cls) -> DataGenerationMethod:
+        return cls.positive
+
+    @classmethod
+    def all(cls) -> list[DataGenerationMethod]:
+        return list(DataGenerationMethod)
+
+    def as_short_name(self) -> str:
+        return {
+            DataGenerationMethod.positive: "P",
+            DataGenerationMethod.negative: "N",
+        }[self]
+
+    @property
+    def is_negative(self) -> bool:
+        return self == DataGenerationMethod.negative
+
+    @classmethod
+    def ensure_list(cls, value: DataGenerationMethodInput) -> list[DataGenerationMethod]:
+        if isinstance(value, DataGenerationMethod):
+            return [value]
+        return list(value)
+
+
+DataGenerationMethodInput = Union[DataGenerationMethod, Iterable[DataGenerationMethod]]