schemathesis 3.35.1__py3-none-any.whl → 3.35.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- schemathesis/_hypothesis.py +2 -4
- schemathesis/cli/__init__.py +352 -357
- schemathesis/cli/context.py +7 -3
- schemathesis/cli/output/default.py +13 -2
- schemathesis/generation/coverage.py +24 -8
- schemathesis/runner/events.py +3 -1
- schemathesis/schemas.py +3 -9
- schemathesis/specs/graphql/loaders.py +2 -1
- schemathesis/specs/openapi/loaders.py +3 -1
- schemathesis/specs/openapi/parameters.py +2 -0
- schemathesis/stateful/context.py +3 -0
- schemathesis/stateful/runner.py +8 -1
- schemathesis/types.py +8 -0
- {schemathesis-3.35.1.dist-info → schemathesis-3.35.2.dist-info}/METADATA +69 -168
- {schemathesis-3.35.1.dist-info → schemathesis-3.35.2.dist-info}/RECORD +18 -18
- {schemathesis-3.35.1.dist-info → schemathesis-3.35.2.dist-info}/WHEEL +0 -0
- {schemathesis-3.35.1.dist-info → schemathesis-3.35.2.dist-info}/entry_points.txt +0 -0
- {schemathesis-3.35.1.dist-info → schemathesis-3.35.2.dist-info}/licenses/LICENSE +0 -0
schemathesis/cli/context.py
CHANGED
|
@@ -4,7 +4,7 @@ import os
|
|
|
4
4
|
import shutil
|
|
5
5
|
from dataclasses import dataclass, field
|
|
6
6
|
from queue import Queue
|
|
7
|
-
from typing import TYPE_CHECKING
|
|
7
|
+
from typing import TYPE_CHECKING, Generator
|
|
8
8
|
|
|
9
9
|
from ..code_samples import CodeSampleStyle
|
|
10
10
|
from ..internal.deprecation import deprecated_property
|
|
@@ -60,11 +60,15 @@ class ExecutionContext:
|
|
|
60
60
|
analysis: Result[AnalysisResult, Exception] | None = None
|
|
61
61
|
output_config: OutputConfig = field(default_factory=OutputConfig)
|
|
62
62
|
state_machine_sink: StateMachineSink | None = None
|
|
63
|
-
|
|
63
|
+
initialization_lines: list[str | Generator[str, None, None]] = field(default_factory=list)
|
|
64
|
+
summary_lines: list[str | Generator[str, None, None]] = field(default_factory=list)
|
|
64
65
|
|
|
65
66
|
@deprecated_property(removed_in="4.0", replacement="show_trace")
|
|
66
67
|
def show_errors_tracebacks(self) -> bool:
|
|
67
68
|
return self.show_trace
|
|
68
69
|
|
|
69
|
-
def
|
|
70
|
+
def add_initialization_line(self, line: str | Generator[str, None, None]) -> None:
|
|
71
|
+
self.initialization_lines.append(line)
|
|
72
|
+
|
|
73
|
+
def add_summary_line(self, line: str | Generator[str, None, None]) -> None:
|
|
70
74
|
self.summary_lines.append(line)
|
|
@@ -7,6 +7,7 @@ import textwrap
|
|
|
7
7
|
import time
|
|
8
8
|
from importlib import metadata
|
|
9
9
|
from queue import Queue
|
|
10
|
+
from types import GeneratorType
|
|
10
11
|
from typing import TYPE_CHECKING, Any, Generator, Literal, cast
|
|
11
12
|
|
|
12
13
|
import click
|
|
@@ -776,6 +777,8 @@ def handle_initialized(context: ExecutionContext, event: events.Initialized) ->
|
|
|
776
777
|
click.secho(f"Collected API links: {links_count}", bold=True)
|
|
777
778
|
if isinstance(context.report, ServiceReportContext):
|
|
778
779
|
click.secho("Report to Schemathesis.io: ENABLED", bold=True)
|
|
780
|
+
if context.initialization_lines:
|
|
781
|
+
_print_lines(context.initialization_lines)
|
|
779
782
|
|
|
780
783
|
|
|
781
784
|
def handle_before_probing(context: ExecutionContext, event: events.BeforeProbing) -> None:
|
|
@@ -852,12 +855,20 @@ def handle_finished(context: ExecutionContext, event: events.Finished) -> None:
|
|
|
852
855
|
display_statistic(context, event)
|
|
853
856
|
if context.summary_lines:
|
|
854
857
|
click.echo()
|
|
855
|
-
|
|
856
|
-
click.echo(line)
|
|
858
|
+
_print_lines(context.summary_lines)
|
|
857
859
|
click.echo()
|
|
858
860
|
display_summary(event)
|
|
859
861
|
|
|
860
862
|
|
|
863
|
+
def _print_lines(lines: list[str | Generator[str, None, None]]) -> None:
|
|
864
|
+
for entry in lines:
|
|
865
|
+
if isinstance(entry, str):
|
|
866
|
+
click.echo(entry)
|
|
867
|
+
elif isinstance(entry, GeneratorType):
|
|
868
|
+
for line in entry:
|
|
869
|
+
click.echo(line)
|
|
870
|
+
|
|
871
|
+
|
|
861
872
|
def handle_interrupted(context: ExecutionContext, event: events.Interrupted) -> None:
|
|
862
873
|
click.echo()
|
|
863
874
|
_handle_interrupted(context)
|
|
@@ -234,6 +234,15 @@ def cover_schema_iter(ctx: CoverageContext, schema: dict | bool) -> Generator[Ge
|
|
|
234
234
|
yield from cover_schema_iter(nctx, sub_schema)
|
|
235
235
|
|
|
236
236
|
|
|
237
|
+
def _get_properties(schema: dict | bool) -> dict | bool:
|
|
238
|
+
if isinstance(schema, dict):
|
|
239
|
+
if "example" in schema:
|
|
240
|
+
return {"const": schema["example"]}
|
|
241
|
+
if schema.get("type") == "object":
|
|
242
|
+
return _get_template_schema(schema, "object")
|
|
243
|
+
return schema
|
|
244
|
+
|
|
245
|
+
|
|
237
246
|
def _get_template_schema(schema: dict, ty: str) -> dict:
|
|
238
247
|
if ty == "object":
|
|
239
248
|
properties = schema.get("properties")
|
|
@@ -242,10 +251,7 @@ def _get_template_schema(schema: dict, ty: str) -> dict:
|
|
|
242
251
|
**schema,
|
|
243
252
|
"required": list(properties),
|
|
244
253
|
"type": ty,
|
|
245
|
-
"properties": {
|
|
246
|
-
k: _get_template_schema(v, "object") if isinstance(v, dict) and v.get("type") == "object" else v
|
|
247
|
-
for k, v in properties.items()
|
|
248
|
-
},
|
|
254
|
+
"properties": {k: _get_properties(v) for k, v in properties.items()},
|
|
249
255
|
}
|
|
250
256
|
return {**schema, "type": ty}
|
|
251
257
|
|
|
@@ -256,7 +262,9 @@ def _positive_string(ctx: CoverageContext, schema: dict) -> Generator[GeneratedV
|
|
|
256
262
|
min_length = schema.get("minLength")
|
|
257
263
|
max_length = schema.get("maxLength")
|
|
258
264
|
|
|
259
|
-
if
|
|
265
|
+
if "example" in schema:
|
|
266
|
+
yield PositiveValue(schema["example"])
|
|
267
|
+
elif not min_length and not max_length:
|
|
260
268
|
# Default positive value
|
|
261
269
|
yield PositiveValue(ctx.generate_from_schema(schema))
|
|
262
270
|
|
|
@@ -311,7 +319,9 @@ def _positive_number(ctx: CoverageContext, schema: dict) -> Generator[GeneratedV
|
|
|
311
319
|
maximum = exclusive_maximum - 1
|
|
312
320
|
multiple_of = schema.get("multipleOf")
|
|
313
321
|
|
|
314
|
-
if
|
|
322
|
+
if "example" in schema:
|
|
323
|
+
yield PositiveValue(schema["example"])
|
|
324
|
+
elif not minimum and not maximum:
|
|
315
325
|
# Default positive value
|
|
316
326
|
yield PositiveValue(ctx.generate_from_schema(schema))
|
|
317
327
|
|
|
@@ -357,7 +367,10 @@ def _positive_number(ctx: CoverageContext, schema: dict) -> Generator[GeneratedV
|
|
|
357
367
|
|
|
358
368
|
def _positive_array(ctx: CoverageContext, schema: dict, template: list) -> Generator[GeneratedValue, None, None]:
|
|
359
369
|
seen = set()
|
|
360
|
-
|
|
370
|
+
if "example" in schema:
|
|
371
|
+
yield PositiveValue(schema["example"])
|
|
372
|
+
else:
|
|
373
|
+
yield PositiveValue(template)
|
|
361
374
|
seen.add(len(template))
|
|
362
375
|
|
|
363
376
|
# Boundary and near-boundary sizes
|
|
@@ -390,7 +403,10 @@ def _positive_array(ctx: CoverageContext, schema: dict, template: list) -> Gener
|
|
|
390
403
|
|
|
391
404
|
|
|
392
405
|
def _positive_object(ctx: CoverageContext, schema: dict, template: dict) -> Generator[GeneratedValue, None, None]:
|
|
393
|
-
|
|
406
|
+
if "example" in schema:
|
|
407
|
+
yield PositiveValue(schema["example"])
|
|
408
|
+
else:
|
|
409
|
+
yield PositiveValue(template)
|
|
394
410
|
# Only required properties
|
|
395
411
|
properties = schema.get("properties", {})
|
|
396
412
|
if set(properties) != set(schema.get("required", {})):
|
schemathesis/runner/events.py
CHANGED
|
@@ -14,7 +14,7 @@ from .serialization import SerializedError, SerializedTestResult
|
|
|
14
14
|
|
|
15
15
|
if TYPE_CHECKING:
|
|
16
16
|
from ..models import APIOperation, Status, TestResult, TestResultSet
|
|
17
|
-
from ..schemas import BaseSchema
|
|
17
|
+
from ..schemas import BaseSchema, Specification
|
|
18
18
|
from ..service.models import AnalysisResult
|
|
19
19
|
from ..stateful import events
|
|
20
20
|
from . import probes
|
|
@@ -39,6 +39,7 @@ class Initialized(ExecutionEvent):
|
|
|
39
39
|
"""Runner is initialized, settings are prepared, requests session is ready."""
|
|
40
40
|
|
|
41
41
|
schema: dict[str, Any]
|
|
42
|
+
specification: Specification
|
|
42
43
|
# Total number of operations in the schema
|
|
43
44
|
operations_count: int | None
|
|
44
45
|
# Total number of links in the schema
|
|
@@ -71,6 +72,7 @@ class Initialized(ExecutionEvent):
|
|
|
71
72
|
"""Computes all needed data from a schema instance."""
|
|
72
73
|
return cls(
|
|
73
74
|
schema=schema.raw_schema,
|
|
75
|
+
specification=schema.specification,
|
|
74
76
|
operations_count=schema.operations_count if count_operations else None,
|
|
75
77
|
links_count=schema.links_count if count_links else None,
|
|
76
78
|
location=schema.location,
|
schemathesis/schemas.py
CHANGED
|
@@ -1,12 +1,3 @@
|
|
|
1
|
-
"""Schema objects provide a convenient interface to raw schemas.
|
|
2
|
-
|
|
3
|
-
Their responsibilities:
|
|
4
|
-
- Provide a unified way to work with different types of schemas
|
|
5
|
-
- Give all paths / methods combinations that are available directly from the schema;
|
|
6
|
-
|
|
7
|
-
They give only static definitions of paths.
|
|
8
|
-
"""
|
|
9
|
-
|
|
10
1
|
from __future__ import annotations
|
|
11
2
|
|
|
12
3
|
from collections.abc import Mapping
|
|
@@ -69,6 +60,7 @@ from .types import (
|
|
|
69
60
|
NotSet,
|
|
70
61
|
PathParameters,
|
|
71
62
|
Query,
|
|
63
|
+
Specification,
|
|
72
64
|
)
|
|
73
65
|
from .utils import PARAMETRIZE_MARKER, GivenInput, given_proxy
|
|
74
66
|
|
|
@@ -89,6 +81,7 @@ def get_full_path(base_path: str, path: str) -> str:
|
|
|
89
81
|
class BaseSchema(Mapping):
|
|
90
82
|
raw_schema: dict[str, Any]
|
|
91
83
|
transport: Transport
|
|
84
|
+
specification: Specification
|
|
92
85
|
location: str | None = None
|
|
93
86
|
base_url: str | None = None
|
|
94
87
|
filter_set: FilterSet = field(default_factory=FilterSet)
|
|
@@ -408,6 +401,7 @@ class BaseSchema(Mapping):
|
|
|
408
401
|
|
|
409
402
|
return self.__class__(
|
|
410
403
|
self.raw_schema,
|
|
404
|
+
specification=self.specification,
|
|
411
405
|
location=self.location,
|
|
412
406
|
base_url=base_url, # type: ignore
|
|
413
407
|
app=app,
|
|
@@ -21,7 +21,7 @@ from ...internal.validation import require_relative_url
|
|
|
21
21
|
from ...loaders import load_schema_from_url
|
|
22
22
|
from ...throttling import build_limiter
|
|
23
23
|
from ...transports.headers import setup_default_headers
|
|
24
|
-
from ...types import PathLike
|
|
24
|
+
from ...types import PathLike, Specification
|
|
25
25
|
|
|
26
26
|
if TYPE_CHECKING:
|
|
27
27
|
from graphql import DocumentNode
|
|
@@ -254,6 +254,7 @@ def from_dict(
|
|
|
254
254
|
rate_limiter = build_limiter(rate_limit)
|
|
255
255
|
instance = GraphQLSchema(
|
|
256
256
|
raw_schema,
|
|
257
|
+
specification=Specification.GRAPHQL,
|
|
257
258
|
location=location,
|
|
258
259
|
base_url=base_url,
|
|
259
260
|
app=app,
|
|
@@ -26,7 +26,7 @@ from ...loaders import load_schema_from_url, load_yaml
|
|
|
26
26
|
from ...throttling import build_limiter
|
|
27
27
|
from ...transports.content_types import is_json_media_type, is_yaml_media_type
|
|
28
28
|
from ...transports.headers import setup_default_headers
|
|
29
|
-
from ...types import Filter, NotSet, PathLike
|
|
29
|
+
from ...types import Filter, NotSet, PathLike, Specification
|
|
30
30
|
from . import definitions, validation
|
|
31
31
|
|
|
32
32
|
if TYPE_CHECKING:
|
|
@@ -349,6 +349,7 @@ def from_dict(
|
|
|
349
349
|
_maybe_validate_schema(raw_schema, definitions.SWAGGER_20_VALIDATOR, validate_schema)
|
|
350
350
|
instance = SwaggerV20(
|
|
351
351
|
raw_schema,
|
|
352
|
+
specification=Specification.OPENAPI,
|
|
352
353
|
app=app,
|
|
353
354
|
base_url=base_url,
|
|
354
355
|
filter_set=filter_set,
|
|
@@ -388,6 +389,7 @@ def from_dict(
|
|
|
388
389
|
_maybe_validate_schema(raw_schema, validator, validate_schema)
|
|
389
390
|
instance = OpenApi30(
|
|
390
391
|
raw_schema,
|
|
392
|
+
specification=Specification.OPENAPI,
|
|
391
393
|
app=app,
|
|
392
394
|
base_url=base_url,
|
|
393
395
|
filter_set=filter_set,
|
|
@@ -117,6 +117,7 @@ class OpenAPI20Parameter(OpenAPIParameter):
|
|
|
117
117
|
"uniqueItems",
|
|
118
118
|
"enum",
|
|
119
119
|
"multipleOf",
|
|
120
|
+
"example",
|
|
120
121
|
)
|
|
121
122
|
|
|
122
123
|
@property
|
|
@@ -163,6 +164,7 @@ class OpenAPI30Parameter(OpenAPIParameter):
|
|
|
163
164
|
"properties",
|
|
164
165
|
"additionalProperties",
|
|
165
166
|
"format",
|
|
167
|
+
"example",
|
|
166
168
|
)
|
|
167
169
|
|
|
168
170
|
@property
|
schemathesis/stateful/context.py
CHANGED
|
@@ -48,6 +48,8 @@ class RunnerContext:
|
|
|
48
48
|
current_response: GenericResponse | None = None
|
|
49
49
|
# Total number of failures
|
|
50
50
|
failures_count: int = 0
|
|
51
|
+
# The total number of completed test scenario
|
|
52
|
+
completed_scenarios: int = 0
|
|
51
53
|
# Metrics collector for targeted testing
|
|
52
54
|
metric_collector: TargetMetricCollector = field(default_factory=lambda: TargetMetricCollector(targets=[]))
|
|
53
55
|
|
|
@@ -64,6 +66,7 @@ class RunnerContext:
|
|
|
64
66
|
return events.ScenarioStatus.REJECTED
|
|
65
67
|
|
|
66
68
|
def reset_scenario(self) -> None:
|
|
69
|
+
self.completed_scenarios += 1
|
|
67
70
|
self.current_step_status = None
|
|
68
71
|
self.current_response = None
|
|
69
72
|
|
schemathesis/stateful/runner.py
CHANGED
|
@@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, Any, Generator, Iterator, Type
|
|
|
9
9
|
import hypothesis
|
|
10
10
|
import requests
|
|
11
11
|
from hypothesis.control import current_build_context
|
|
12
|
-
from hypothesis.errors import Flaky
|
|
12
|
+
from hypothesis.errors import Flaky, Unsatisfiable
|
|
13
13
|
from hypothesis.stateful import Rule
|
|
14
14
|
|
|
15
15
|
from ..exceptions import CheckFailed
|
|
@@ -265,6 +265,13 @@ def _execute_state_machine_loop(
|
|
|
265
265
|
ctx.mark_current_suite_as_seen_in_run()
|
|
266
266
|
continue
|
|
267
267
|
except Exception as exc:
|
|
268
|
+
if isinstance(exc, Unsatisfiable) and ctx.completed_scenarios > 0:
|
|
269
|
+
# Sometimes Hypothesis randomly gives up on generating some complex cases. However, if we know that
|
|
270
|
+
# values are possible to generate based on the previous observations, we retry the generation
|
|
271
|
+
if ctx.completed_scenarios >= config.hypothesis_settings.max_examples:
|
|
272
|
+
# Avoid infinite restarts
|
|
273
|
+
break
|
|
274
|
+
continue
|
|
268
275
|
# Any other exception is an inner error and the test run should be stopped
|
|
269
276
|
suite_status = events.SuiteStatus.ERROR
|
|
270
277
|
event_queue.put(events.Errored(exception=exc))
|
schemathesis/types.py
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import enum
|
|
1
2
|
from pathlib import Path
|
|
2
3
|
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Set, Tuple, Union
|
|
3
4
|
|
|
@@ -34,3 +35,10 @@ Hook = Union[
|
|
|
34
35
|
RawAuth = Tuple[str, str]
|
|
35
36
|
# Generic test with any arguments and no return
|
|
36
37
|
GenericTest = Callable[..., None]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class Specification(str, enum.Enum):
|
|
41
|
+
"""Specification of the given schema."""
|
|
42
|
+
|
|
43
|
+
OPENAPI = "openapi"
|
|
44
|
+
GRAPHQL = "graphql"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: schemathesis
|
|
3
|
-
Version: 3.35.
|
|
3
|
+
Version: 3.35.2
|
|
4
4
|
Summary: Property-based testing framework for Open API and GraphQL based apps
|
|
5
5
|
Project-URL: Documentation, https://schemathesis.readthedocs.io/en/stable/
|
|
6
6
|
Project-URL: Changelog, https://schemathesis.readthedocs.io/en/stable/changelog.html
|
|
@@ -113,9 +113,6 @@ Description-Content-Type: text/markdown
|
|
|
113
113
|
<a href="https://discord.gg/R9ASRAmHnA" target="_blank">
|
|
114
114
|
<img src="https://img.shields.io/discord/938139740912369755" alt="Discord">
|
|
115
115
|
</a>
|
|
116
|
-
<a href="[https://discord.gg/R9ASRAmHnA](https://schemathesis.readthedocs.io/en/stable/)" target="_blank">
|
|
117
|
-
<img src="https://readthedocs.org/projects/schemathesis/badge/?version=stable" alt="Documentation">
|
|
118
|
-
</a>
|
|
119
116
|
<a href="https://opensource.org/licenses/MIT" target="_blank">
|
|
120
117
|
<img src="https://img.shields.io/pypi/l/schemathesis.svg" alt="License">
|
|
121
118
|
</a>
|
|
@@ -123,78 +120,73 @@ Description-Content-Type: text/markdown
|
|
|
123
120
|
|
|
124
121
|
## Schemathesis
|
|
125
122
|
|
|
126
|
-
Schemathesis is
|
|
127
|
-
|
|
128
|
-
🎯 **Catch Hard-to-Find Bugs**
|
|
129
|
-
|
|
130
|
-
- Uncover hidden crashes and edge cases that manual testing might miss
|
|
131
|
-
- Identify spec violations and ensure your API adheres to its defined contract
|
|
132
|
-
|
|
133
|
-
⚡ **Accelerate Testing Cycles**
|
|
123
|
+
Schemathesis is an API testing tool that automatically finds crashes and validates spec compliance.
|
|
134
124
|
|
|
135
|
-
|
|
136
|
-
|
|
125
|
+
<p align="center">
|
|
126
|
+
<img src="https://raw.githubusercontent.com/schemathesis/schemathesis/master/img/demo.gif" alt="Schemathesis Demo"/>
|
|
127
|
+
</p>
|
|
137
128
|
|
|
138
|
-
|
|
129
|
+
<p align="center">
|
|
130
|
+
<i>Finding server crashes in the Demo API.</i>
|
|
131
|
+
</p>
|
|
139
132
|
|
|
140
|
-
|
|
141
|
-
- Easily integrate into your existing testing pipeline and CI/CD workflows
|
|
133
|
+
### Highlights
|
|
142
134
|
|
|
143
|
-
|
|
135
|
+
🎯 **Catches Hard-to-Find Bugs**
|
|
144
136
|
|
|
145
|
-
-
|
|
146
|
-
-
|
|
137
|
+
- Uncover hidden crashes and edge cases that manual testing might miss
|
|
138
|
+
- Identify spec violations and ensure your API adheres to its contract
|
|
147
139
|
|
|
148
|
-
|
|
140
|
+
⚡ **Accelerates Testing Cycles**
|
|
149
141
|
|
|
150
|
-
-
|
|
151
|
-
-
|
|
142
|
+
- Automatically generate a wide range of test cases based on your API schema
|
|
143
|
+
- Save time by reducing the need for manual test case creation
|
|
152
144
|
|
|
153
|
-
|
|
145
|
+
🧩 **Integrates Seamlessly**
|
|
154
146
|
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
</p>
|
|
147
|
+
- Works with popular API formats such as OpenAPI, GraphQL.
|
|
148
|
+
- Easily integrate into your existing CI/CD workflows.
|
|
158
149
|
|
|
159
|
-
|
|
150
|
+
🔧 **Customizable and Extendable**
|
|
160
151
|
|
|
161
|
-
|
|
152
|
+
- Tune the testing process using Python extensions.
|
|
153
|
+
- Adjust the testing flow to suit your needs with rich configuration options.
|
|
162
154
|
|
|
163
|
-
|
|
155
|
+
🐞 **Simplifies Debugging**
|
|
164
156
|
|
|
165
|
-
|
|
157
|
+
- Get detailed reports to identify and fix issues quickly.
|
|
158
|
+
- Reproduce failing test cases with cURL commands.
|
|
166
159
|
|
|
167
|
-
|
|
160
|
+
🔬 **Proven by Research**
|
|
168
161
|
|
|
169
|
-
|
|
162
|
+
- Validated through academic studies on API testing automation
|
|
163
|
+
- Featured in [ICSE 2022 paper](https://ieeexplore.ieee.org/document/9793781) on semantics-aware fuzzing
|
|
164
|
+
- Recognized in [ACM survey](https://dl.acm.org/doi/10.1145/3617175) as state-of-the-art RESTful API testing tool
|
|
170
165
|
|
|
171
|
-
|
|
166
|
+
## Installation
|
|
172
167
|
|
|
173
|
-
|
|
168
|
+
Use Schemathesis via Docker, or install it from [PyPI](https://pypi.org/project/schemathesis/)
|
|
174
169
|
|
|
175
|
-
|
|
176
|
-
|
|
170
|
+
```console
|
|
171
|
+
# Via Docker.
|
|
172
|
+
$ docker pull schemathesis/schemathesis:stable
|
|
177
173
|
|
|
178
|
-
|
|
179
|
-
|
|
174
|
+
# With pip.
|
|
175
|
+
$ pip install schemathesis
|
|
180
176
|
```
|
|
181
177
|
|
|
182
|
-
|
|
178
|
+
## Getting Started
|
|
183
179
|
|
|
184
|
-
|
|
185
|
-
2. Run tests
|
|
180
|
+
Schemathesis works as a standalone CLI:
|
|
186
181
|
|
|
187
|
-
```
|
|
182
|
+
```console
|
|
188
183
|
docker run schemathesis/schemathesis:stable
|
|
189
184
|
run --checks all https://example.schemathesis.io/openapi.json
|
|
185
|
+
# Or when installed with pip
|
|
186
|
+
schemathesis run --checks all https://example.schemathesis.io/openapi.json
|
|
190
187
|
```
|
|
191
188
|
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
For more control and customization, integrate Schemathesis into your Python codebase.
|
|
195
|
-
|
|
196
|
-
1. Install via pip: `python -m pip install schemathesis`
|
|
197
|
-
2. Add to your tests:
|
|
189
|
+
Or a Python library:
|
|
198
190
|
|
|
199
191
|
```python
|
|
200
192
|
import schemathesis
|
|
@@ -207,19 +199,13 @@ def test_api(case):
|
|
|
207
199
|
case.call_and_validate()
|
|
208
200
|
```
|
|
209
201
|
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
### :octocat: GitHub Integration
|
|
202
|
+
See a complete working example project in the [/example](https://github.com/schemathesis/schemathesis/tree/master/example) directory.
|
|
213
203
|
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
Run Schemathesis tests as a part of your CI/CD pipeline.
|
|
217
|
-
|
|
218
|
-
Add this YAML configuration to your GitHub Actions:
|
|
204
|
+
Schemathesis can be easily integrated into your CI/CD pipeline using GitHub Actions. Add this block to your GitHub Actions to run Schemathesis against your API:
|
|
219
205
|
|
|
220
206
|
```yaml
|
|
221
207
|
api-tests:
|
|
222
|
-
runs-on: ubuntu-
|
|
208
|
+
runs-on: ubuntu-latest
|
|
223
209
|
steps:
|
|
224
210
|
- uses: schemathesis/action@v1
|
|
225
211
|
with:
|
|
@@ -228,42 +214,30 @@ api-tests:
|
|
|
228
214
|
token: ${{ secrets.SCHEMATHESIS_TOKEN }}
|
|
229
215
|
```
|
|
230
216
|
|
|
231
|
-
For more details, check out our [GitHub Action](https://github.com/schemathesis/action) repository.
|
|
232
|
-
|
|
233
|
-
> 💡 See our [GitHub Tutorial](https://docs.schemathesis.io/tutorials/github) for a step-by-step guidance.
|
|
234
|
-
|
|
235
|
-
**GitHub App**
|
|
236
|
-
|
|
237
|
-
Receive automatic comments in your pull requests and updates on GitHub checks status. Requires usage of our SaaS platform.
|
|
238
|
-
|
|
239
|
-
1. Install the [GitHub app](https://github.com/apps/schemathesis).
|
|
240
|
-
2. Enable in your repository settings.
|
|
241
|
-
|
|
242
|
-
### Software as a Service
|
|
217
|
+
For more details, check out our [GitHub Action](https://github.com/schemathesis/action) repository or see our [GitHub Tutorial](https://docs.schemathesis.io/tutorials/github).
|
|
243
218
|
|
|
244
|
-
|
|
219
|
+
For test reports in your pull requests, install the [GitHub app](https://github.com/apps/schemathesis):
|
|
245
220
|
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
## How it works
|
|
249
|
-
|
|
250
|
-
Here’s a simplified overview of how Schemathesis operates:
|
|
251
|
-
|
|
252
|
-
1. **Test Generation**: Using the API schema to create a test generator that you can fine-tune to your testing requirements.
|
|
253
|
-
2. **Execution and Adaptation**: Sending tests to the API and adapting through statistical models and heuristics to optimize subsequent cases based on responses.
|
|
254
|
-
3. **Analysis and Minimization**: Checking responses to identify issues. Minimizing means simplifying failing test cases for easier debugging.
|
|
255
|
-
4. **Stateful Testing**: Running multistep tests to assess API operations in both isolated and integrated scenarios.
|
|
256
|
-
5. **Reporting**: Generating detailed reports with insights and cURL commands for easy issue reproduction.
|
|
221
|
+

|
|
257
222
|
|
|
258
|
-
###
|
|
223
|
+
### Schemathesis.io
|
|
259
224
|
|
|
260
|
-
|
|
225
|
+
Schemathesis CLI integrates with Schemathesis.io to enhance bug detection by optimizing test case generation. It also provides a user-friendly UI for viewing and analyzing test results. For a quick setup all-in-one solution, we offer a [free tier](https://schemathesis.io/#pricing).
|
|
261
226
|
|
|
262
|
-
|
|
227
|
+
## Who's Using Schemathesis?
|
|
263
228
|
|
|
264
|
-
|
|
229
|
+
Schemathesis is used by a number of projects and companies, including direct usage or integration into other tools:
|
|
265
230
|
|
|
266
|
-
|
|
231
|
+
- Abstract Machines ([Magistrala](https://github.com/absmach/magistrala))
|
|
232
|
+
- Bundesstelle für Open Data ([smard-api](https://github.com/bundesAPI/smard-api))
|
|
233
|
+
- [CheckMK](https://github.com/Checkmk/checkmk)
|
|
234
|
+
- Chronosphere.io ([Calyptia](https://github.com/chronosphereio/calyptia-api))
|
|
235
|
+
- HXSecurity ([DongTai](https://github.com/HXSecurity/DongTai))
|
|
236
|
+
- Netflix ([Dispatch](https://github.com/Netflix/dispatch))
|
|
237
|
+
- [Pixie](https://github.com/pixie-io/pixie)
|
|
238
|
+
- [Qdrant](https://github.com/qdrant/qdrant)
|
|
239
|
+
- Spotify ([Backstage](https://github.com/backstage/backstage))
|
|
240
|
+
- WordPress ([OpenVerse](https://github.com/WordPress/openverse))
|
|
267
241
|
|
|
268
242
|
## Testimonials
|
|
269
243
|
|
|
@@ -299,20 +273,18 @@ Explore the full paper at https://ieeexplore.ieee.org/document/9793781 or pre-pr
|
|
|
299
273
|
|
|
300
274
|
## Contributing
|
|
301
275
|
|
|
302
|
-
We welcome contributions in code and are especially interested in learning about your use cases.
|
|
303
|
-
Understanding how you use Schemathesis helps us extend its capabilities to better meet your needs.
|
|
276
|
+
We welcome contributions in code and are especially interested in learning about your use cases. Your input is essential for improving Schemathesis and directly influences future updates.
|
|
304
277
|
|
|
305
|
-
|
|
306
|
-
For more details on how to contribute, see our [contributing guidelines](https://github.com/schemathesis/schemathesis/blob/master/CONTRIBUTING.rst).
|
|
278
|
+
### How to Contribute
|
|
307
279
|
|
|
308
|
-
|
|
280
|
+
1. Discuss ideas and questions through [GitHub issues](https://github.com/schemathesis/schemathesis/issues) or on our [Discord channel](https://discord.gg/R9ASRAmHnA).
|
|
281
|
+
2. For code contributions, see our [contributing guidelines](https://github.com/schemathesis/schemathesis/blob/master/CONTRIBUTING.rst).
|
|
282
|
+
3. Share your experience and thoughts using [this feedback form](https://forms.gle/kJ4hSxc1Yp6Ga96t5).
|
|
309
283
|
|
|
310
|
-
|
|
311
|
-
By sharing your thoughts, you help us develop features that meet your needs and expedite bug fixes.
|
|
284
|
+
### Why Your Input Matters
|
|
312
285
|
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
3. **Data Privacy**: We value your privacy. All data is kept confidential and may be used in anonymized form to improve our test suite and documentation.
|
|
286
|
+
- Enables us to develop useful features and fix bugs faster
|
|
287
|
+
- Improves our test suite and documentation
|
|
316
288
|
|
|
317
289
|
Thank you for contributing to making Schemathesis better! 👍
|
|
318
290
|
|
|
@@ -330,77 +302,6 @@ To discuss a custom support arrangement that best suits your organization, pleas
|
|
|
330
302
|
|
|
331
303
|
Schemathesis is built on top of <a href="https://hypothesis.works/" target="_blank">Hypothesis</a>, a powerful property-based testing library for Python.
|
|
332
304
|
|
|
333
|
-
## Who's Using Schemathesis?
|
|
334
|
-
|
|
335
|
-
Schemathesis is used by a number of projects and companies, including direct usage or integration into other tools:
|
|
336
|
-
|
|
337
|
-
- Abstract Machines ([Magistrala](https://github.com/absmach/magistrala))
|
|
338
|
-
- Bundesstelle für Open Data ([smard-api](https://github.com/bundesAPI/smard-api))
|
|
339
|
-
- [CheckMK](https://github.com/Checkmk/checkmk)
|
|
340
|
-
- [Chronosphere.io](https://github.com/chronosphereio/calyptia-api)
|
|
341
|
-
- HXSecurity ([DongTai](https://github.com/HXSecurity/DongTai))
|
|
342
|
-
- Netflix ([Dispatch](https://github.com/Netflix/dispatch))
|
|
343
|
-
- [Pixie](https://github.com/pixie-io/pixie)
|
|
344
|
-
- [Qdrant](https://github.com/qdrant/qdrant)
|
|
345
|
-
- Spotify ([Backstage](https://github.com/backstage/backstage))
|
|
346
|
-
- WordPress ([OpenVerse](https://github.com/WordPress/openverse))
|
|
347
|
-
|
|
348
|
-
## Additional content
|
|
349
|
-
|
|
350
|
-
### Papers
|
|
351
|
-
|
|
352
|
-
- [Deriving Semantics-Aware Fuzzers from Web API Schemas](https://ieeexplore.ieee.org/document/9793781) by **@Zac-HD** and **@Stranger6667**
|
|
353
|
-
- **Description**: Explores the automation of API testing through semantics-aware fuzzing. Presented at ICSE 2022.
|
|
354
|
-
- **Date**: 20 Dec 2021
|
|
355
|
-
|
|
356
|
-
### Articles
|
|
357
|
-
|
|
358
|
-
- [Boost Your FastAPI Reliability with Schemathesis Automated Testing](https://medium.com/@jeremy3/boost-your-fastapi-reliability-with-schemathesis-automated-testing-e8b70ff704f6) by **Jeremy**
|
|
359
|
-
- **Description**: Showcases how easy Schemathesis covers API testing in FastAPI projects.
|
|
360
|
-
- **Date**: 17 Jul 2024
|
|
361
|
-
|
|
362
|
-
- [Implementing Schemathesis at PayLead](https://medium.com/paylead/implementing-schemathesis-at-paylead-a469a5d43626) by **Jérémy Pelletier** at **PayLead**
|
|
363
|
-
- **Description**: In-depth walkthrough including custom hooks, stateful testing and CI/CD integration.
|
|
364
|
-
- **Date**: 29 May 2024
|
|
365
|
-
|
|
366
|
-
- [Auto-Generating & Validating OpenAPI Docs in Rust: A Streamlined Approach with Utoipa and Schemathesis](https://identeco.de/en/blog/generating_and_validating_openapi_docs_in_rust/) by **identeco**
|
|
367
|
-
- **Description**: Demonstrates OpenAPI doc generation with Utoipa and validating it with Schemathesis.
|
|
368
|
-
- **Date**: 01 Jun 2023
|
|
369
|
-
- [Testing APIFlask with schemathesis](http://blog.pamelafox.org/2023/02/testing-apiflask-with-schemathesis.html) by **@pamelafox**
|
|
370
|
-
- **Description**: Explains how to test APIFlask applications using Schemathesis.
|
|
371
|
-
- **Date**: 27 Feb 2023
|
|
372
|
-
- [Using Hypothesis and Schemathesis to Test FastAPI](https://testdriven.io/blog/fastapi-hypothesis/) by **@amalshaji**
|
|
373
|
-
- **Description**: Discusses property-based testing in FastAPI with Hypothesis and Schemathesis.
|
|
374
|
-
- **Date**: 06 Sep 2022
|
|
375
|
-
- [How to use Schemathesis to test Flask API in GitHub Actions](https://notes.lina-is-here.com/2022/08/04/schemathesis-docker-compose.html) by **@lina-is-here**
|
|
376
|
-
- **Description**: Guides you through setting up Schemathesis with Flask API in GitHub Actions.
|
|
377
|
-
- **Date**: 04 Aug 2022
|
|
378
|
-
- [Using API schemas for property-based testing](https://habr.com/ru/company/oleg-bunin/blog/576496/) (RUS) about Schemathesis by **@Stranger6667**
|
|
379
|
-
- **Description**: Covers the usage of Schemathesis for property-based API testing.
|
|
380
|
-
- **Date**: 07 Sep 2021
|
|
381
|
-
- [Schemathesis: property-based testing for API schemas](https://dygalo.dev/blog/schemathesis-property-based-testing-for-api-schemas/) by **@Stranger6667**
|
|
382
|
-
- **Description**: Introduces property-based testing for OpenAPI schemas using Schemathesis.
|
|
383
|
-
- **Date**: 26 Nov 2019
|
|
384
|
-
|
|
385
|
-
### Videos
|
|
386
|
-
|
|
387
|
-
- [API Testing With Unittest Mock, Pytest & Schemathesis](https://youtu.be/56whiv57T9I) by **
|
|
388
|
-
Ssali Jonathan**
|
|
389
|
-
- **Description**: Describes various API testing techniques and features Schemathesis for automatic test generation.
|
|
390
|
-
- **Date**: 29 Jul 2024
|
|
391
|
-
- [API Fuzzing: What it is and why you should use it](https://youtu.be/wX3GMJY9B6A) by **José Haro Peralta**
|
|
392
|
-
- **Description**: A comprehensive overview and demo of Schemathesis.
|
|
393
|
-
- **Date**: 14 Feb 2023
|
|
394
|
-
- [Schemathesis tutorial](https://appdev.consulting.redhat.com/tracks/contract-first/automated-testing-with-schemathesis.html) with an accompanying [video](https://www.youtube.com/watch?v=4r7OC-lBKMg) by **Red Hat**
|
|
395
|
-
- **Description**: Provides a hands-on tutorial for API testing with Schemathesis.
|
|
396
|
-
- **Date**: 09 Feb 2023
|
|
397
|
-
- [Effective API schemas testing](https://youtu.be/VVLZ25JgjD4) from DevConf.cz by **@Stranger6667**
|
|
398
|
-
- **Description**: Talks about using Schemathesis for property-based API schema testing.
|
|
399
|
-
- **Date**: 24 Mar 2021
|
|
400
|
-
- [API-schema-based testing with schemathesis](https://www.youtube.com/watch?v=9FHRwrv-xuQ) from EuroPython 2020 by **@hultner**
|
|
401
|
-
- **Description**: Introduces property-based API testing with Schemathesis.
|
|
402
|
-
- **Date**: 23 Jul 2020
|
|
403
|
-
|
|
404
305
|
## License
|
|
405
306
|
|
|
406
307
|
This project is licensed under the terms of the [MIT license](https://opensource.org/licenses/MIT).
|