whattotest 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- whattotest-0.1.0/LICENSE +21 -0
- whattotest-0.1.0/PKG-INFO +171 -0
- whattotest-0.1.0/README.md +156 -0
- whattotest-0.1.0/pyproject.toml +29 -0
- whattotest-0.1.0/setup.cfg +4 -0
- whattotest-0.1.0/tests/test_engine_cli.py +118 -0
- whattotest-0.1.0/tests/test_units.py +165 -0
- whattotest-0.1.0/whattotest/__init__.py +3 -0
- whattotest-0.1.0/whattotest/cli.py +75 -0
- whattotest-0.1.0/whattotest/config.py +84 -0
- whattotest-0.1.0/whattotest/engine.py +265 -0
- whattotest-0.1.0/whattotest/files.py +178 -0
- whattotest-0.1.0/whattotest/gitio.py +122 -0
- whattotest-0.1.0/whattotest/report.py +175 -0
- whattotest-0.1.0/whattotest/signals.py +230 -0
- whattotest-0.1.0/whattotest/testlink.py +81 -0
- whattotest-0.1.0/whattotest.egg-info/PKG-INFO +171 -0
- whattotest-0.1.0/whattotest.egg-info/SOURCES.txt +20 -0
- whattotest-0.1.0/whattotest.egg-info/dependency_links.txt +1 -0
- whattotest-0.1.0/whattotest.egg-info/entry_points.txt +2 -0
- whattotest-0.1.0/whattotest.egg-info/requires.txt +3 -0
- whattotest-0.1.0/whattotest.egg-info/top_level.txt +1 -0
whattotest-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 whattotest contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: whattotest
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: What to test first in every release: risk-based test prioritisation from your git history.
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: Homepage, https://github.com/jussaragranja/whattotest
|
|
7
|
+
Project-URL: Issues, https://github.com/jussaragranja/whattotest/issues
|
|
8
|
+
Keywords: testing,qa,regression,release,risk,git
|
|
9
|
+
Requires-Python: >=3.11
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Provides-Extra: dev
|
|
13
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
14
|
+
Dynamic: license-file
|
|
15
|
+
|
|
16
|
+
# whattotest: Release Impact Map for QA
|
|
17
|
+
|
|
18
|
+
**What to test first in every release.**
|
|
19
|
+
|
|
20
|
+
Risk-based test prioritisation from your git history: between version A and
|
|
21
|
+
version B, what changed, what it can break, what to test first, and what was
|
|
22
|
+
not changed.
|
|
23
|
+
|
|
24
|
+
Every release, QA gets a list of commits and has to decide what goes into the
|
|
25
|
+
regression. `whattotest` turns the git history into an impact map, explained
|
|
26
|
+
in business areas, with the changes that rarely show up in a PR description:
|
|
27
|
+
database migrations, new environment variables, feature flags, major
|
|
28
|
+
dependency upgrades.
|
|
29
|
+
|
|
30
|
+
- **No AI, no API key, no dependencies.** Python standard library and git.
|
|
31
|
+
- **Zero configuration** to start; areas are inferred from the project
|
|
32
|
+
structure. A small optional file makes them match your business language.
|
|
33
|
+
- **Every risk rating explains why.**
|
|
34
|
+
|
|
35
|
+
Illustrative output for a typical shop application:
|
|
36
|
+
|
|
37
|
+
```text
|
|
38
|
+
$ whattotest v1.4.0..v1.5.0
|
|
39
|
+
|
|
40
|
+
RELEASE IMPACT MAP · v1.4.0 → v1.5.0 · 47 commits · 6 stories
|
|
41
|
+
|
|
42
|
+
SUMMARY
|
|
43
|
+
🔴 2 high · 🟡 3 medium · 🟢 4 low · ⚪ 5 not impacted
|
|
44
|
+
⚠ 1 migration · 2 env vars · 1 dependency
|
|
45
|
+
|
|
46
|
+
AREAS
|
|
47
|
+
🔴 Payments HIGH large change (14 files, 612 lines); 9 bug-fix commits in these files
|
|
48
|
+
recently; new database migration (V42__add_installments.sql);
|
|
49
|
+
no linked tests for InstallmentService.java
|
|
50
|
+
🔴 Checkout HIGH hotspot: CheckoutController.java change often; new environment variable
|
|
51
|
+
PAYMENT_PROVIDER_TIMEOUT; 4 different authors in this release
|
|
52
|
+
🟡 Coupons MEDIUM 6 files, 140 lines changed; 3 bug-fix commits in these files recently
|
|
53
|
+
🟢 Profile LOW small change (1 file, 4 lines)
|
|
54
|
+
⚪ not impacted: Login, Search, Catalog, Notifications, Admin
|
|
55
|
+
|
|
56
|
+
SILENT CHANGES
|
|
57
|
+
Migration new database migration db/migration/V42__add_installments.sql
|
|
58
|
+
Env var new environment variable PAYMENT_PROVIDER_TIMEOUT application.yml
|
|
59
|
+
Dependency com.stripe:stripe-java 22.1.0 → 24.0.0 (major) pom.xml
|
|
60
|
+
|
|
61
|
+
STORIES
|
|
62
|
+
SHOP-123 Free shipping coupon Coupons, Checkout
|
|
63
|
+
SHOP-130 Installments Payments
|
|
64
|
+
|
|
65
|
+
SUGGESTED REGRESSION SCOPE
|
|
66
|
+
Full regression: Payments, Checkout
|
|
67
|
+
Focused tests: Coupons
|
|
68
|
+
Smoke test: Profile
|
|
69
|
+
Not changed: Login, Search, Catalog, Notifications, Admin (lowest priority)
|
|
70
|
+
|
|
71
|
+
TEST GAPS (changed code with no linked test)
|
|
72
|
+
Payments src/main/java/com/shop/payment/InstallmentService.java
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
## Install
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
pip install whattotest
|
|
79
|
+
# until the first PyPI release, install from GitHub:
|
|
80
|
+
pip install git+https://github.com/jussaragranja/whattotest.git
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Requires Python 3.11+ and git.
|
|
84
|
+
|
|
85
|
+
## Usage
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
whattotest v1.4.0..v1.5.0 # between two tags
|
|
89
|
+
whattotest main..release/2.8 # between branches
|
|
90
|
+
whattotest --base main # current branch vs main (a PR)
|
|
91
|
+
whattotest v1.4.0 --format md # markdown for the release issue / wiki
|
|
92
|
+
whattotest v1.4.0 --format json # for tools and AI agents
|
|
93
|
+
whattotest v1.4.0 --history-months 12 # longer history for risk
|
|
94
|
+
whattotest --init # create .whattotest.toml
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
## What it analyses
|
|
98
|
+
|
|
99
|
+
| Analysis | How |
|
|
100
|
+
|---|---|
|
|
101
|
+
| **Areas** | Files grouped into business areas: Java/Kotlin packages below the common prefix, Python packages and modules, `features/`, `modules/`, `packages/` folders; or your own mapping |
|
|
102
|
+
| **Risk per area, with reasons** | Size of the change, bug-fix commits on those files in recent history, hotspots (files that change often), silent changes, changed code with no linked test, number of authors |
|
|
103
|
+
| **Silent changes** | Database migrations (Flyway, Liquibase, Alembic, Rails, Prisma, Django; editing an old migration is flagged), dependency changes in Maven, Gradle, version catalogs, npm, pip and pyproject (major upgrades flagged), new environment variables in Java/Python/JS code, Spring/compose placeholders and `.env` files, feature flags (with your patterns) |
|
|
104
|
+
| **Tests** | Tests linked to changed code by naming convention (`PaymentService` ↔ `PaymentServiceTest`, `pricing.py` ↔ `test_pricing.py`, `Cart.tsx` ↔ `Cart.test.tsx`) and by imports; test gaps |
|
|
105
|
+
| **Stories** | Issue keys (`SHOP-123`) from commit messages, mapped to the areas they touched. Dependency bots and non-ticket keys (CVE-, GHSA-…) are ignored |
|
|
106
|
+
| **Scope** | High → full regression, medium → focused tests, low → smoke, not changed → lowest priority |
|
|
107
|
+
|
|
108
|
+
## Configuration (optional): `.whattotest.toml`
|
|
109
|
+
|
|
110
|
+
```toml
|
|
111
|
+
[areas]
|
|
112
|
+
Checkout = ["src/main/java/**/checkout/**", "web/src/features/checkout/**"]
|
|
113
|
+
Payments = ["src/main/java/**/payment/**", "db/migration/**"]
|
|
114
|
+
|
|
115
|
+
[tests]
|
|
116
|
+
Checkout = ["e2e/checkout/**"]
|
|
117
|
+
|
|
118
|
+
[signals]
|
|
119
|
+
flags = ['isEnabled\("([\w.-]+)"\)', 'featureFlags\.(\w+)']
|
|
120
|
+
tickets = '\b[A-Z][A-Z0-9]+-\d+\b'
|
|
121
|
+
|
|
122
|
+
[risk]
|
|
123
|
+
history_months = 6
|
|
124
|
+
fix_patterns = ["fix", "bug", "hotfix", "revert"]
|
|
125
|
+
|
|
126
|
+
ignore = ["docs/**", "**/*.md"]
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
The configuration is read from the head ref of the range.
|
|
130
|
+
|
|
131
|
+
## Does it work? Retrospective validation
|
|
132
|
+
|
|
133
|
+
We mapped 25 consecutive releases of three open-source projects with no
|
|
134
|
+
configuration, then looked at the bug fixes made after each release.
|
|
135
|
+
|
|
136
|
+
| Project | Areas rated high that got a later bug fix | Medium | Low | Not changed |
|
|
137
|
+
|---|---|---|---|---|
|
|
138
|
+
| jsoup (Java) | **96%** | 73% | 38% | **8%** |
|
|
139
|
+
| click (Python) | **67%** | 40% | 27% | **9%** |
|
|
140
|
+
| requests (Python) | — | 30% | 12% | **7%** |
|
|
141
|
+
|
|
142
|
+
In jsoup, the 29% of areas rated high or medium concentrated 79% of later
|
|
143
|
+
fixes. But "not changed" is not "bug-free": in requests, half of later fixes
|
|
144
|
+
landed in areas the release did not touch (old bugs, not regressions). Details
|
|
145
|
+
and limitations: [research/RESULTS.md](research/RESULTS.md).
|
|
146
|
+
|
|
147
|
+
## For AI coding agents
|
|
148
|
+
|
|
149
|
+
`skills/impact-to-test-plan/SKILL.md` lets the assistant you already use
|
|
150
|
+
(Claude Code, Copilot, Codex, Cursor…) turn the JSON map into exploratory test
|
|
151
|
+
charters per risky area. Copy it to `.agents/skills/` and/or `.claude/skills/`.
|
|
152
|
+
|
|
153
|
+
## Limitations
|
|
154
|
+
|
|
155
|
+
- Areas inferred without configuration follow the code structure, which is
|
|
156
|
+
not always the business structure. Configure `[areas]` for best results.
|
|
157
|
+
- Risk is a prioritisation aid built from history, not a prediction of bugs.
|
|
158
|
+
- Test links are static (names and imports), not coverage data.
|
|
159
|
+
- One repository per run; a release spanning backend, web and mobile
|
|
160
|
+
repositories is planned.
|
|
161
|
+
|
|
162
|
+
## Development
|
|
163
|
+
|
|
164
|
+
```bash
|
|
165
|
+
python -m venv .venv && .venv/bin/pip install -e ".[dev]"
|
|
166
|
+
.venv/bin/python -m pytest
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
## License
|
|
170
|
+
|
|
171
|
+
MIT
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
# whattotest: Release Impact Map for QA
|
|
2
|
+
|
|
3
|
+
**What to test first in every release.**
|
|
4
|
+
|
|
5
|
+
Risk-based test prioritisation from your git history: between version A and
|
|
6
|
+
version B, what changed, what it can break, what to test first, and what was
|
|
7
|
+
not changed.
|
|
8
|
+
|
|
9
|
+
Every release, QA gets a list of commits and has to decide what goes into the
|
|
10
|
+
regression. `whattotest` turns the git history into an impact map, explained
|
|
11
|
+
in business areas, with the changes that rarely show up in a PR description:
|
|
12
|
+
database migrations, new environment variables, feature flags, major
|
|
13
|
+
dependency upgrades.
|
|
14
|
+
|
|
15
|
+
- **No AI, no API key, no dependencies.** Python standard library and git.
|
|
16
|
+
- **Zero configuration** to start; areas are inferred from the project
|
|
17
|
+
structure. A small optional file makes them match your business language.
|
|
18
|
+
- **Every risk rating explains why.**
|
|
19
|
+
|
|
20
|
+
Illustrative output for a typical shop application:
|
|
21
|
+
|
|
22
|
+
```text
|
|
23
|
+
$ whattotest v1.4.0..v1.5.0
|
|
24
|
+
|
|
25
|
+
RELEASE IMPACT MAP · v1.4.0 → v1.5.0 · 47 commits · 6 stories
|
|
26
|
+
|
|
27
|
+
SUMMARY
|
|
28
|
+
🔴 2 high · 🟡 3 medium · 🟢 4 low · ⚪ 5 not impacted
|
|
29
|
+
⚠ 1 migration · 2 env vars · 1 dependency
|
|
30
|
+
|
|
31
|
+
AREAS
|
|
32
|
+
🔴 Payments HIGH large change (14 files, 612 lines); 9 bug-fix commits in these files
|
|
33
|
+
recently; new database migration (V42__add_installments.sql);
|
|
34
|
+
no linked tests for InstallmentService.java
|
|
35
|
+
🔴 Checkout HIGH hotspot: CheckoutController.java change often; new environment variable
|
|
36
|
+
PAYMENT_PROVIDER_TIMEOUT; 4 different authors in this release
|
|
37
|
+
🟡 Coupons MEDIUM 6 files, 140 lines changed; 3 bug-fix commits in these files recently
|
|
38
|
+
🟢 Profile LOW small change (1 file, 4 lines)
|
|
39
|
+
⚪ not impacted: Login, Search, Catalog, Notifications, Admin
|
|
40
|
+
|
|
41
|
+
SILENT CHANGES
|
|
42
|
+
Migration new database migration db/migration/V42__add_installments.sql
|
|
43
|
+
Env var new environment variable PAYMENT_PROVIDER_TIMEOUT application.yml
|
|
44
|
+
Dependency com.stripe:stripe-java 22.1.0 → 24.0.0 (major) pom.xml
|
|
45
|
+
|
|
46
|
+
STORIES
|
|
47
|
+
SHOP-123 Free shipping coupon Coupons, Checkout
|
|
48
|
+
SHOP-130 Installments Payments
|
|
49
|
+
|
|
50
|
+
SUGGESTED REGRESSION SCOPE
|
|
51
|
+
Full regression: Payments, Checkout
|
|
52
|
+
Focused tests: Coupons
|
|
53
|
+
Smoke test: Profile
|
|
54
|
+
Not changed: Login, Search, Catalog, Notifications, Admin (lowest priority)
|
|
55
|
+
|
|
56
|
+
TEST GAPS (changed code with no linked test)
|
|
57
|
+
Payments src/main/java/com/shop/payment/InstallmentService.java
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## Install
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
pip install whattotest
|
|
64
|
+
# until the first PyPI release, install from GitHub:
|
|
65
|
+
pip install git+https://github.com/jussaragranja/whattotest.git
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
Requires Python 3.11+ and git.
|
|
69
|
+
|
|
70
|
+
## Usage
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
whattotest v1.4.0..v1.5.0 # between two tags
|
|
74
|
+
whattotest main..release/2.8 # between branches
|
|
75
|
+
whattotest --base main # current branch vs main (a PR)
|
|
76
|
+
whattotest v1.4.0 --format md # markdown for the release issue / wiki
|
|
77
|
+
whattotest v1.4.0 --format json # for tools and AI agents
|
|
78
|
+
whattotest v1.4.0 --history-months 12 # longer history for risk
|
|
79
|
+
whattotest --init # create .whattotest.toml
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
## What it analyses
|
|
83
|
+
|
|
84
|
+
| Analysis | How |
|
|
85
|
+
|---|---|
|
|
86
|
+
| **Areas** | Files grouped into business areas: Java/Kotlin packages below the common prefix, Python packages and modules, `features/`, `modules/`, `packages/` folders; or your own mapping |
|
|
87
|
+
| **Risk per area, with reasons** | Size of the change, bug-fix commits on those files in recent history, hotspots (files that change often), silent changes, changed code with no linked test, number of authors |
|
|
88
|
+
| **Silent changes** | Database migrations (Flyway, Liquibase, Alembic, Rails, Prisma, Django; editing an old migration is flagged), dependency changes in Maven, Gradle, version catalogs, npm, pip and pyproject (major upgrades flagged), new environment variables in Java/Python/JS code, Spring/compose placeholders and `.env` files, feature flags (with your patterns) |
|
|
89
|
+
| **Tests** | Tests linked to changed code by naming convention (`PaymentService` ↔ `PaymentServiceTest`, `pricing.py` ↔ `test_pricing.py`, `Cart.tsx` ↔ `Cart.test.tsx`) and by imports; test gaps |
|
|
90
|
+
| **Stories** | Issue keys (`SHOP-123`) from commit messages, mapped to the areas they touched. Dependency bots and non-ticket keys (CVE-, GHSA-…) are ignored |
|
|
91
|
+
| **Scope** | High → full regression, medium → focused tests, low → smoke, not changed → lowest priority |
|
|
92
|
+
|
|
93
|
+
## Configuration (optional): `.whattotest.toml`
|
|
94
|
+
|
|
95
|
+
```toml
|
|
96
|
+
[areas]
|
|
97
|
+
Checkout = ["src/main/java/**/checkout/**", "web/src/features/checkout/**"]
|
|
98
|
+
Payments = ["src/main/java/**/payment/**", "db/migration/**"]
|
|
99
|
+
|
|
100
|
+
[tests]
|
|
101
|
+
Checkout = ["e2e/checkout/**"]
|
|
102
|
+
|
|
103
|
+
[signals]
|
|
104
|
+
flags = ['isEnabled\("([\w.-]+)"\)', 'featureFlags\.(\w+)']
|
|
105
|
+
tickets = '\b[A-Z][A-Z0-9]+-\d+\b'
|
|
106
|
+
|
|
107
|
+
[risk]
|
|
108
|
+
history_months = 6
|
|
109
|
+
fix_patterns = ["fix", "bug", "hotfix", "revert"]
|
|
110
|
+
|
|
111
|
+
ignore = ["docs/**", "**/*.md"]
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
The configuration is read from the head ref of the range.
|
|
115
|
+
|
|
116
|
+
## Does it work? Retrospective validation
|
|
117
|
+
|
|
118
|
+
We mapped 25 consecutive releases of three open-source projects with no
|
|
119
|
+
configuration, then looked at the bug fixes made after each release.
|
|
120
|
+
|
|
121
|
+
| Project | Areas rated high that got a later bug fix | Medium | Low | Not changed |
|
|
122
|
+
|---|---|---|---|---|
|
|
123
|
+
| jsoup (Java) | **96%** | 73% | 38% | **8%** |
|
|
124
|
+
| click (Python) | **67%** | 40% | 27% | **9%** |
|
|
125
|
+
| requests (Python) | — | 30% | 12% | **7%** |
|
|
126
|
+
|
|
127
|
+
In jsoup, the 29% of areas rated high or medium concentrated 79% of later
|
|
128
|
+
fixes. But "not changed" is not "bug-free": in requests, half of later fixes
|
|
129
|
+
landed in areas the release did not touch (old bugs, not regressions). Details
|
|
130
|
+
and limitations: [research/RESULTS.md](research/RESULTS.md).
|
|
131
|
+
|
|
132
|
+
## For AI coding agents
|
|
133
|
+
|
|
134
|
+
`skills/impact-to-test-plan/SKILL.md` lets the assistant you already use
|
|
135
|
+
(Claude Code, Copilot, Codex, Cursor…) turn the JSON map into exploratory test
|
|
136
|
+
charters per risky area. Copy it to `.agents/skills/` and/or `.claude/skills/`.
|
|
137
|
+
|
|
138
|
+
## Limitations
|
|
139
|
+
|
|
140
|
+
- Areas inferred without configuration follow the code structure, which is
|
|
141
|
+
not always the business structure. Configure `[areas]` for best results.
|
|
142
|
+
- Risk is a prioritisation aid built from history, not a prediction of bugs.
|
|
143
|
+
- Test links are static (names and imports), not coverage data.
|
|
144
|
+
- One repository per run; a release spanning backend, web and mobile
|
|
145
|
+
repositories is planned.
|
|
146
|
+
|
|
147
|
+
## Development
|
|
148
|
+
|
|
149
|
+
```bash
|
|
150
|
+
python -m venv .venv && .venv/bin/pip install -e ".[dev]"
|
|
151
|
+
.venv/bin/python -m pytest
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
## License
|
|
155
|
+
|
|
156
|
+
MIT
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "whattotest"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "What to test first in every release: risk-based test prioritisation from your git history."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
requires-python = ">=3.11"
|
|
12
|
+
dependencies = []
|
|
13
|
+
keywords = ["testing", "qa", "regression", "release", "risk", "git"]
|
|
14
|
+
|
|
15
|
+
[project.scripts]
|
|
16
|
+
whattotest = "whattotest.cli:main"
|
|
17
|
+
|
|
18
|
+
[project.optional-dependencies]
|
|
19
|
+
dev = ["pytest>=8"]
|
|
20
|
+
|
|
21
|
+
[tool.setuptools.packages.find]
|
|
22
|
+
include = ["whattotest*"]
|
|
23
|
+
|
|
24
|
+
[tool.pytest.ini_options]
|
|
25
|
+
testpaths = ["tests"]
|
|
26
|
+
|
|
27
|
+
[project.urls]
|
|
28
|
+
Homepage = "https://github.com/jussaragranja/whattotest"
|
|
29
|
+
Issues = "https://github.com/jussaragranja/whattotest/issues"
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import json
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from whattotest import cli
|
|
6
|
+
|
|
7
|
+
PAY = "src/main/java/com/shop/payment/PaymentService.java"
|
|
8
|
+
PAY_TEST = "src/test/java/com/shop/payment/PaymentServiceTest.java"
|
|
9
|
+
CHECKOUT = "src/main/java/com/shop/checkout/CheckoutService.java"
|
|
10
|
+
LOGIN = "src/main/java/com/shop/login/LoginService.java"
|
|
11
|
+
SEARCH = "src/main/java/com/shop/search/SearchService.java"
|
|
12
|
+
POM = """<project><dependencies>
|
|
13
|
+
<dependency><groupId>com.stripe</groupId><artifactId>stripe-java</artifactId><version>22.0.0</version></dependency>
|
|
14
|
+
</dependencies></project>"""
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def java(cls, body=""):
|
|
18
|
+
pkg = {"PaymentService": "payment", "CheckoutService": "checkout", "LoginService": "login",
|
|
19
|
+
"SearchService": "search", "PaymentServiceTest": "payment"}[cls]
|
|
20
|
+
return f"package com.shop.{pkg};\npublic class {cls} {{\n{body}\n}}\n"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@pytest.fixture
|
|
24
|
+
def release(repo):
|
|
25
|
+
"""v1.0 -> v1.1: payments heavily changed (with history of fixes), checkout touched,
|
|
26
|
+
login and search untouched."""
|
|
27
|
+
repo.commit({PAY: java("PaymentService"), PAY_TEST: java("PaymentServiceTest"),
|
|
28
|
+
CHECKOUT: java("CheckoutService"), LOGIN: java("LoginService"),
|
|
29
|
+
SEARCH: java("SearchService"), "pom.xml": POM}, "initial")
|
|
30
|
+
for i in range(3):
|
|
31
|
+
repo.commit({PAY: java("PaymentService", f"// fix {i}")}, f"fix: payment rounding bug {i}")
|
|
32
|
+
repo.tag("v1.0")
|
|
33
|
+
body = "\n".join(f" int f{i}() {{ return {i}; }}" for i in range(80))
|
|
34
|
+
repo.commit({PAY: java("PaymentService", body + '\n String t = System.getenv("PAY_TIMEOUT");'),
|
|
35
|
+
"db/migration/V2__installments.sql": "alter table payment add installments int;"},
|
|
36
|
+
"SHOP-130: installments", author="Ana <ana@example.invalid>")
|
|
37
|
+
repo.commit({CHECKOUT: java("CheckoutService", "int total() { return 1; }")},
|
|
38
|
+
"SHOP-123 free shipping coupon", author="Bia <bia@example.invalid>")
|
|
39
|
+
repo.commit({"pom.xml": POM.replace("22.0.0", "24.1.0")},
|
|
40
|
+
"Bump stripe-java. Fixes SUREFIRE-123 upstream, CVE-2026-1",
|
|
41
|
+
author="dependabot[bot] <bot@example.invalid>")
|
|
42
|
+
repo.commit({"docs/notes.md": "release notes"}, "docs")
|
|
43
|
+
repo.tag("v1.1")
|
|
44
|
+
return repo
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def run(repo, capsys, *args):
|
|
48
|
+
code = cli.main(["--repo", str(repo.path), *args])
|
|
49
|
+
return code, capsys.readouterr().out
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def test_full_report_json(release, capsys):
|
|
53
|
+
code, out = run(release, capsys, "v1.0..v1.1", "--format", "json")
|
|
54
|
+
data = json.loads(out)
|
|
55
|
+
assert code == 0
|
|
56
|
+
areas = {a["name"]: a for a in data["areas"]}
|
|
57
|
+
assert set(areas) == {"Payment", "Checkout", "Database"}
|
|
58
|
+
pay = areas["Payment"]
|
|
59
|
+
# 3 bug-fix commits + hotspot + new env var + 80-line change; the migration belongs to
|
|
60
|
+
# "Database" without configuration (see test_config_from_head_defines_areas)
|
|
61
|
+
assert pay["risk"] == "medium" and pay["score"] == 4
|
|
62
|
+
assert pay["bug_fix_commits"] == 3
|
|
63
|
+
assert areas["Database"]["risk"] == "medium"
|
|
64
|
+
assert any("PAY_TIMEOUT" in s["detail"] for s in pay["signals"])
|
|
65
|
+
assert PAY_TEST in pay["related_tests"]
|
|
66
|
+
assert areas["Checkout"]["untested_files"] == [CHECKOUT]
|
|
67
|
+
assert data["not_impacted"] == ["Login", "Search"]
|
|
68
|
+
assert [s["key"] for s in data["stories"]] == ["SHOP-123", "SHOP-130"] # no CVE/SUREFIRE, no bot
|
|
69
|
+
assert any("(major)" in s["detail"] for s in data["release_signals"])
|
|
70
|
+
assert "docs/notes.md" in data["other_files"]
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def test_text_and_markdown(release, capsys):
|
|
74
|
+
_, text = run(release, capsys, "v1.0..v1.1")
|
|
75
|
+
assert "SUGGESTED REGRESSION SCOPE" in text and "Not changed:" in text and "Login, Search" in text
|
|
76
|
+
_, md = run(release, capsys, "v1.0..v1.1", "--format", "md")
|
|
77
|
+
for section in ("### Suggested regression scope", "### Areas", "### Silent changes",
|
|
78
|
+
"### Stories in this release", "Test gaps"):
|
|
79
|
+
assert section in md
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def test_base_option_and_single_ref(release, capsys):
|
|
83
|
+
code, out = run(release, capsys, "--base", "v1.0", "--format", "json")
|
|
84
|
+
assert code == 0 and json.loads(out)["commits"] == 4
|
|
85
|
+
code, out = run(release, capsys, "v1.0", "--format", "json")
|
|
86
|
+
assert json.loads(out)["head"] == "HEAD"
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def test_config_from_head_defines_areas(release, capsys):
|
|
90
|
+
release.commit({".whattotest.toml": '[areas]\nMoney = ["**/payment/**", "db/migration/**"]\n'})
|
|
91
|
+
_, out = run(release, capsys, "v1.0..HEAD", "--format", "json")
|
|
92
|
+
areas = {a["name"]: a for a in json.loads(out)["areas"]}
|
|
93
|
+
assert "Money" in areas and "Payment" not in areas and "Database" not in areas
|
|
94
|
+
# with the migration in the same area, the risk adds up to high
|
|
95
|
+
assert areas["Money"]["risk"] == "high"
|
|
96
|
+
assert any("migration" in r for r in areas["Money"]["reasons"])
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def test_tests_only_area(repo, capsys):
|
|
100
|
+
repo.commit({PAY: java("PaymentService"), PAY_TEST: java("PaymentServiceTest")})
|
|
101
|
+
repo.tag("a")
|
|
102
|
+
repo.commit({PAY_TEST: java("PaymentServiceTest", "void t() {}")})
|
|
103
|
+
_, out = run(repo, capsys, "a..HEAD", "--format", "json")
|
|
104
|
+
data = json.loads(out)
|
|
105
|
+
assert data["areas"] == [] and data["tests_only"] == ["Payment"]
|
|
106
|
+
assert "Payment" not in data["not_impacted"]
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def test_errors(repo, capsys):
|
|
110
|
+
repo.commit({"a.txt": "x"})
|
|
111
|
+
assert run(repo, capsys, "nope..HEAD")[0] == 2
|
|
112
|
+
assert run(repo, capsys)[0] == 2
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def test_init(repo, capsys):
|
|
116
|
+
assert run(repo, capsys, "--init")[0] == 0
|
|
117
|
+
assert (repo.path / ".whattotest.toml").exists()
|
|
118
|
+
assert run(repo, capsys, "--init")[0] == 2
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
from whattotest.config import Config, ConfigError, from_text
|
|
2
|
+
from whattotest.files import AreaMapper, kind
|
|
3
|
+
from whattotest.signals import (Signal, _gradle, _maven, _npm, _pip, _pyproject, _version_catalog,
|
|
4
|
+
dependencies, env_vars, flags, is_major, migrations)
|
|
5
|
+
from whattotest.gitio import FileChange
|
|
6
|
+
from whattotest.testlink import LinkIndex
|
|
7
|
+
|
|
8
|
+
import pytest
|
|
9
|
+
import re
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
# ---------------------------------------------------------------- kinds
|
|
13
|
+
|
|
14
|
+
@pytest.mark.parametrize("path, expected", [
|
|
15
|
+
("src/main/java/com/shop/Pay.java", "code"),
|
|
16
|
+
("src/test/java/com/shop/PayTest.java", "test"),
|
|
17
|
+
("tests/test_pricing.py", "test"),
|
|
18
|
+
("web/src/Cart.test.tsx", "test"),
|
|
19
|
+
("db/migration/V2__add.sql", "migration"),
|
|
20
|
+
("app/alembic/versions/abc_add.py", "migration"),
|
|
21
|
+
("pom.xml", "dependency"),
|
|
22
|
+
("package-lock.json", "lockfile"),
|
|
23
|
+
("src/main/resources/application.yml", "config"),
|
|
24
|
+
("app/src/main/res/values/strings.xml", "i18n"),
|
|
25
|
+
("Dockerfile", "infra"),
|
|
26
|
+
("README.md", "docs"),
|
|
27
|
+
("src/main/java/com/shop/package-info.java", "other"),
|
|
28
|
+
])
|
|
29
|
+
def test_kind(path, expected):
|
|
30
|
+
assert kind(path) == expected
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
# ---------------------------------------------------------------- areas
|
|
34
|
+
|
|
35
|
+
def test_java_areas_below_common_package():
|
|
36
|
+
files = ["src/main/java/com/shop/checkout/CheckoutService.java",
|
|
37
|
+
"src/main/java/com/shop/payment/PaymentService.java",
|
|
38
|
+
"src/main/java/com/shop/payment/gateway/Stripe.java",
|
|
39
|
+
"src/test/java/com/shop/payment/PaymentServiceTest.java"]
|
|
40
|
+
m = AreaMapper(Config(), files)
|
|
41
|
+
assert [m.area(f) for f in files] == ["Checkout", "Payment", "Payment", "Payment"]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def test_single_package_java_groups_class_families():
|
|
45
|
+
files = ["src/main/java/shop/Coupon.java", "src/main/java/shop/Coupons.java",
|
|
46
|
+
"src/main/java/shop/CouponException.java", "src/test/java/shop/CouponsTest.java",
|
|
47
|
+
"src/main/java/shop/ShippingMethod.java"]
|
|
48
|
+
m = AreaMapper(Config(), files)
|
|
49
|
+
assert [m.area(f) for f in files] == ["Coupon", "Coupon", "Coupon", "Coupon", "Shipping"]
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def test_python_flat_package_and_subpackages():
|
|
53
|
+
files = ["shop/__init__.py", "shop/pricing.py", "tests/test_pricing.py",
|
|
54
|
+
"app/__init__.py", "app/checkout/__init__.py", "app/checkout/service.py"]
|
|
55
|
+
m = AreaMapper(Config(), files)
|
|
56
|
+
assert m.area("shop/pricing.py") == "Pricing"
|
|
57
|
+
assert m.area("tests/test_pricing.py") == "Pricing"
|
|
58
|
+
assert m.area("app/checkout/service.py") == "Checkout"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def test_js_feature_folders():
|
|
62
|
+
m = AreaMapper(Config(), [])
|
|
63
|
+
assert m.area("web/src/features/cart/Cart.tsx") == "Cart"
|
|
64
|
+
assert m.area("packages/payments/src/index.ts") == "Payments"
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def test_configured_areas_win():
|
|
68
|
+
cfg = from_text('[areas]\nCheckout = ["web/**/checkout/**", "src/**/cart/**"]\n')
|
|
69
|
+
m = AreaMapper(cfg, [])
|
|
70
|
+
assert m.area("src/main/java/com/shop/cart/Cart.java") == "Checkout"
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def test_bad_config_is_an_error():
|
|
74
|
+
with pytest.raises(ConfigError):
|
|
75
|
+
from_text("[signals]\nflags = ['(unclosed']\n")
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
# ---------------------------------------------------------------- signals
|
|
79
|
+
|
|
80
|
+
def fc(path, status="M", old=None):
|
|
81
|
+
return FileChange(path, old if old is not None else (None if status == "A" else path), status)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def test_migrations():
|
|
85
|
+
sigs = migrations([fc("db/migration/V3__x.sql", "A"), fc("db/migration/V1__init.sql", "M"),
|
|
86
|
+
fc("src/A.java")])
|
|
87
|
+
assert [s.detail for s in sigs] == ["new database migration",
|
|
88
|
+
"existing migration edited (already-applied migrations should not change)"]
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def test_major_versions():
|
|
92
|
+
assert is_major("2.7.18", "3.0.0")
|
|
93
|
+
assert is_major("0.11.0", "0.12.0")
|
|
94
|
+
assert not is_major("3.5.6", "3.6.0")
|
|
95
|
+
assert not is_major("1.0", "x")
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def test_maven_with_properties():
|
|
99
|
+
pom = """<project><parent><artifactId>spring-boot-starter-parent</artifactId><version>2.7.18</version></parent>
|
|
100
|
+
<properties><stripe.version>22.1.0</stripe.version></properties>
|
|
101
|
+
<dependencies><dependency><groupId>com.stripe</groupId><artifactId>stripe-java</artifactId>
|
|
102
|
+
<version>${stripe.version}</version></dependency></dependencies></project>"""
|
|
103
|
+
deps = _maven(pom)
|
|
104
|
+
assert deps["com.stripe:stripe-java"] == "22.1.0"
|
|
105
|
+
assert deps["parent:spring-boot-starter-parent"] == "2.7.18"
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def test_other_manifests():
|
|
109
|
+
assert _gradle('implementation "com.squareup.okhttp3:okhttp:4.12.0"') == {"com.squareup.okhttp3:okhttp": "4.12.0"}
|
|
110
|
+
assert _npm('{"dependencies": {"react": "^18.3.1"}}') == {"react": "^18.3.1"}
|
|
111
|
+
assert _pip("requests==2.31.0 # http\nflask>=3.0") == {"requests": "2.31.0", "flask": "3.0"}
|
|
112
|
+
assert _pyproject('[project]\ndependencies = ["httpx>=0.27"]\n') == {"httpx": "0.27"}
|
|
113
|
+
assert _version_catalog('[versions]\nktor="2.3.0"\n[libraries]\nktor-core={module="io.ktor:ktor-core",version.ref="ktor"}\n')["io.ktor:ktor-core"] == "2.3.0"
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def test_dependency_signals():
|
|
117
|
+
base = {"pom.xml": "<dependency><groupId>g</groupId><artifactId>a</artifactId><version>1.0.0</version></dependency>"
|
|
118
|
+
"<dependency><groupId>g</groupId><artifactId>b</artifactId><version>1.0.0</version></dependency>"}
|
|
119
|
+
head = {"pom.xml": "<dependency><groupId>g</groupId><artifactId>a</artifactId><version>2.0.0</version></dependency>"
|
|
120
|
+
"<dependency><groupId>g</groupId><artifactId>c</artifactId><version>1.0.0</version></dependency>"}
|
|
121
|
+
sigs = dependencies([fc("pom.xml")], base.get, head.get)
|
|
122
|
+
assert {(s.severity, s.detail) for s in sigs} == {
|
|
123
|
+
("high", "g:a 1.0.0 → 2.0.0 (major)"), ("medium", "removed g:b 1.0.0"), ("medium", "added g:c 1.0.0")}
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def test_env_vars_in_code_config_and_dotenv():
|
|
127
|
+
base = {"src/Pay.java": 'String k = System.getenv("API_KEY");',
|
|
128
|
+
"app/settings.py": "x = os.getenv('DB_URL')",
|
|
129
|
+
"src/main/resources/application.yml": "url: ${DB_URL}",
|
|
130
|
+
".env.example": "DB_URL=\n"}
|
|
131
|
+
head = {"src/Pay.java": 'String k = System.getenv("API_KEY"); int t = Integer.parseInt(System.getenv("PAY_TIMEOUT"));',
|
|
132
|
+
"app/settings.py": "x = os.environ['DB_URL']",
|
|
133
|
+
"src/main/resources/application.yml": "url: ${DB_URL}\ntimeout: ${PAY_TIMEOUT:30}",
|
|
134
|
+
".env.example": "DB_URL=\nPAY_TIMEOUT=30\n"}
|
|
135
|
+
sigs = env_vars([fc(p) for p in base], base.get, head.get)
|
|
136
|
+
assert sorted((s.file, s.name) for s in sigs) == [
|
|
137
|
+
(".env.example", "PAY_TIMEOUT"), ("src/Pay.java", "PAY_TIMEOUT"),
|
|
138
|
+
("src/main/resources/application.yml", "PAY_TIMEOUT")]
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def test_flags_need_patterns():
|
|
142
|
+
base = {"src/A.java": 'if (flags.isEnabled("old")) {}'}
|
|
143
|
+
head = {"src/A.java": 'if (flags.isEnabled("new_checkout")) {}'}
|
|
144
|
+
assert flags([fc("src/A.java")], base.get, head.get, []) == []
|
|
145
|
+
sigs = flags([fc("src/A.java")], base.get, head.get, [re.compile(r'isEnabled\("([\w.-]+)"\)')])
|
|
146
|
+
assert {(s.name, s.detail.split()[0]) for s in sigs} == {("new_checkout", "new"), ("old", "feature")}
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
# ---------------------------------------------------------------- test links
|
|
150
|
+
|
|
151
|
+
def test_test_links():
|
|
152
|
+
index = LinkIndex({
|
|
153
|
+
"src/test/java/com/shop/payment/PaymentServiceTest.java": "package com.shop.payment;",
|
|
154
|
+
"src/test/java/com/shop/payment/RefundFlowTest.java": "package com.shop.payment;\n new Refunds();",
|
|
155
|
+
"tests/test_checkout.py": "from shop.pricing import apply_discount",
|
|
156
|
+
"web/src/Cart.test.tsx": "import { Cart } from './Cart'",
|
|
157
|
+
"web/src/flows/checkout.spec.ts": "import { Cart } from '../Cart'",
|
|
158
|
+
})
|
|
159
|
+
assert index.linked("src/main/java/com/shop/payment/PaymentService.java") == {
|
|
160
|
+
"src/test/java/com/shop/payment/PaymentServiceTest.java"}
|
|
161
|
+
assert index.linked("src/main/java/com/shop/payment/Refunds.java") == {
|
|
162
|
+
"src/test/java/com/shop/payment/RefundFlowTest.java"}
|
|
163
|
+
assert index.linked("shop/pricing.py") == {"tests/test_checkout.py"}
|
|
164
|
+
assert index.linked("web/src/Cart.tsx") == {"web/src/Cart.test.tsx", "web/src/flows/checkout.spec.ts"}
|
|
165
|
+
assert index.linked("shop/unrelated.py") == set()
|