pytest-tia 1.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pytest_tia-1.1.0/PKG-INFO +230 -0
- pytest_tia-1.1.0/README.md +217 -0
- pytest_tia-1.1.0/pyproject.toml +26 -0
- pytest_tia-1.1.0/pytest_tia.egg-info/PKG-INFO +230 -0
- pytest_tia-1.1.0/pytest_tia.egg-info/SOURCES.txt +24 -0
- pytest_tia-1.1.0/pytest_tia.egg-info/dependency_links.txt +1 -0
- pytest_tia-1.1.0/pytest_tia.egg-info/entry_points.txt +2 -0
- pytest_tia-1.1.0/pytest_tia.egg-info/requires.txt +8 -0
- pytest_tia-1.1.0/pytest_tia.egg-info/top_level.txt +1 -0
- pytest_tia-1.1.0/setup.cfg +4 -0
- pytest_tia-1.1.0/tests/test_core.py +531 -0
- pytest_tia-1.1.0/tests/test_mutation_guarantee.py +251 -0
- pytest_tia-1.1.0/tia/__init__.py +7 -0
- pytest_tia-1.1.0/tia/__main__.py +4 -0
- pytest_tia-1.1.0/tia/astmap.py +39 -0
- pytest_tia-1.1.0/tia/cli.py +308 -0
- pytest_tia-1.1.0/tia/diff.py +56 -0
- pytest_tia-1.1.0/tia/dynscan.py +58 -0
- pytest_tia-1.1.0/tia/plugin.py +137 -0
- pytest_tia-1.1.0/tia/remotestore.py +222 -0
- pytest_tia-1.1.0/tia/report.py +96 -0
- pytest_tia-1.1.0/tia/resolve.py +80 -0
- pytest_tia-1.1.0/tia/select.py +76 -0
- pytest_tia-1.1.0/tia/semantic.py +128 -0
- pytest_tia-1.1.0/tia/server.py +97 -0
- pytest_tia-1.1.0/tia/store.py +85 -0
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pytest-tia
|
|
3
|
+
Version: 1.1.0
|
|
4
|
+
Summary: Test Impact Analysis for pytest — run only the tests your changes affect
|
|
5
|
+
Requires-Python: >=3.10
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
Requires-Dist: pytest>=7
|
|
8
|
+
Requires-Dist: coverage>=7
|
|
9
|
+
Provides-Extra: s3
|
|
10
|
+
Requires-Dist: boto3>=1.26; extra == "s3"
|
|
11
|
+
Provides-Extra: gcs
|
|
12
|
+
Requires-Dist: google-cloud-storage>=2.0; extra == "gcs"
|
|
13
|
+
|
|
14
|
+
# tia — Test Impact Analysis for pytest
|
|
15
|
+
|
|
16
|
+
Run only the tests your changes actually affect. Big suites spend most
|
|
17
|
+
of their CI time re-running tests that couldn't possibly have broken;
|
|
18
|
+
`tia` builds a per-test coverage map once, then uses your `git diff` to
|
|
19
|
+
select the minimal set of tests to run.
|
|
20
|
+
|
|
21
|
+
This is the same idea Google/Meta run internally (Test Impact Analysis).
|
|
22
|
+
|
|
23
|
+
## Honest disclosure (read this first)
|
|
24
|
+
|
|
25
|
+
Test selection has exactly one cardinal sin: skipping a test that would
|
|
26
|
+
have failed. A tool you can't trust on that is worse than no tool. So
|
|
27
|
+
before any feature list, here is where this project earned — or failed to
|
|
28
|
+
earn — that trust.
|
|
29
|
+
|
|
30
|
+
While benchmarking on Flask, an early run reported a **median skip rate of
|
|
31
|
+
73%**. It looked great. It was a lie. coverage.py's default measurement
|
|
32
|
+
core on Python 3.12+ (`sysmon`) records only the *first* test to hit each
|
|
33
|
+
line and silently drops the rest, so any test that reused a shared helper
|
|
34
|
+
was never mapped — and changing that helper would have skipped them. The
|
|
35
|
+
high number was false negatives wearing a costume. We forced the C tracer
|
|
36
|
+
(`COVERAGE_CORE=ctrace`), every test mapped, and the number fell to the
|
|
37
|
+
truth: **median ~21% skip on Flask** (≈40% mean; cosmetic commits 99%+).
|
|
38
|
+
The toy demo could never have shown this — only a real codebase did.
|
|
39
|
+
|
|
40
|
+
So the Flask number is deliberately the *lower bound*: Flask is small and
|
|
41
|
+
tightly coupled, near the worst case for test selection. We'd rather
|
|
42
|
+
publish the honest floor than a cherry-picked ceiling. But a floor alone
|
|
43
|
+
is misleading too, so we measured the other end: on
|
|
44
|
+
[boltons](benchmark/RESULTS-boltons.md), a modular utility library, the
|
|
45
|
+
honest median skip on real logic changes is **~96%**. Same tool, same
|
|
46
|
+
rules — the variable is how decoupled your tests are. The truth is a
|
|
47
|
+
**range, ~21% (worst case) ↔ ~96% (modular)**, and both ends are measured
|
|
48
|
+
on real third-party suites, not toys. Per-commit tables and the "I
|
|
49
|
+
distrusted this number first" audits:
|
|
50
|
+
[Flask](benchmark/RESULTS.md) · [boltons](benchmark/RESULTS-boltons.md).
|
|
51
|
+
|
|
52
|
+
## How it works
|
|
53
|
+
|
|
54
|
+
1. **`tia record`** runs the full suite once with a pytest plugin that
|
|
55
|
+
switches coverage.py's *dynamic context* to each test's nodeid, then
|
|
56
|
+
maps every executed line to its enclosing function via the AST. The
|
|
57
|
+
result is a method-level map `{test -> {file -> {qualnames}}}` saved
|
|
58
|
+
to `.tia/map.json`, stamped with the git ref it was recorded at.
|
|
59
|
+
Because coverage is dynamic, the whole call chain a test exercises
|
|
60
|
+
(controller → service → repo → utils) is captured for free.
|
|
61
|
+
2. **`tia run`** diffs your tree against that ref, reading the **old
|
|
62
|
+
side** of each hunk (same coordinate system as the map), then parses
|
|
63
|
+
each file *as it existed at that ref* (`git show`) to resolve changed
|
|
64
|
+
lines to changed functions.
|
|
65
|
+
3. It selects tests by three rules (see `tia/select.py`) and runs only
|
|
66
|
+
those via pytest.
|
|
67
|
+
|
|
68
|
+
## Selection rules
|
|
69
|
+
|
|
70
|
+
1. **Function hit** — a test executed a function whose body changed.
|
|
71
|
+
Immune to line shifts elsewhere in the file (the whole point of
|
|
72
|
+
going method-level).
|
|
73
|
+
2. **Module-level fallback** — a file had a module-level *modification*
|
|
74
|
+
(constant, import, class body). Runs every test touching that file.
|
|
75
|
+
Module-level *insertions* (a new function/test) are ignored so they
|
|
76
|
+
don't drag the whole file in.
|
|
77
|
+
3. **Data dependency** — a non-`.py` file changed (config, fixture,
|
|
78
|
+
template). Runs every test that *opened* that file while recording.
|
|
79
|
+
These reads are captured with a `sys.addaudithook` on the `open`
|
|
80
|
+
event, so dependencies coverage can't see don't become silent
|
|
81
|
+
false negatives.
|
|
82
|
+
4. **New test** — any collected test not in the map has never been
|
|
83
|
+
measured, so it always runs.
|
|
84
|
+
|
|
85
|
+
A **dynamic-safety modifier** sits on top of rules 1–2: a file flagged
|
|
86
|
+
at record time as using reflection (`getattr` by computed name, `eval`,
|
|
87
|
+
`importlib`, `__getattr__`) is widened from method-level to file-level
|
|
88
|
+
when it changes — coverage can't be trusted to have captured every edge
|
|
89
|
+
in/out of it. `--trust-dynamic` opts out. This is mitigation, not a
|
|
90
|
+
solution; nothing resolves dynamic dispatch precisely (it's undecidable
|
|
91
|
+
in general), so still run the full suite on a cadence.
|
|
92
|
+
|
|
93
|
+
A **cosmetic-change filter** sits *underneath* everything: before
|
|
94
|
+
selecting, a changed `.py` file whose edit is only comments, whitespace,
|
|
95
|
+
or docstrings is dropped entirely. The test is the AST — comments and
|
|
96
|
+
formatting aren't in it, docstrings are stripped before comparing — and
|
|
97
|
+
it compares the **old and new** trees, so uncommenting a line still
|
|
98
|
+
counts as real. So a "fix typo" / "add docstring" / "run black" commit on
|
|
99
|
+
a core file selects *nothing* instead of half the suite. `--all-changes`
|
|
100
|
+
opts out. Safe by construction: it only removes false positives.
|
|
101
|
+
|
|
102
|
+
The same filter also drops **type-only** edits — but only the kind that
|
|
103
|
+
is *provably* dead at runtime, because the common belief that "type hints
|
|
104
|
+
don't run" is false: `dataclasses`, `pydantic` and `attrs` all read
|
|
105
|
+
`__annotations__`. So it strips exactly two things: the body of an
|
|
106
|
+
`if TYPE_CHECKING:` block (never executed at runtime — where most hint
|
|
107
|
+
churn lives) and **function-local** annotations (PEP 526: never evaluated
|
|
108
|
+
nor stored). It deliberately *keeps* a change to a function signature or a
|
|
109
|
+
class/module-level annotation as real, because a dataclass field's type
|
|
110
|
+
genuinely changes behavior. Stripping those would be a false negative —
|
|
111
|
+
the one sin this tool won't commit. So "added the type hints" stops
|
|
112
|
+
triggering tests, without ever hiding a `dataclass` field-type change.
|
|
113
|
+
|
|
114
|
+
## Usage
|
|
115
|
+
|
|
116
|
+
```sh
|
|
117
|
+
pip install -e .
|
|
118
|
+
|
|
119
|
+
tia record [PATH] # build the map (run from the repo root)
|
|
120
|
+
tia run [PATH] # run only affected tests
|
|
121
|
+
tia run --since main # diff against another ref
|
|
122
|
+
tia run --list # show the selection, don't run
|
|
123
|
+
tia run --report - # also print a Markdown summary (see CI mode)
|
|
124
|
+
tia status # summarize the recorded map
|
|
125
|
+
|
|
126
|
+
tia serve --dir ./maps # run the bundled map store (zero deps)
|
|
127
|
+
tia push --to <dir|url> # publish the local map (for CI)
|
|
128
|
+
tia pull --from <dir|url> # fetch a published map
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
Remotes for `push`/`pull`/`--remote` can be a directory, an `http(s)://`
|
|
132
|
+
URL (the bundled store), or native object storage: `s3://bucket/prefix`
|
|
133
|
+
(`pip install pytest-tia[s3]`) or `gs://bucket/prefix`
|
|
134
|
+
(`pytest-tia[gcs]`). The cloud SDKs are imported lazily, so the core
|
|
135
|
+
install stays dependency-light unless you actually use those URLs.
|
|
136
|
+
|
|
137
|
+
Run from the repository root (where `pyproject.toml` / `.git` live) so
|
|
138
|
+
nodeids and file paths stay consistent.
|
|
139
|
+
|
|
140
|
+
### CI mode
|
|
141
|
+
|
|
142
|
+
The map a base-branch job builds has to reach the PR job that consumes
|
|
143
|
+
it. Publish it to a shared remote keyed by the git ref it was recorded
|
|
144
|
+
at. The remote can be a **directory** (a cache volume / artifact dir), an
|
|
145
|
+
**`http(s)://` URL** served by the bundled store, or a native
|
|
146
|
+
**`s3://` / `gs://`** bucket:
|
|
147
|
+
|
|
148
|
+
```sh
|
|
149
|
+
# one zero-dependency map store for the team / CI (stdlib only)
|
|
150
|
+
python -m tia.server --dir ./tia-maps --port 8000 # or: tia serve ...
|
|
151
|
+
|
|
152
|
+
# base branch job
|
|
153
|
+
tia record && tia push --to s3://my-company-ci/tia-maps
|
|
154
|
+
|
|
155
|
+
# PR job — no local map needed; pulls by base ref, falls back to latest
|
|
156
|
+
tia run --remote s3://my-company-ci/tia-maps --since "$(git merge-base origin/main HEAD)"
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
In a GitHub Actions PR job, `tia run` auto-detects `$GITHUB_STEP_SUMMARY`
|
|
160
|
+
and posts a Markdown table to the check page — which files changed, the
|
|
161
|
+
impact of each, and exactly which tests were selected and why. No flag
|
|
162
|
+
needed; it's the same explanation as the terminal output, rendered for
|
|
163
|
+
the PR. (`--report <path>`, or `--report -` for stdout, forces it
|
|
164
|
+
anywhere.)
|
|
165
|
+
|
|
166
|
+
A ready-to-copy GitHub Actions workflow is in
|
|
167
|
+
[`examples/ci/github-actions.yml`](examples/ci/github-actions.yml).
|
|
168
|
+
|
|
169
|
+
`run` resolves the diff against line→function tables **baked into the
|
|
170
|
+
map** at record time, so it never needs `git show` on the base blob —
|
|
171
|
+
which is what makes it safe under shallow clones (`clone --depth=1`),
|
|
172
|
+
where that blob may not be fetched. The diff itself still needs the base
|
|
173
|
+
*commit*; in a shallow checkout, fetch just that ref first
|
|
174
|
+
(`git fetch --depth=1 origin <base-sha>`).
|
|
175
|
+
|
|
176
|
+
## Known limitations (honest list)
|
|
177
|
+
|
|
178
|
+
- **Insertion anchoring.** Appending a function/test right after an
|
|
179
|
+
existing one anchors on that one's last line, pulling it in as one
|
|
180
|
+
extra test. A bounded false positive, never a false negative.
|
|
181
|
+
- **Coordinate sync.** The map is stamped with the ref it was recorded
|
|
182
|
+
at and `run` diffs against it automatically. Re-run `tia record`
|
|
183
|
+
after you commit so the map stays fresh.
|
|
184
|
+
- **Subprocess / other-thread execution isn't traced.** We attribute a
|
|
185
|
+
test's setup, call, and teardown (so fixture work and fixture file
|
|
186
|
+
reads count), but code that runs in a child process or a non-measured
|
|
187
|
+
thread is invisible to coverage and can hide a dependency.
|
|
188
|
+
- **Needs coverage's C tracer.** On Python 3.12+ coverage defaults to the
|
|
189
|
+
`sysmon` core, which records only the *first* test to hit each line and
|
|
190
|
+
silently drops the rest — a false negative for any shared helper. tia
|
|
191
|
+
forces `COVERAGE_CORE=ctrace` during recording to get correct per-test
|
|
192
|
+
contexts; don't override it back to `sysmon`.
|
|
193
|
+
- **Dynamic dispatch / reflection / subprocesses** aren't traced by
|
|
194
|
+
coverage and can hide a real dependency. tia *detects* reflection and
|
|
195
|
+
degrades to file-level there (see the dynamic-safety modifier), but a
|
|
196
|
+
`getattr` target in one file pointing at a function in another is still
|
|
197
|
+
beyond it. Re-record periodically and run the full suite on a cadence.
|
|
198
|
+
|
|
199
|
+
## Roadmap
|
|
200
|
+
|
|
201
|
+
- [x] **Method-level analysis** (AST) — done.
|
|
202
|
+
- [x] **Silent dependencies** — track non-`.py` files each test reads.
|
|
203
|
+
- [x] **CI mode** — remote map storage + shallow-clone-safe diffing.
|
|
204
|
+
- [x] **Static fallback** for dynamic dispatch / DI frameworks —
|
|
205
|
+
detect-and-degrade (mitigation, not a precise solution).
|
|
206
|
+
- [x] **Industrialize** — zero-dep HTTP map store (`tia serve`) + GitHub
|
|
207
|
+
Actions template, so adopting it is a few lines, not a project.
|
|
208
|
+
- [x] **Real-repo benchmark, both ends of the range** — Flask (worst
|
|
209
|
+
case, ~21%, [`RESULTS.md`](benchmark/RESULTS.md)) and boltons (modular,
|
|
210
|
+
~96% on logic changes, [`RESULTS-boltons.md`](benchmark/RESULTS-boltons.md)).
|
|
211
|
+
The Flask run also caught a recorder false-negative bug (the `sysmon`
|
|
212
|
+
core dropping contexts).
|
|
213
|
+
- [x] **Cosmetic-change filter** — ignore comment/docstring/format-only
|
|
214
|
+
edits so they select nothing instead of a whole file.
|
|
215
|
+
- [x] **Type-only filter (v1.1)** — also ignore `if TYPE_CHECKING:` and
|
|
216
|
+
function-local annotation edits; keep signature/dataclass-field type
|
|
217
|
+
changes (provably-safe subset only — no false negatives).
|
|
218
|
+
- [x] **Native S3 / GCS remotes (v1.1)** — `s3://` / `gs://` map stores
|
|
219
|
+
via lazily-imported SDKs; core install stays dependency-light.
|
|
220
|
+
- [x] **CI step summary (v1.1)** — auto-post a Markdown impact table to
|
|
221
|
+
the PR via `$GITHUB_STEP_SUMMARY`.
|
|
222
|
+
- [ ] **Not planned: precise cross-file dynamic dispatch.** Resolving a
|
|
223
|
+
`getattr`/`eval` target across files is undecidable in general; the
|
|
224
|
+
detect-and-degrade modifier plus periodic full runs is the honest
|
|
225
|
+
answer, not a static call graph that drowns in edge cases.
|
|
226
|
+
|
|
227
|
+
## Demo
|
|
228
|
+
|
|
229
|
+
`examples/calc/` is a tiny suite that proves the behavior end to end.
|
|
230
|
+
See the scenarios in the project history.
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
# tia — Test Impact Analysis for pytest
|
|
2
|
+
|
|
3
|
+
Run only the tests your changes actually affect. Big suites spend most
|
|
4
|
+
of their CI time re-running tests that couldn't possibly have broken;
|
|
5
|
+
`tia` builds a per-test coverage map once, then uses your `git diff` to
|
|
6
|
+
select the minimal set of tests to run.
|
|
7
|
+
|
|
8
|
+
This is the same idea Google/Meta run internally (Test Impact Analysis).
|
|
9
|
+
|
|
10
|
+
## Honest disclosure (read this first)
|
|
11
|
+
|
|
12
|
+
Test selection has exactly one cardinal sin: skipping a test that would
|
|
13
|
+
have failed. A tool you can't trust on that is worse than no tool. So
|
|
14
|
+
before any feature list, here is where this project earned — or failed to
|
|
15
|
+
earn — that trust.
|
|
16
|
+
|
|
17
|
+
While benchmarking on Flask, an early run reported a **median skip rate of
|
|
18
|
+
73%**. It looked great. It was a lie. coverage.py's default measurement
|
|
19
|
+
core on Python 3.12+ (`sysmon`) records only the *first* test to hit each
|
|
20
|
+
line and silently drops the rest, so any test that reused a shared helper
|
|
21
|
+
was never mapped — and changing that helper would have skipped them. The
|
|
22
|
+
high number was false negatives wearing a costume. We forced the C tracer
|
|
23
|
+
(`COVERAGE_CORE=ctrace`), every test mapped, and the number fell to the
|
|
24
|
+
truth: **median ~21% skip on Flask** (≈40% mean; cosmetic commits 99%+).
|
|
25
|
+
The toy demo could never have shown this — only a real codebase did.
|
|
26
|
+
|
|
27
|
+
So the Flask number is deliberately the *lower bound*: Flask is small and
|
|
28
|
+
tightly coupled, near the worst case for test selection. We'd rather
|
|
29
|
+
publish the honest floor than a cherry-picked ceiling. But a floor alone
|
|
30
|
+
is misleading too, so we measured the other end: on
|
|
31
|
+
[boltons](benchmark/RESULTS-boltons.md), a modular utility library, the
|
|
32
|
+
honest median skip on real logic changes is **~96%**. Same tool, same
|
|
33
|
+
rules — the variable is how decoupled your tests are. The truth is a
|
|
34
|
+
**range, ~21% (worst case) ↔ ~96% (modular)**, and both ends are measured
|
|
35
|
+
on real third-party suites, not toys. Per-commit tables and the "I
|
|
36
|
+
distrusted this number first" audits:
|
|
37
|
+
[Flask](benchmark/RESULTS.md) · [boltons](benchmark/RESULTS-boltons.md).
|
|
38
|
+
|
|
39
|
+
## How it works
|
|
40
|
+
|
|
41
|
+
1. **`tia record`** runs the full suite once with a pytest plugin that
|
|
42
|
+
switches coverage.py's *dynamic context* to each test's nodeid, then
|
|
43
|
+
maps every executed line to its enclosing function via the AST. The
|
|
44
|
+
result is a method-level map `{test -> {file -> {qualnames}}}` saved
|
|
45
|
+
to `.tia/map.json`, stamped with the git ref it was recorded at.
|
|
46
|
+
Because coverage is dynamic, the whole call chain a test exercises
|
|
47
|
+
(controller → service → repo → utils) is captured for free.
|
|
48
|
+
2. **`tia run`** diffs your tree against that ref, reading the **old
|
|
49
|
+
side** of each hunk (same coordinate system as the map), then parses
|
|
50
|
+
each file *as it existed at that ref* (`git show`) to resolve changed
|
|
51
|
+
lines to changed functions.
|
|
52
|
+
3. It selects tests by three rules (see `tia/select.py`) and runs only
|
|
53
|
+
those via pytest.
|
|
54
|
+
|
|
55
|
+
## Selection rules
|
|
56
|
+
|
|
57
|
+
1. **Function hit** — a test executed a function whose body changed.
|
|
58
|
+
Immune to line shifts elsewhere in the file (the whole point of
|
|
59
|
+
going method-level).
|
|
60
|
+
2. **Module-level fallback** — a file had a module-level *modification*
|
|
61
|
+
(constant, import, class body). Runs every test touching that file.
|
|
62
|
+
Module-level *insertions* (a new function/test) are ignored so they
|
|
63
|
+
don't drag the whole file in.
|
|
64
|
+
3. **Data dependency** — a non-`.py` file changed (config, fixture,
|
|
65
|
+
template). Runs every test that *opened* that file while recording.
|
|
66
|
+
These reads are captured with a `sys.addaudithook` on the `open`
|
|
67
|
+
event, so dependencies coverage can't see don't become silent
|
|
68
|
+
false negatives.
|
|
69
|
+
4. **New test** — any collected test not in the map has never been
|
|
70
|
+
measured, so it always runs.
|
|
71
|
+
|
|
72
|
+
A **dynamic-safety modifier** sits on top of rules 1–2: a file flagged
|
|
73
|
+
at record time as using reflection (`getattr` by computed name, `eval`,
|
|
74
|
+
`importlib`, `__getattr__`) is widened from method-level to file-level
|
|
75
|
+
when it changes — coverage can't be trusted to have captured every edge
|
|
76
|
+
in/out of it. `--trust-dynamic` opts out. This is mitigation, not a
|
|
77
|
+
solution; nothing resolves dynamic dispatch precisely (it's undecidable
|
|
78
|
+
in general), so still run the full suite on a cadence.
|
|
79
|
+
|
|
80
|
+
A **cosmetic-change filter** sits *underneath* everything: before
|
|
81
|
+
selecting, a changed `.py` file whose edit is only comments, whitespace,
|
|
82
|
+
or docstrings is dropped entirely. The test is the AST — comments and
|
|
83
|
+
formatting aren't in it, docstrings are stripped before comparing — and
|
|
84
|
+
it compares the **old and new** trees, so uncommenting a line still
|
|
85
|
+
counts as real. So a "fix typo" / "add docstring" / "run black" commit on
|
|
86
|
+
a core file selects *nothing* instead of half the suite. `--all-changes`
|
|
87
|
+
opts out. Safe by construction: it only removes false positives.
|
|
88
|
+
|
|
89
|
+
The same filter also drops **type-only** edits — but only the kind that
|
|
90
|
+
is *provably* dead at runtime, because the common belief that "type hints
|
|
91
|
+
don't run" is false: `dataclasses`, `pydantic` and `attrs` all read
|
|
92
|
+
`__annotations__`. So it strips exactly two things: the body of an
|
|
93
|
+
`if TYPE_CHECKING:` block (never executed at runtime — where most hint
|
|
94
|
+
churn lives) and **function-local** annotations (PEP 526: never evaluated
|
|
95
|
+
nor stored). It deliberately *keeps* a change to a function signature or a
|
|
96
|
+
class/module-level annotation as real, because a dataclass field's type
|
|
97
|
+
genuinely changes behavior. Stripping those would be a false negative —
|
|
98
|
+
the one sin this tool won't commit. So "added the type hints" stops
|
|
99
|
+
triggering tests, without ever hiding a `dataclass` field-type change.
|
|
100
|
+
|
|
101
|
+
## Usage
|
|
102
|
+
|
|
103
|
+
```sh
|
|
104
|
+
pip install -e .
|
|
105
|
+
|
|
106
|
+
tia record [PATH] # build the map (run from the repo root)
|
|
107
|
+
tia run [PATH] # run only affected tests
|
|
108
|
+
tia run --since main # diff against another ref
|
|
109
|
+
tia run --list # show the selection, don't run
|
|
110
|
+
tia run --report - # also print a Markdown summary (see CI mode)
|
|
111
|
+
tia status # summarize the recorded map
|
|
112
|
+
|
|
113
|
+
tia serve --dir ./maps # run the bundled map store (zero deps)
|
|
114
|
+
tia push --to <dir|url> # publish the local map (for CI)
|
|
115
|
+
tia pull --from <dir|url> # fetch a published map
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
Remotes for `push`/`pull`/`--remote` can be a directory, an `http(s)://`
|
|
119
|
+
URL (the bundled store), or native object storage: `s3://bucket/prefix`
|
|
120
|
+
(`pip install pytest-tia[s3]`) or `gs://bucket/prefix`
|
|
121
|
+
(`pytest-tia[gcs]`). The cloud SDKs are imported lazily, so the core
|
|
122
|
+
install stays dependency-light unless you actually use those URLs.
|
|
123
|
+
|
|
124
|
+
Run from the repository root (where `pyproject.toml` / `.git` live) so
|
|
125
|
+
nodeids and file paths stay consistent.
|
|
126
|
+
|
|
127
|
+
### CI mode
|
|
128
|
+
|
|
129
|
+
The map a base-branch job builds has to reach the PR job that consumes
|
|
130
|
+
it. Publish it to a shared remote keyed by the git ref it was recorded
|
|
131
|
+
at. The remote can be a **directory** (a cache volume / artifact dir), an
|
|
132
|
+
**`http(s)://` URL** served by the bundled store, or a native
|
|
133
|
+
**`s3://` / `gs://`** bucket:
|
|
134
|
+
|
|
135
|
+
```sh
|
|
136
|
+
# one zero-dependency map store for the team / CI (stdlib only)
|
|
137
|
+
python -m tia.server --dir ./tia-maps --port 8000 # or: tia serve ...
|
|
138
|
+
|
|
139
|
+
# base branch job
|
|
140
|
+
tia record && tia push --to s3://my-company-ci/tia-maps
|
|
141
|
+
|
|
142
|
+
# PR job — no local map needed; pulls by base ref, falls back to latest
|
|
143
|
+
tia run --remote s3://my-company-ci/tia-maps --since "$(git merge-base origin/main HEAD)"
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
In a GitHub Actions PR job, `tia run` auto-detects `$GITHUB_STEP_SUMMARY`
|
|
147
|
+
and posts a Markdown table to the check page — which files changed, the
|
|
148
|
+
impact of each, and exactly which tests were selected and why. No flag
|
|
149
|
+
needed; it's the same explanation as the terminal output, rendered for
|
|
150
|
+
the PR. (`--report <path>`, or `--report -` for stdout, forces it
|
|
151
|
+
anywhere.)
|
|
152
|
+
|
|
153
|
+
A ready-to-copy GitHub Actions workflow is in
|
|
154
|
+
[`examples/ci/github-actions.yml`](examples/ci/github-actions.yml).
|
|
155
|
+
|
|
156
|
+
`run` resolves the diff against line→function tables **baked into the
|
|
157
|
+
map** at record time, so it never needs `git show` on the base blob —
|
|
158
|
+
which is what makes it safe under shallow clones (`clone --depth=1`),
|
|
159
|
+
where that blob may not be fetched. The diff itself still needs the base
|
|
160
|
+
*commit*; in a shallow checkout, fetch just that ref first
|
|
161
|
+
(`git fetch --depth=1 origin <base-sha>`).
|
|
162
|
+
|
|
163
|
+
## Known limitations (honest list)
|
|
164
|
+
|
|
165
|
+
- **Insertion anchoring.** Appending a function/test right after an
|
|
166
|
+
existing one anchors on that one's last line, pulling it in as one
|
|
167
|
+
extra test. A bounded false positive, never a false negative.
|
|
168
|
+
- **Coordinate sync.** The map is stamped with the ref it was recorded
|
|
169
|
+
at and `run` diffs against it automatically. Re-run `tia record`
|
|
170
|
+
after you commit so the map stays fresh.
|
|
171
|
+
- **Subprocess / other-thread execution isn't traced.** We attribute a
|
|
172
|
+
test's setup, call, and teardown (so fixture work and fixture file
|
|
173
|
+
reads count), but code that runs in a child process or a non-measured
|
|
174
|
+
thread is invisible to coverage and can hide a dependency.
|
|
175
|
+
- **Needs coverage's C tracer.** On Python 3.12+ coverage defaults to the
|
|
176
|
+
`sysmon` core, which records only the *first* test to hit each line and
|
|
177
|
+
silently drops the rest — a false negative for any shared helper. tia
|
|
178
|
+
forces `COVERAGE_CORE=ctrace` during recording to get correct per-test
|
|
179
|
+
contexts; don't override it back to `sysmon`.
|
|
180
|
+
- **Dynamic dispatch / reflection / subprocesses** aren't traced by
|
|
181
|
+
coverage and can hide a real dependency. tia *detects* reflection and
|
|
182
|
+
degrades to file-level there (see the dynamic-safety modifier), but a
|
|
183
|
+
`getattr` target in one file pointing at a function in another is still
|
|
184
|
+
beyond it. Re-record periodically and run the full suite on a cadence.
|
|
185
|
+
|
|
186
|
+
## Roadmap
|
|
187
|
+
|
|
188
|
+
- [x] **Method-level analysis** (AST) — done.
|
|
189
|
+
- [x] **Silent dependencies** — track non-`.py` files each test reads.
|
|
190
|
+
- [x] **CI mode** — remote map storage + shallow-clone-safe diffing.
|
|
191
|
+
- [x] **Static fallback** for dynamic dispatch / DI frameworks —
|
|
192
|
+
detect-and-degrade (mitigation, not a precise solution).
|
|
193
|
+
- [x] **Industrialize** — zero-dep HTTP map store (`tia serve`) + GitHub
|
|
194
|
+
Actions template, so adopting it is a few lines, not a project.
|
|
195
|
+
- [x] **Real-repo benchmark, both ends of the range** — Flask (worst
|
|
196
|
+
case, ~21%, [`RESULTS.md`](benchmark/RESULTS.md)) and boltons (modular,
|
|
197
|
+
~96% on logic changes, [`RESULTS-boltons.md`](benchmark/RESULTS-boltons.md)).
|
|
198
|
+
The Flask run also caught a recorder false-negative bug (the `sysmon`
|
|
199
|
+
core dropping contexts).
|
|
200
|
+
- [x] **Cosmetic-change filter** — ignore comment/docstring/format-only
|
|
201
|
+
edits so they select nothing instead of a whole file.
|
|
202
|
+
- [x] **Type-only filter (v1.1)** — also ignore `if TYPE_CHECKING:` and
|
|
203
|
+
function-local annotation edits; keep signature/dataclass-field type
|
|
204
|
+
changes (provably-safe subset only — no false negatives).
|
|
205
|
+
- [x] **Native S3 / GCS remotes (v1.1)** — `s3://` / `gs://` map stores
|
|
206
|
+
via lazily-imported SDKs; core install stays dependency-light.
|
|
207
|
+
- [x] **CI step summary (v1.1)** — auto-post a Markdown impact table to
|
|
208
|
+
the PR via `$GITHUB_STEP_SUMMARY`.
|
|
209
|
+
- [ ] **Not planned: precise cross-file dynamic dispatch.** Resolving a
|
|
210
|
+
`getattr`/`eval` target across files is undecidable in general; the
|
|
211
|
+
detect-and-degrade modifier plus periodic full runs is the honest
|
|
212
|
+
answer, not a static call graph that drowns in edge cases.
|
|
213
|
+
|
|
214
|
+
## Demo
|
|
215
|
+
|
|
216
|
+
`examples/calc/` is a tiny suite that proves the behavior end to end.
|
|
217
|
+
See the scenarios in the project history.
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
# Distribution name is pytest-tia (PyPI: `tia` is taken; the pytest- prefix
|
|
3
|
+
# is the ecosystem convention). The command and import stay `tia`, like
|
|
4
|
+
# pytest-cov installs but you type `--cov`.
|
|
5
|
+
name = "pytest-tia"
|
|
6
|
+
version = "1.1.0"
|
|
7
|
+
description = "Test Impact Analysis for pytest — run only the tests your changes affect"
|
|
8
|
+
readme = "README.md"
|
|
9
|
+
requires-python = ">=3.10"
|
|
10
|
+
dependencies = ["pytest>=7", "coverage>=7"]
|
|
11
|
+
|
|
12
|
+
[project.optional-dependencies]
|
|
13
|
+
# Only needed if you point a remote at an s3:// or gs:// URL. The core
|
|
14
|
+
# install stays dependency-light; these are imported lazily on demand.
|
|
15
|
+
s3 = ["boto3>=1.26"]
|
|
16
|
+
gcs = ["google-cloud-storage>=2.0"]
|
|
17
|
+
|
|
18
|
+
[project.scripts]
|
|
19
|
+
tia = "tia.cli:main"
|
|
20
|
+
|
|
21
|
+
[build-system]
|
|
22
|
+
requires = ["setuptools>=61"]
|
|
23
|
+
build-backend = "setuptools.build_meta"
|
|
24
|
+
|
|
25
|
+
[tool.setuptools]
|
|
26
|
+
packages = ["tia"]
|