pytest-tzcheck 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pytest_tzcheck-0.1.0/.github/workflows/ci.yml +29 -0
- pytest_tzcheck-0.1.0/.github/workflows/release.yml +41 -0
- pytest_tzcheck-0.1.0/.gitignore +7 -0
- pytest_tzcheck-0.1.0/LICENSE +21 -0
- pytest_tzcheck-0.1.0/PKG-INFO +148 -0
- pytest_tzcheck-0.1.0/README.md +128 -0
- pytest_tzcheck-0.1.0/pyproject.toml +45 -0
- pytest_tzcheck-0.1.0/src/pytest_tzcheck/__init__.py +1 -0
- pytest_tzcheck-0.1.0/src/pytest_tzcheck/plugin.py +164 -0
- pytest_tzcheck-0.1.0/src/pytest_tzcheck/zones.py +118 -0
- pytest_tzcheck-0.1.0/tests/conftest.py +23 -0
- pytest_tzcheck-0.1.0/tests/test_plugin.py +187 -0
- pytest_tzcheck-0.1.0/tests/test_zones.py +67 -0
- pytest_tzcheck-0.1.0/uv.lock +289 -0
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
lint:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
steps:
|
|
12
|
+
- uses: actions/checkout@v4
|
|
13
|
+
- uses: astral-sh/setup-uv@v6
|
|
14
|
+
- run: uv run ruff check .
|
|
15
|
+
- run: uv run ruff format --check .
|
|
16
|
+
|
|
17
|
+
test:
|
|
18
|
+
runs-on: ${{ matrix.os }}
|
|
19
|
+
strategy:
|
|
20
|
+
fail-fast: false
|
|
21
|
+
matrix:
|
|
22
|
+
os: [ubuntu-latest, macos-latest]
|
|
23
|
+
python: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
|
|
24
|
+
steps:
|
|
25
|
+
- uses: actions/checkout@v4
|
|
26
|
+
- uses: astral-sh/setup-uv@v6
|
|
27
|
+
with:
|
|
28
|
+
python-version: ${{ matrix.python }}
|
|
29
|
+
- run: uv run pytest
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags: ["v*"]
|
|
6
|
+
|
|
7
|
+
jobs:
|
|
8
|
+
build:
|
|
9
|
+
runs-on: ubuntu-latest
|
|
10
|
+
steps:
|
|
11
|
+
- uses: actions/checkout@v4
|
|
12
|
+
- uses: astral-sh/setup-uv@v6
|
|
13
|
+
- name: Check the tag matches the package version
|
|
14
|
+
run: |
|
|
15
|
+
version=$(uv version --short)
|
|
16
|
+
if [ "v$version" != "$GITHUB_REF_NAME" ]; then
|
|
17
|
+
echo "::error::tag $GITHUB_REF_NAME does not match pyproject version $version"
|
|
18
|
+
exit 1
|
|
19
|
+
fi
|
|
20
|
+
- run: uv run ruff check .
|
|
21
|
+
- run: uv run pytest
|
|
22
|
+
- run: uv build
|
|
23
|
+
- uses: actions/upload-artifact@v4
|
|
24
|
+
with:
|
|
25
|
+
name: dist
|
|
26
|
+
path: dist/
|
|
27
|
+
|
|
28
|
+
publish:
|
|
29
|
+
needs: build
|
|
30
|
+
runs-on: ubuntu-latest
|
|
31
|
+
environment:
|
|
32
|
+
name: pypi
|
|
33
|
+
url: https://pypi.org/project/pytest-tzcheck/
|
|
34
|
+
permissions:
|
|
35
|
+
id-token: write
|
|
36
|
+
steps:
|
|
37
|
+
- uses: actions/download-artifact@v4
|
|
38
|
+
with:
|
|
39
|
+
name: dist
|
|
40
|
+
path: dist/
|
|
41
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ariel Even
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: pytest-tzcheck
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Run your tests in the timezone CI uses, and find the ones that only pass in yours.
|
|
5
|
+
Project-URL: Repository, https://github.com/taltal-beep/pytest-tzcheck
|
|
6
|
+
Author-email: Ariel Even <ttlgptinc@gmail.com>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Keywords: flaky,pytest,testing,timezone,tz
|
|
10
|
+
Classifier: Framework :: Pytest
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Operating System :: MacOS
|
|
14
|
+
Classifier: Operating System :: POSIX
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Topic :: Software Development :: Testing
|
|
17
|
+
Requires-Python: >=3.9
|
|
18
|
+
Requires-Dist: pytest>=7
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
|
|
21
|
+
# pytest-tzcheck
|
|
22
|
+
|
|
23
|
+
Run your tests in the timezone CI uses, and find the ones that only pass in yours.
|
|
24
|
+
|
|
25
|
+
"Passes on my machine, fails in CI" often comes down to the local timezone: your laptop is in Asia/Jerusalem, the CI runner is in UTC. `pytest --tzcheck` runs every test in both zones and tells you which ones change outcome, before you push.
|
|
26
|
+
|
|
27
|
+
```
|
|
28
|
+
$ pytest --tzcheck tests/server/schemas/test_schedules.py
|
|
29
|
+
tzcheck: local (Asia/Jerusalem), UTC
|
|
30
|
+
...
|
|
31
|
+
=========================== timezone-dependent tests ===========================
|
|
32
|
+
tests/server/schemas/test_schedules.py::TestRRuleSchedule::test_rrule_schedule_preserves_and_localizes_rdates passed: local (Asia/Jerusalem) failed: UTC
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
## Install
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
pip install pytest-tzcheck
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Python 3.9+, pytest 7+, Linux or macOS (it needs `time.tzset()`, which Windows does not have).
|
|
42
|
+
|
|
43
|
+
## Usage
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
pytest --tzcheck # local zone + UTC (the "ci" preset)
|
|
47
|
+
pytest --tzcheck --tzcheck-zones=wide # local, UTC, +14h, half-hour offsets, southern DST
|
|
48
|
+
pytest --tzcheck --tzcheck-zones=UTC,America/New_York
|
|
49
|
+
pytest --tzcheck -k "tz=UTC" # only the UTC runs
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
Or turn it on in config:
|
|
53
|
+
|
|
54
|
+
```ini
|
|
55
|
+
[pytest]
|
|
56
|
+
tzcheck = true
|
|
57
|
+
tzcheck_zones = ci
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
| Zone list | Zones |
|
|
61
|
+
|---|---|
|
|
62
|
+
| `ci` (default) | `local`, `UTC` |
|
|
63
|
+
| `wide` | `local`, `UTC`, `Pacific/Kiritimati` (UTC+14, the local date is not UTC's most of the day), `America/St_Johns` (half-hour offset), `Australia/Lord_Howe` (half-hour DST shift, southern hemisphere) |
|
|
64
|
+
|
|
65
|
+
`local` is the environment as you started pytest. Presets and IANA names can be mixed. Zones with the same UTC offsets all year are merged, so on a UTC CI runner `--tzcheck` does not double the run.
|
|
66
|
+
|
|
67
|
+
Opt a test out with `@pytest.mark.no_tzcheck`. A test can ask which zone it runs in through the `tzcheck_zone` fixture.
|
|
68
|
+
|
|
69
|
+
A failure in any zone fails the run, so `--tzcheck` works as a CI gate as well as a local check.
|
|
70
|
+
|
|
71
|
+
## How it works
|
|
72
|
+
|
|
73
|
+
1. Each test is parametrized over the zones (`test_x[tz=local]`, `test_x[tz=UTC]`), so the runs are ordinary pytest items: `-k`, `--lf`, JUnit XML and pytest-xdist all work.
|
|
74
|
+
2. Around each run (setup, call and teardown) the plugin sets `TZ`, calls `time.tzset()`, and restores both afterwards. Subprocesses started by the test inherit the zone. If `tzlocal` is imported, its cache is reloaded.
|
|
75
|
+
3. The outcome of each run is recorded, and the summary lists every test whose outcome differs between zones. A test that fails everywhere is not listed: that is a bug, but not a timezone bug.
|
|
76
|
+
|
|
77
|
+
## Case study: a prefect PR
|
|
78
|
+
|
|
79
|
+
This plugin started from a real bug. While fixing RDATE/EXDATE handling in prefect's `RRuleSchedule` ([PrefectHQ/prefect#23436](https://github.com/PrefectHQ/prefect/pull/23436)), the new regression tests passed locally (Asia/Jerusalem) and failed in CI (UTC).
|
|
80
|
+
|
|
81
|
+
The cause: dateutil parses a timestamp ending in `Z` as `tzutc()`, unless the machine's local zone is UTC, in which case it returns `tzlocal()`. The fix checked `isinstance(dt.tzinfo, tzutc)`, which is true in Jerusalem and false in UTC. Nothing in that code path calls a local-time function; dateutil just compares against `time.tzname`.
|
|
82
|
+
|
|
83
|
+
Prefect's schedule tests on the commit that failed CI (`bb4f72475`):
|
|
84
|
+
|
|
85
|
+
```
|
|
86
|
+
$ pytest tests/client/schemas/test_schedules.py tests/server/schemas/test_schedules.py
|
|
87
|
+
155 passed, 6 skipped, 2 xfailed
|
|
88
|
+
|
|
89
|
+
$ pytest --tzcheck tests/client/schemas/test_schedules.py tests/server/schemas/test_schedules.py
|
|
90
|
+
=========================== timezone-dependent tests ===========================
|
|
91
|
+
tests/client/schemas/test_schedules.py::TestRRuleScheduleFromRRule::test_rruleset_keeps_rdate_and_exdate_instants_in_another_timezone passed: local (Asia/Jerusalem) failed: UTC
|
|
92
|
+
tests/server/schemas/test_schedules.py::TestRRuleSchedule::test_rrule_schedule_preserves_and_localizes_exdates passed: local (Asia/Jerusalem) failed: UTC
|
|
93
|
+
tests/server/schemas/test_schedules.py::TestRRuleSchedule::test_rrule_schedule_preserves_and_localizes_rdates passed: local (Asia/Jerusalem) failed: UTC
|
|
94
|
+
tests/server/schemas/test_schedules.py::TestRRuleSchedule::test_serialization_preserves_rdate_and_exdate_in_another_timezone passed: local (Asia/Jerusalem) failed: UTC
|
|
95
|
+
tests/server/schemas/test_schedules.py::TestRRuleSchedule::test_serialization_preserves_rrules_rdates_exrules_exdates passed: local (Asia/Jerusalem) failed: UTC
|
|
96
|
+
5 of 163 tests changed outcome with the timezone
|
|
97
|
+
5 failed, 305 passed, 12 skipped, 4 xfailed
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
On the fixed commit (`9e24d058d`, which checks the UTC offset instead of the tzinfo type):
|
|
101
|
+
|
|
102
|
+
```
|
|
103
|
+
$ pytest --tzcheck tests/client/schemas/test_schedules.py tests/server/schemas/test_schedules.py
|
|
104
|
+
tzcheck: no timezone-dependent tests (163 tests x 2 zones)
|
|
105
|
+
310 passed, 12 skipped, 4 xfailed
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
## Why run every test, rather than detect the risky ones?
|
|
109
|
+
|
|
110
|
+
The cheaper design is to trace calls to local-time APIs (`time.localtime`, naive `datetime.now()`, `astimezone()` without arguments) and rerun only the tests that make them. It would have missed the bug above: in Jerusalem, dateutil only *reads* `time.tzname`, an attribute, so no call is made. Running each test in each zone and comparing outcomes has no false positives and no blind spots, at the cost of one extra run per zone. Linters such as ruff's `DTZ` rules cover the static side.
|
|
111
|
+
|
|
112
|
+
## Limitations
|
|
113
|
+
|
|
114
|
+
- Fixtures with module, class or session scope are created once, in the zone of the first test that uses them. Values computed at import time (a module-level `tz.tzlocal()`, a cached `get_localzone()` result) also stay in the first zone.
|
|
115
|
+
- The zone is per process: threads running during the test see it too.
|
|
116
|
+
- POSIX only.
|
|
117
|
+
- libc silently treats an unknown `TZ` as UTC, so the plugin refuses zone names that are not in the system tz database.
|
|
118
|
+
|
|
119
|
+
## Roadmap
|
|
120
|
+
|
|
121
|
+
- `--tzcheck-mode=probe`: trace local-time calls with `sys.monitoring` (Python 3.12+) and rerun only those tests in the extra zones, as a fast pre-filter for large suites, with the blind spot above documented.
|
|
122
|
+
- Run each zone at a few fixed instants (around midnight UTC, DST transitions), since many timezone bugs only show up at certain hours.
|
|
123
|
+
- A JSON report for CI annotations.
|
|
124
|
+
|
|
125
|
+
## Development
|
|
126
|
+
|
|
127
|
+
```bash
|
|
128
|
+
uv sync
|
|
129
|
+
uv run pytest
|
|
130
|
+
uv run ruff check . && uv run ruff format --check .
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
To try it on another project without installing it: `PYTHONPATH=path/to/pytest-tzcheck/src pytest -p pytest_tzcheck.plugin --tzcheck`.
|
|
134
|
+
|
|
135
|
+
## Releasing
|
|
136
|
+
|
|
137
|
+
Pushing a `v*` tag runs `.github/workflows/release.yml`: it checks the tag matches the version in `pyproject.toml`, runs lint and tests, builds, and publishes to PyPI through trusted publishing (no API token).
|
|
138
|
+
|
|
139
|
+
```bash
|
|
140
|
+
uv version --bump patch # or minor / major
|
|
141
|
+
git commit -am "Release $(uv version --short)"
|
|
142
|
+
git tag "v$(uv version --short)"
|
|
143
|
+
git push origin main --tags
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
## License
|
|
147
|
+
|
|
148
|
+
MIT
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
# pytest-tzcheck
|
|
2
|
+
|
|
3
|
+
Run your tests in the timezone CI uses, and find the ones that only pass in yours.
|
|
4
|
+
|
|
5
|
+
"Passes on my machine, fails in CI" often comes down to the local timezone: your laptop is in Asia/Jerusalem, the CI runner is in UTC. `pytest --tzcheck` runs every test in both zones and tells you which ones change outcome, before you push.
|
|
6
|
+
|
|
7
|
+
```
|
|
8
|
+
$ pytest --tzcheck tests/server/schemas/test_schedules.py
|
|
9
|
+
tzcheck: local (Asia/Jerusalem), UTC
|
|
10
|
+
...
|
|
11
|
+
=========================== timezone-dependent tests ===========================
|
|
12
|
+
tests/server/schemas/test_schedules.py::TestRRuleSchedule::test_rrule_schedule_preserves_and_localizes_rdates passed: local (Asia/Jerusalem) failed: UTC
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
## Install
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
pip install pytest-tzcheck
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
Python 3.9+, pytest 7+, Linux or macOS (it needs `time.tzset()`, which Windows does not have).
|
|
22
|
+
|
|
23
|
+
## Usage
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
pytest --tzcheck # local zone + UTC (the "ci" preset)
|
|
27
|
+
pytest --tzcheck --tzcheck-zones=wide # local, UTC, +14h, half-hour offsets, southern DST
|
|
28
|
+
pytest --tzcheck --tzcheck-zones=UTC,America/New_York
|
|
29
|
+
pytest --tzcheck -k "tz=UTC" # only the UTC runs
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
Or turn it on in config:
|
|
33
|
+
|
|
34
|
+
```ini
|
|
35
|
+
[pytest]
|
|
36
|
+
tzcheck = true
|
|
37
|
+
tzcheck_zones = ci
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
| Zone list | Zones |
|
|
41
|
+
|---|---|
|
|
42
|
+
| `ci` (default) | `local`, `UTC` |
|
|
43
|
+
| `wide` | `local`, `UTC`, `Pacific/Kiritimati` (UTC+14, the local date is not UTC's most of the day), `America/St_Johns` (half-hour offset), `Australia/Lord_Howe` (half-hour DST shift, southern hemisphere) |
|
|
44
|
+
|
|
45
|
+
`local` is the environment as you started pytest. Presets and IANA names can be mixed. Zones with the same UTC offsets all year are merged, so on a UTC CI runner `--tzcheck` does not double the run.
|
|
46
|
+
|
|
47
|
+
Opt a test out with `@pytest.mark.no_tzcheck`. A test can ask which zone it runs in through the `tzcheck_zone` fixture.
|
|
48
|
+
|
|
49
|
+
A failure in any zone fails the run, so `--tzcheck` works as a CI gate as well as a local check.
|
|
50
|
+
|
|
51
|
+
## How it works
|
|
52
|
+
|
|
53
|
+
1. Each test is parametrized over the zones (`test_x[tz=local]`, `test_x[tz=UTC]`), so the runs are ordinary pytest items: `-k`, `--lf`, JUnit XML and pytest-xdist all work.
|
|
54
|
+
2. Around each run (setup, call and teardown) the plugin sets `TZ`, calls `time.tzset()`, and restores both afterwards. Subprocesses started by the test inherit the zone. If `tzlocal` is imported, its cache is reloaded.
|
|
55
|
+
3. The outcome of each run is recorded, and the summary lists every test whose outcome differs between zones. A test that fails everywhere is not listed: that is a bug, but not a timezone bug.
|
|
56
|
+
|
|
57
|
+
## Case study: a prefect PR
|
|
58
|
+
|
|
59
|
+
This plugin started from a real bug. While fixing RDATE/EXDATE handling in prefect's `RRuleSchedule` ([PrefectHQ/prefect#23436](https://github.com/PrefectHQ/prefect/pull/23436)), the new regression tests passed locally (Asia/Jerusalem) and failed in CI (UTC).
|
|
60
|
+
|
|
61
|
+
The cause: dateutil parses a timestamp ending in `Z` as `tzutc()`, unless the machine's local zone is UTC, in which case it returns `tzlocal()`. The fix checked `isinstance(dt.tzinfo, tzutc)`, which is true in Jerusalem and false in UTC. Nothing in that code path calls a local-time function; dateutil just compares against `time.tzname`.
|
|
62
|
+
|
|
63
|
+
Prefect's schedule tests on the commit that failed CI (`bb4f72475`):
|
|
64
|
+
|
|
65
|
+
```
|
|
66
|
+
$ pytest tests/client/schemas/test_schedules.py tests/server/schemas/test_schedules.py
|
|
67
|
+
155 passed, 6 skipped, 2 xfailed
|
|
68
|
+
|
|
69
|
+
$ pytest --tzcheck tests/client/schemas/test_schedules.py tests/server/schemas/test_schedules.py
|
|
70
|
+
=========================== timezone-dependent tests ===========================
|
|
71
|
+
tests/client/schemas/test_schedules.py::TestRRuleScheduleFromRRule::test_rruleset_keeps_rdate_and_exdate_instants_in_another_timezone passed: local (Asia/Jerusalem) failed: UTC
|
|
72
|
+
tests/server/schemas/test_schedules.py::TestRRuleSchedule::test_rrule_schedule_preserves_and_localizes_exdates passed: local (Asia/Jerusalem) failed: UTC
|
|
73
|
+
tests/server/schemas/test_schedules.py::TestRRuleSchedule::test_rrule_schedule_preserves_and_localizes_rdates passed: local (Asia/Jerusalem) failed: UTC
|
|
74
|
+
tests/server/schemas/test_schedules.py::TestRRuleSchedule::test_serialization_preserves_rdate_and_exdate_in_another_timezone passed: local (Asia/Jerusalem) failed: UTC
|
|
75
|
+
tests/server/schemas/test_schedules.py::TestRRuleSchedule::test_serialization_preserves_rrules_rdates_exrules_exdates passed: local (Asia/Jerusalem) failed: UTC
|
|
76
|
+
5 of 163 tests changed outcome with the timezone
|
|
77
|
+
5 failed, 305 passed, 12 skipped, 4 xfailed
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
On the fixed commit (`9e24d058d`, which checks the UTC offset instead of the tzinfo type):
|
|
81
|
+
|
|
82
|
+
```
|
|
83
|
+
$ pytest --tzcheck tests/client/schemas/test_schedules.py tests/server/schemas/test_schedules.py
|
|
84
|
+
tzcheck: no timezone-dependent tests (163 tests x 2 zones)
|
|
85
|
+
310 passed, 12 skipped, 4 xfailed
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
## Why run every test, rather than detect the risky ones?
|
|
89
|
+
|
|
90
|
+
The cheaper design is to trace calls to local-time APIs (`time.localtime`, naive `datetime.now()`, `astimezone()` without arguments) and rerun only the tests that make them. It would have missed the bug above: in Jerusalem, dateutil only *reads* `time.tzname`, an attribute, so no call is made. Running each test in each zone and comparing outcomes has no false positives and no blind spots, at the cost of one extra run per zone. Linters such as ruff's `DTZ` rules cover the static side.
|
|
91
|
+
|
|
92
|
+
## Limitations
|
|
93
|
+
|
|
94
|
+
- Fixtures with module, class or session scope are created once, in the zone of the first test that uses them. Values computed at import time (a module-level `tz.tzlocal()`, a cached `get_localzone()` result) also stay in the first zone.
|
|
95
|
+
- The zone is per process: threads running during the test see it too.
|
|
96
|
+
- POSIX only.
|
|
97
|
+
- libc silently treats an unknown `TZ` as UTC, so the plugin refuses zone names that are not in the system tz database.
|
|
98
|
+
|
|
99
|
+
## Roadmap
|
|
100
|
+
|
|
101
|
+
- `--tzcheck-mode=probe`: trace local-time calls with `sys.monitoring` (Python 3.12+) and rerun only those tests in the extra zones, as a fast pre-filter for large suites, with the blind spot above documented.
|
|
102
|
+
- Run each zone at a few fixed instants (around midnight UTC, DST transitions), since many timezone bugs only show up at certain hours.
|
|
103
|
+
- A JSON report for CI annotations.
|
|
104
|
+
|
|
105
|
+
## Development
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
uv sync
|
|
109
|
+
uv run pytest
|
|
110
|
+
uv run ruff check . && uv run ruff format --check .
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
To try it on another project without installing it: `PYTHONPATH=path/to/pytest-tzcheck/src pytest -p pytest_tzcheck.plugin --tzcheck`.
|
|
114
|
+
|
|
115
|
+
## Releasing
|
|
116
|
+
|
|
117
|
+
Pushing a `v*` tag runs `.github/workflows/release.yml`: it checks the tag matches the version in `pyproject.toml`, runs lint and tests, builds, and publishes to PyPI through trusted publishing (no API token).
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
uv version --bump patch # or minor / major
|
|
121
|
+
git commit -am "Release $(uv version --short)"
|
|
122
|
+
git tag "v$(uv version --short)"
|
|
123
|
+
git push origin main --tags
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
## License
|
|
127
|
+
|
|
128
|
+
MIT
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "pytest-tzcheck"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Run your tests in the timezone CI uses, and find the ones that only pass in yours."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
authors = [{ name = "Ariel Even", email = "ttlgptinc@gmail.com" }]
|
|
13
|
+
requires-python = ">=3.9"
|
|
14
|
+
dependencies = ["pytest>=7"]
|
|
15
|
+
keywords = ["pytest", "timezone", "tz", "flaky", "testing"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Framework :: Pytest",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"License :: OSI Approved :: MIT License",
|
|
20
|
+
"Operating System :: POSIX",
|
|
21
|
+
"Operating System :: MacOS",
|
|
22
|
+
"Programming Language :: Python :: 3",
|
|
23
|
+
"Topic :: Software Development :: Testing",
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
[project.urls]
|
|
27
|
+
Repository = "https://github.com/taltal-beep/pytest-tzcheck"
|
|
28
|
+
|
|
29
|
+
[project.entry-points.pytest11]
|
|
30
|
+
tzcheck = "pytest_tzcheck.plugin"
|
|
31
|
+
|
|
32
|
+
[dependency-groups]
|
|
33
|
+
dev = ["pytest-xdist", "ruff"]
|
|
34
|
+
|
|
35
|
+
[tool.pytest.ini_options]
|
|
36
|
+
testpaths = ["tests"]
|
|
37
|
+
addopts = "-p pytester"
|
|
38
|
+
|
|
39
|
+
[tool.ruff]
|
|
40
|
+
target-version = "py39"
|
|
41
|
+
line-length = 100
|
|
42
|
+
|
|
43
|
+
[tool.ruff.lint]
|
|
44
|
+
select = ["E", "F", "I", "UP", "B", "DTZ"]
|
|
45
|
+
ignore = ["DTZ001", "DTZ005"]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Run tests in several timezones and report the ones whose outcome changes."""
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
"""pytest plugin: run each test in several timezones, report tests whose outcome changes."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections import defaultdict
|
|
6
|
+
|
|
7
|
+
import pytest
|
|
8
|
+
|
|
9
|
+
from pytest_tzcheck import zones
|
|
10
|
+
|
|
11
|
+
FIXTURE = "tzcheck_zone"
|
|
12
|
+
PROP_ZONE = "tzcheck_zone"
|
|
13
|
+
PROP_BASE = "tzcheck_base_nodeid"
|
|
14
|
+
|
|
15
|
+
# Worst outcome wins when a test has several phases (setup, call, teardown).
|
|
16
|
+
_RANK = {"passed": 0, "xfailed": 1, "xpassed": 1, "skipped": 2, "failed": 3, "error": 4}
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def pytest_addoption(parser: pytest.Parser) -> None:
|
|
20
|
+
group = parser.getgroup("tzcheck", "timezone dependence")
|
|
21
|
+
group.addoption(
|
|
22
|
+
"--tzcheck",
|
|
23
|
+
action="store_true",
|
|
24
|
+
default=None,
|
|
25
|
+
help="run each test in several timezones and report tests whose outcome depends on it.",
|
|
26
|
+
)
|
|
27
|
+
group.addoption(
|
|
28
|
+
"--tzcheck-zones",
|
|
29
|
+
metavar="ZONES",
|
|
30
|
+
help=f"comma-separated IANA zones or presets ({', '.join(zones.PRESETS)}). "
|
|
31
|
+
f"'{zones.LOCAL}' is the current zone. Default: ci ({','.join(zones.PRESETS['ci'])}).",
|
|
32
|
+
)
|
|
33
|
+
parser.addini("tzcheck", "enable --tzcheck by default", type="bool", default=False)
|
|
34
|
+
parser.addini("tzcheck_zones", "zones for --tzcheck (see --tzcheck-zones)", default="ci")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def pytest_configure(config: pytest.Config) -> None:
|
|
38
|
+
config.addinivalue_line(
|
|
39
|
+
"markers",
|
|
40
|
+
"no_tzcheck: run this test only in the local timezone, even with --tzcheck.",
|
|
41
|
+
)
|
|
42
|
+
enabled = config.getoption("tzcheck")
|
|
43
|
+
if enabled is None:
|
|
44
|
+
enabled = config.getini("tzcheck")
|
|
45
|
+
if not enabled:
|
|
46
|
+
return
|
|
47
|
+
if not zones.supported():
|
|
48
|
+
raise pytest.UsageError("--tzcheck needs time.tzset(), which this platform does not have.")
|
|
49
|
+
names = zones.parse(config.getoption("tzcheck_zones") or config.getini("tzcheck_zones"))
|
|
50
|
+
for name in names:
|
|
51
|
+
try:
|
|
52
|
+
zones.validate(name)
|
|
53
|
+
except ValueError as exc:
|
|
54
|
+
raise pytest.UsageError(f"--tzcheck-zones: {exc}") from None
|
|
55
|
+
config.pluginmanager.register(TzCheck(zones.dedupe(names)), "tzcheck-session")
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@pytest.fixture
|
|
59
|
+
def tzcheck_zone(request: pytest.FixtureRequest) -> str:
|
|
60
|
+
"""The timezone this test runs in: an IANA name, or "local" for the untouched environment."""
|
|
61
|
+
return getattr(request, "param", zones.LOCAL)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _zone_of(item: pytest.Item) -> str | None:
|
|
65
|
+
callspec = getattr(item, "callspec", None)
|
|
66
|
+
return callspec.params.get(FIXTURE) if callspec else None
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _base_nodeid(nodeid: str, zone: str) -> str:
|
|
70
|
+
suffix = f"tz={zone}]"
|
|
71
|
+
if not nodeid.endswith(suffix):
|
|
72
|
+
return nodeid
|
|
73
|
+
head = nodeid[: -len(suffix)]
|
|
74
|
+
return head[:-1] if head.endswith("[") else head[:-1] + "]"
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _outcome(report: pytest.TestReport) -> str | None:
|
|
78
|
+
if hasattr(report, "wasxfail"):
|
|
79
|
+
return "xfailed" if report.skipped else "xpassed"
|
|
80
|
+
if report.failed:
|
|
81
|
+
return "failed" if report.when == "call" else "error"
|
|
82
|
+
if report.skipped:
|
|
83
|
+
return "skipped"
|
|
84
|
+
return "passed" if report.when == "call" else None
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class TzCheck:
|
|
88
|
+
def __init__(self, names: list[str]) -> None:
|
|
89
|
+
self.names = names
|
|
90
|
+
self.local = zones.local_name()
|
|
91
|
+
self.results: dict[str, dict[str, str]] = defaultdict(dict)
|
|
92
|
+
|
|
93
|
+
def label(self, zone: str) -> str:
|
|
94
|
+
return f"{zone} ({self.local})" if zone == zones.LOCAL else zone
|
|
95
|
+
|
|
96
|
+
def pytest_report_header(self) -> str:
|
|
97
|
+
if len(self.names) < 2:
|
|
98
|
+
return (
|
|
99
|
+
f"tzcheck: only one distinct zone ({self.label(self.names[0])}), nothing to compare"
|
|
100
|
+
)
|
|
101
|
+
return "tzcheck: " + ", ".join(self.label(z) for z in self.names)
|
|
102
|
+
|
|
103
|
+
@pytest.hookimpl(trylast=True)
|
|
104
|
+
def pytest_generate_tests(self, metafunc: pytest.Metafunc) -> None:
|
|
105
|
+
if len(self.names) < 2 or metafunc.definition.get_closest_marker("no_tzcheck"):
|
|
106
|
+
return
|
|
107
|
+
if FIXTURE not in metafunc.fixturenames:
|
|
108
|
+
metafunc.fixturenames.append(FIXTURE)
|
|
109
|
+
metafunc.parametrize(
|
|
110
|
+
FIXTURE, self.names, indirect=True, ids=[f"tz={z}" for z in self.names]
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
@pytest.hookimpl(hookwrapper=True)
|
|
114
|
+
def pytest_runtest_protocol(self, item: pytest.Item, nextitem: pytest.Item | None):
|
|
115
|
+
zone = _zone_of(item)
|
|
116
|
+
if zone is None:
|
|
117
|
+
yield
|
|
118
|
+
return
|
|
119
|
+
item.user_properties.append((PROP_ZONE, zone))
|
|
120
|
+
item.user_properties.append((PROP_BASE, _base_nodeid(item.nodeid, zone)))
|
|
121
|
+
with zones.set_local_zone(zone):
|
|
122
|
+
yield
|
|
123
|
+
|
|
124
|
+
def pytest_runtest_logreport(self, report: pytest.TestReport) -> None:
|
|
125
|
+
props = dict(report.user_properties)
|
|
126
|
+
zone, base = props.get(PROP_ZONE), props.get(PROP_BASE)
|
|
127
|
+
outcome = _outcome(report)
|
|
128
|
+
if zone is None or base is None or outcome is None:
|
|
129
|
+
return
|
|
130
|
+
current = self.results[base].get(zone)
|
|
131
|
+
if current is None or _RANK[outcome] > _RANK[current]:
|
|
132
|
+
self.results[base][zone] = outcome
|
|
133
|
+
|
|
134
|
+
def dependent(self) -> dict[str, dict[str, str]]:
|
|
135
|
+
return {
|
|
136
|
+
base: by_zone
|
|
137
|
+
for base, by_zone in self.results.items()
|
|
138
|
+
if len(set(by_zone.values())) > 1
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
def pytest_terminal_summary(self, terminalreporter) -> None:
|
|
142
|
+
if not self.results:
|
|
143
|
+
return
|
|
144
|
+
dependent = self.dependent()
|
|
145
|
+
if not dependent:
|
|
146
|
+
terminalreporter.write_line(
|
|
147
|
+
f"tzcheck: no timezone-dependent tests "
|
|
148
|
+
f"({len(self.results)} tests x {len(self.names)} zones)"
|
|
149
|
+
)
|
|
150
|
+
return
|
|
151
|
+
terminalreporter.section("timezone-dependent tests", red=True, bold=True)
|
|
152
|
+
for base in sorted(dependent):
|
|
153
|
+
by_outcome: dict[str, list[str]] = defaultdict(list)
|
|
154
|
+
for zone in self.names:
|
|
155
|
+
if zone in dependent[base]:
|
|
156
|
+
by_outcome[dependent[base][zone]].append(self.label(zone))
|
|
157
|
+
parts = " ".join(
|
|
158
|
+
f"{outcome}: {', '.join(labels)}" for outcome, labels in by_outcome.items()
|
|
159
|
+
)
|
|
160
|
+
terminalreporter.write_line(f"{base} {parts}")
|
|
161
|
+
terminalreporter.write_line(
|
|
162
|
+
f"{len(dependent)} of {len(self.results)} tests changed outcome with the timezone",
|
|
163
|
+
red=True,
|
|
164
|
+
)
|